session_settlement_recovery.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. """Fence an unsettled claim in a live owner without inventing a terminal result."""
  2. import asyncio
  3. import logging
  4. import sqlite3
  5. from gateway.session_producer_lifecycle import drop_producer_turn
  6. from hermes_state_runtime import RuntimeStoreError, _admission, _epoch, get_session_admission, settle_steer_transfers
  7. logger = logging.getLogger(__name__)
  8. def _mark_unknown(authority, row):
  9. transfers = _transfers(authority, authority.sessions.get(row['target_session_id']), row)
  10. def write(conn):
  11. _epoch(conn, authority.epoch)
  12. current = _admission(conn, row['admission_id'])
  13. if current['owner_epoch'] != authority.epoch or current['generation'] != row['generation']:
  14. raise RuntimeStoreError('stale_generation')
  15. if current['status'] in {'terminal', 'unknown'}:
  16. return current['status']
  17. if current['status'] != 'started':
  18. raise RuntimeStoreError('stale_generation')
  19. conn.execute("UPDATE session_admissions SET status='unknown' WHERE admission_id=?", (row['admission_id'],))
  20. # A steer it carried is never replayed: interrupted if delivered, else queued again.
  21. settle_steer_transfers(conn, epoch=authority.epoch, generation=row['generation'],
  22. transfers=transfers, outcome='unknown')
  23. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (row['target_session_id'],))
  24. return 'unknown'
  25. return authority.db._execute_write(write)
  26. def _transfers(authority, live, row):
  27. """The steers handed to this claim (``prompt.steer_queued``), read once its execution returned."""
  28. if live is None:
  29. return None
  30. from gateway.session_queued_controls import steer_transfers
  31. with live.event_stream.lock:
  32. return steer_transfers(authority, row['target_session_id'], row['generation'])
  33. async def recover_failed_settlement(authority, row):
  34. # A full disk or held SQLite writer may also block the recovery stamp. Retain this owner
  35. # and its captured result until the storage fence can commit; never manufacture a reply.
  36. while True:
  37. try:
  38. from gateway.session_runtime_workers import track_mutation
  39. writer = track_mutation(authority, asyncio.to_thread(_mark_unknown, authority, row))
  40. return await asyncio.shield(writer)
  41. except (OSError, sqlite3.Error):
  42. await asyncio.sleep(1)
  43. def completion_payload(row, settled, response, captured):
  44. # ``status`` is the message.complete contract's TurnStatus: the Desktop
  45. # extends a Stopped bubble to the persisted partial only on 'interrupted'.
  46. complete = {
  47. 'text': response, 'content': response, 'admission_id': row['admission_id'],
  48. 'outcome': 'cancelled' if settled['outcome'] == 'interrupted' else settled['outcome'],
  49. 'status': {'completed': 'complete', 'interrupted': 'interrupted'}.get(
  50. settled['outcome'], 'error')}
  51. # Only the agent's reuse site sets this (never inferred from equal text): the
  52. # final repeats a reply the viewer already painted, so it settles in place.
  53. captured_result = (captured or {}).get('result') or {}
  54. # A gateway-owned refusal brings its own descriptor (``session_cron.execute``); every other failed
  55. # turn gets the one the TUI path builds from the same result, so the viewer shows the right card.
  56. surface = captured_result.get('error_surface')
  57. if not isinstance(surface, dict) and complete['status'] == 'error':
  58. from agent.error_surface import build_error_surface_from_result
  59. surface = build_error_surface_from_result(
  60. captured_result, provider=str(captured_result.get('provider') or ''),
  61. model=str(captured_result.get('model') or ''))
  62. if isinstance(surface, dict):
  63. complete['error_surface'] = surface
  64. if response and captured_result.get('response_reused'):
  65. complete['response_reused'] = True
  66. if captured_result.get('local_viewer'):
  67. if captured_result.get('response_previewed'):
  68. complete['response_previewed'] = True
  69. if captured_result.get('response_transformed'):
  70. complete['response_transformed'] = True
  71. # The stream carried the raw reply and the turn counts as delivered (``response`` is
  72. # None): this frame is the only carrier of the rewrite the session stored.
  73. if not response and (final := captured_result.get('final_response')):
  74. complete['text'] = complete['content'] = final
  75. if isinstance(captured_result.get('warning'), str) and captured_result['warning']:
  76. complete['warning'] = captured_result['warning']
  77. if isinstance(captured_result.get('billing_block'), dict):
  78. complete['billing'] = captured_result['billing_block']
  79. if isinstance(captured_result.get('failure_reason'), str) and captured_result['failure_reason']:
  80. complete['failure_reason'] = captured_result['failure_reason']
  81. if isinstance(captured_result.get('last_reasoning'), str) and captured_result['last_reasoning'].strip():
  82. complete['reasoning'] = captured_result['last_reasoning'].strip()
  83. # The committed row addresses of the turn: a viewer binds the streamed reply to
  84. # its stored row, so a transcript read racing this frame never paints it twice.
  85. # ``submission_id`` names whose turn it is: the sending viewer binds its optimistic
  86. # prompt (``user-<submission_id>``), which the queued admission ack could not name.
  87. # A local viewer's turn end recorded the session-cumulative stats (``session_usage.publish_usage``):
  88. # main's TUI read ``message.complete.usage`` for its status bar and ``/usage`` totals.
  89. if isinstance((captured or {}).get('session_usage'), dict):
  90. from gateway.session_usage import tui_usage
  91. complete['usage'] = tui_usage(captured['session_usage'])
  92. if isinstance(captured_result.get('persisted_turn'), dict):
  93. complete['persisted_turn'] = {**captured_result['persisted_turn'],
  94. 'submission_id': row['request_id']}
  95. return complete
  96. def _publish_completion(authority, live, ref, row, settled, response, captured):
  97. """Thread-safe half of terminal publication; the caller holds the stream lock. No SQLite write
  98. happens here: loop-side readers wait on this lock, so it never spans storage I/O."""
  99. try:
  100. live.controls.snapshot(ref.session_id, None)
  101. except Exception:
  102. logger.exception('Terminal control cleanup failed for admission %s', row['admission_id'])
  103. complete = completion_payload(row, settled, response, captured)
  104. notice = owner_stop_notice(captured, settled['outcome'], row)
  105. if notice and not complete.get('warning'):
  106. # ``warning`` is the slot every surface already renders on message.complete (Ink activity
  107. # line, Desktop toast), so a turn the owner's stop cut short says why (#134889 F1).
  108. complete['warning'] = notice
  109. live.event_stream.publish(ref.session_id, complete)
  110. RESTART_INTERRUPT_NOTICE = ('Gateway restarting: this turn was interrupted and will not be run again. '
  111. 'Send your message again once Hermes reconnects.')
  112. SHUTDOWN_INTERRUPT_NOTICE = ('Gateway shutting down: this turn was interrupted and will not be run again. '
  113. 'Send your message again once Hermes is back.')
  114. def owner_stop_notice(captured, outcome, row=None):
  115. """The named notice for a turn the owner's restart/stop interrupted, or None. With the default
  116. ``agent.restart_drain_timeout: 0`` a restart interrupts in-flight turns at once; the admission
  117. settles ``interrupted`` and is never replayed, so the viewer is told the restart did it.
  118. An unattended ``row`` (cron, automation, webhook) has nobody to resend it, so it gets the
  119. neutral automation copy instead of resend advice (#134889 GW-20, GW-10's rule)."""
  120. # The cause is the one the turn recorded when it was interrupted (stop() interrupts with these
  121. # reasons), never the runner's flags at publication: a user Stop whose settlement was delayed
  122. # until after shutdown began is still the user's Stop.
  123. from gateway.run import _INTERRUPT_REASON_GATEWAY_RESTART, _INTERRUPT_REASON_GATEWAY_SHUTDOWN
  124. cause = ((captured or {}).get('result') or {}).get('interrupt_message') if outcome == 'interrupted' else None
  125. notice = {_INTERRUPT_REASON_GATEWAY_RESTART: RESTART_INTERRUPT_NOTICE,
  126. _INTERRUPT_REASON_GATEWAY_SHUTDOWN: SHUTDOWN_INTERRUPT_NOTICE}.get(cause)
  127. if notice is not None and row is not None:
  128. from gateway.session_unattended_unknown import is_unattended
  129. if is_unattended(row):
  130. from agent.turn_failure_copy import AUTOMATION_INTERRUPTED_NOTICE
  131. return AUTOMATION_INTERRUPTED_NOTICE
  132. return notice
  133. def gateway_refusal_boundary(captured, outcome):
  134. """``(notice, display_metadata)`` for a failed turn the gateway itself refused with a renderable
  135. descriptor (``result.error_surface``: ``session_cron.execute``, ``session_policy.refused_turn_result``), else None."""
  136. value = (captured or {}).get('result') or {}
  137. surface, error = value.get('error_surface'), value.get('error')
  138. if outcome != 'failed' or not isinstance(surface, dict) or not isinstance(error, str) or not error:
  139. return None
  140. return error, {'error': error, 'error_surface': dict(surface)}
  141. def _release_media(authority, admission_id):
  142. """Terminal media cleanup: its own write txn after publication, never under the stream lock."""
  143. try:
  144. from gateway.session_ingress_media import release_admission_media
  145. release_admission_media(authority.db, admission_id)
  146. except Exception:
  147. logger.exception('Terminal media cleanup failed for admission %s', admission_id)
  148. def commit_and_publish(authority, live, ref, row, response, outcome, captured, settlement):
  149. """Worker-thread settlement: redaction, encoding and the SQLite write never stall the owner loop.
  150. The write runs OUTSIDE ``event_stream.lock``: attach, replay, receipts and pending publication
  151. take that lock on the loop, and a writer held by another process (up to ``_WRITE_PATIENCE_S``)
  152. would stall every session behind it. Ordering is kept by ``live.settling``: from before the
  153. write until the completion frame is published, lock holders read this claim as still started,
  154. so none sees the terminal row without its completion. ``settlement`` receives the committed
  155. ``(settled, response)`` before publication, so a publish failure is not a lost commit.
  156. The caller publishes idle ``session.info`` on the loop afterwards (bot receipt wakeups are tasks).
  157. """
  158. from gateway.session_results import finish_result, prepare_result
  159. # Compaction, redaction and outcome flags are CPU-only.
  160. prepared = prepare_result(row, response, outcome, captured)
  161. admission_id = row['admission_id']
  162. # ``live`` is the drain's own entry: a delete committing on the loop after this write may
  163. # already have dropped it from ``authority.sessions``.
  164. with live.event_stream.lock:
  165. live.settling[admission_id] = row
  166. transfers = _transfers(authority, live, row)
  167. try:
  168. settlement['settled'], settlement['response'] = finish_result(
  169. authority.db, epoch=authority.epoch, row=row, response=response, outcome=outcome,
  170. result=captured, prepared=prepared, transfers=transfers)
  171. with live.event_stream.lock:
  172. live.settling.pop(admission_id, None)
  173. _publish_completion(authority, live, ref, row, settlement['settled'], settlement['response'], captured)
  174. finally:
  175. with live.event_stream.lock:
  176. live.settling.pop(admission_id, None)
  177. # Pauses between settlement attempts after a transient storage error (held writer past its
  178. # patience, full disk). Bounded: the attempts after the last one fall back to the unknown fence.
  179. _SETTLE_RETRY_DELAYS_S = (0.5, 2.0)
  180. async def settle_with_retry(authority, live, ref, row, response, outcome, captured, settlement):
  181. """Commit the captured result, retrying a transient storage failure a bounded number of times.
  182. One locked or full write must not turn a finished answer into an ``unknown`` turn whose result
  183. lives only in memory. Retrying is exact: a write that committed before reporting failure is
  184. read back by ``finish_result``, and a failure after the commit (``settled`` set) never retries.
  185. A fence refusal (``RuntimeStoreError``) is not transient and is raised at once."""
  186. from gateway.session_runtime_workers import track_mutation
  187. for delay in (*_SETTLE_RETRY_DELAYS_S, None):
  188. try:
  189. return await asyncio.shield(track_mutation(authority, asyncio.to_thread(
  190. commit_and_publish, authority, live, ref, row, response, outcome, captured, settlement)))
  191. except (OSError, sqlite3.Error) as exc:
  192. if delay is None or 'settled' in settlement:
  193. raise
  194. logger.warning('Settlement of admission %s deferred by storage (%r); retrying in %.1fs',
  195. row['admission_id'], exc, delay)
  196. await asyncio.sleep(delay)
  197. def publish_terminal(authority, ref, row, settled, response, captured):
  198. """Cleanup cannot withhold a committed outcome; publication is attempted once."""
  199. live = authority.sessions[ref.session_id]
  200. with live.event_stream.lock:
  201. _publish_completion(authority, live, ref, row, settled, response, captured)
  202. # Idle follows the completion so a viewer cannot mistake a settled turn for a lost frame.
  203. authority._publish_pending(ref)
  204. async def settle_and_deliver(authority, live, ref, row, response, outcome):
  205. """Own settlement through delivery even if the drain's waiter is cancelled.
  206. Terminal media collection can scan a multi-GB transcript. Start it only after
  207. delivery (or a failed attempt), tracked separately so it cannot delay the FIFO.
  208. """
  209. admission_id = row['admission_id']
  210. settled, discarded = None, False
  211. try:
  212. settlement = {}
  213. try:
  214. # Redaction, encoding and the SQLite write run off-loop: one turn's settlement must
  215. # not stall every other session. Publication takes the stream lock only after
  216. # the write commits; this owner also outlives a cancelled drain.
  217. captured = authority.pending_results.get(admission_id)
  218. try:
  219. await settle_with_retry(authority, live, ref, row, response, outcome, captured, settlement)
  220. finally:
  221. if 'settled' in settlement:
  222. settled, response = settlement['settled'], settlement['response']
  223. authority.pending_results.pop(admission_id, None)
  224. # Idle follows the completion; its bot receipt wakeups are loop tasks. A delete
  225. # that committed while the write yielded already retired this live entry.
  226. if authority.sessions.get(ref.session_id) is live:
  227. authority._publish_pending(ref)
  228. except Exception:
  229. logger.exception(
  230. 'Settlement or publication of admission %s failed', admission_id)
  231. if settled is None:
  232. authority.pending_results.setdefault(admission_id, {
  233. 'result': {'final_response': response, 'failed': outcome == 'failed'}, 'usage': {}})
  234. try:
  235. status = await recover_failed_settlement(authority, row)
  236. except RuntimeStoreError:
  237. # Ownership moved; this owner may only release its process-local observers.
  238. authority._pause(ref, 'unknown_execution')
  239. return False
  240. # Discard can race the off-loop recovery stamp; its terminal decision wins.
  241. current = get_session_admission(authority.db, admission_id=admission_id)
  242. status = current['status']
  243. if status == 'unknown':
  244. from gateway.session_ingress import retain_unsettled_delivery
  245. retain_unsettled_delivery(authority, admission_id)
  246. authority._publish_pending(ref)
  247. authority._pause(ref, 'unknown_execution')
  248. return False
  249. # A write can commit before reporting failure. Re-read its exact result,
  250. # then publish the terminal outcome rather than inventing uncertainty.
  251. from gateway.session_results import admission_result
  252. with live.event_stream.lock:
  253. captured = admission_result(authority.db, admission_id)
  254. authority.pending_results.pop(admission_id, None)
  255. settled = current
  256. if captured is None:
  257. # An operator discarded uncertainty while recovery yielded. There is
  258. # no committed answer to replay; its queued successor may now run.
  259. authority.pending_deliveries.pop(admission_id, None)
  260. waiter = authority.waiters.pop(admission_id, None)
  261. if waiter is not None and not waiter.done():
  262. waiter.set_exception(RuntimeStoreError('unknown_execution'))
  263. discarded = True
  264. return True
  265. response = captured['result'].get('final_response') or ''
  266. publish_terminal(authority, ref, row, settled, response, captured)
  267. finally:
  268. # The stamp names a claimed, unsettled execution. Left in place, idle
  269. # mutations (session.updated) would carry a terminal generation and
  270. # a versioned viewer fence would discard them as late frames.
  271. with live.event_stream.lock:
  272. live.event_stream.execution = {}
  273. authority.pending_stops.pop(ref.session_id, None)
  274. authority.pending_stop_causes.pop(ref.session_id, None)
  275. authority.pending_steers.pop(ref.session_id, None)
  276. authority.adopted.pop(ref.session_id, None)
  277. from gateway.session_queued_controls import publish_transfers
  278. publish_transfers(authority, ref, row['generation'])
  279. if settled is None or discarded:
  280. # Uncommitted (unknown or discarded): the user is not told an answer the FIFO lost.
  281. authority.pending_deliveries.pop(admission_id, None)
  282. drop_producer_turn(authority, admission_id)
  283. from gateway.session_ingress import fire_post_delivery
  284. await fire_post_delivery(authority, admission_id)
  285. from gateway.session_ingress import waiter_replies
  286. waiter_replies(authority).pop(admission_id, None)
  287. # Taken on every exit, so a turn that never settled leaves no reset for a later one.
  288. reset = authority.deferred_resets.pop(live.route, None)
  289. await authority._run_deferred_reset(live.route, reset)
  290. from gateway.session_ingress import deliver_settled, fire_post_delivery
  291. await deliver_settled(authority, admission_id)
  292. await fire_post_delivery(authority, admission_id)
  293. await authority._resolve_settled_waiter(admission_id, response)
  294. return True
  295. finally:
  296. # A resolution that raced this drain handed its collection here (``_resolved_unknown``):
  297. # honour it on every exit, settled or not.
  298. owed = admission_id in getattr(authority, '_drain_owned_collection', ())
  299. if owed:
  300. authority._drain_owned_collection.discard(admission_id)
  301. if settled is not None or owed:
  302. release_media_after_delivery(authority, row)
  303. def hand_collection_to_drain(authority, admission_id):
  304. """The drain still owns this admission's claim: its exit queues the media collection."""
  305. owed = getattr(authority, '_drain_owned_collection', None)
  306. if owed is None:
  307. owed = authority._drain_owned_collection = set()
  308. owed.add(admission_id)
  309. def release_media_after_delivery(authority, row):
  310. """Queue terminal media collection on the authority's one off-loop collector.
  311. Each collection can scan the whole transcript; running them one at a time bounds the worker
  312. threads and pooled read connections they hold however many media turns settle meanwhile.
  313. The collector is tracked, so shutdown and profile retirement join it. Only the admission id
  314. waits: the row can carry inline images and a client's whole conversation history."""
  315. queue = getattr(authority, '_media_release_queue', None)
  316. if queue is None:
  317. queue = authority._media_release_queue = []
  318. queue.append(row['admission_id'])
  319. task = getattr(authority, '_media_release_task', None)
  320. if task is None or task.done():
  321. from gateway.session_runtime_workers import track_mutation
  322. authority._media_release_task = track_mutation(authority, _collect_queued_media(authority, queue))
  323. async def _collect_queued_media(authority, queue):
  324. while queue:
  325. await asyncio.to_thread(_release_media, authority, queue.pop(0))