session_authority.py 89 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486
  1. """Gateway-owned live sessions and canonical durable FIFO scheduling.
  2. Only the runtime bootstrap holding profile ownership may initialize this service.
  3. Transport attachment never constructs an agent or takes a turn lease.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import contextlib
  8. from contextvars import ContextVar
  9. from dataclasses import asdict, dataclass, field
  10. from functools import partial
  11. import sqlite3
  12. from types import SimpleNamespace
  13. import uuid
  14. from gateway.session_contract import (
  15. CANONICAL_GATEWAY_PROTOCOL, AdmissionReceipt, PendingAdmission, SessionActor, SessionHandle, SessionRef, Submission,
  16. SubscriptionSnapshot,
  17. )
  18. from gateway.session_events import SessionEvents
  19. from gateway.session_pending_controls import PendingControls
  20. from gateway.session_producer_lifecycle import drop_producer_turn, mark_producer_admission
  21. from hermes_cli.observability.shared_metrics_busy import busy_row_settled
  22. from hermes_state_runtime import (
  23. STEERING, RuntimeStoreError, admit_session_input, begin_runtime_epoch,
  24. cancel_session_input, claim_session_input, get_session_admission,
  25. list_session_admissions, recover_session_inputs, resolve_unknown_session_input,
  26. )
  27. # Capped backoff for a claim refused by transient storage (writer lock, full disk).
  28. _CLAIM_RETRY_MIN_S, _CLAIM_RETRY_MAX_S = 0.25, 5.0
  29. # Longest a settled turn's adapter may hold its adopted crash marker before the follower is claimed.
  30. _MARKER_RELEASE_WAIT_S = 60.0
  31. # True in the admitting task once ``admit_native``'s shielded ledger write began: a caller cancelled
  32. # after that may still see the row commit, so it must not release the media the write commits.
  33. native_write_started = ContextVar('native_write_started', default=False)
  34. # Set by the messaging ingress around ``admit_native``: True, or the busy caller's acceptance future.
  35. # Such an input is fenced by a later /stop or /new of its route (``fence_route``), and the future is
  36. # released once the input queues behind an earlier one of its chat, so a serial reader moves on
  37. # while acceptance itself still waits for the commit.
  38. messaging_admission = ContextVar('messaging_admission', default=None)
  39. @dataclass
  40. class LiveSession:
  41. source: object
  42. route: str
  43. task: asyncio.Task | None = None
  44. subscribers: dict = field(default_factory=dict)
  45. event_stream: SessionEvents = field(default_factory=SessionEvents)
  46. controls: PendingControls = field(init=False)
  47. # The messaging ingress tells the platform user once per pause episode (unknown head,
  48. # preflight refusal), not per message; the drain clears it when the FIFO moves again.
  49. pause_notified: bool = False
  50. mutation_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
  51. # Held by the FIFO claim and by compress across its summarization (which runs outside
  52. # ``mutation_lock``, so admissions are not parked behind an LLM call): no turn is claimed
  53. # under a compression that is still preparing its replacement transcript.
  54. claim_gate: asyncio.Lock = field(default_factory=asyncio.Lock)
  55. # Set by ``_schedule`` while a drain is alive. The claim commits off-loop, so input admitted
  56. # after its transaction read the FIFO would otherwise be stranded by a drain that exits idle.
  57. rescan: bool = False
  58. # admission id -> its started claim, from the moment the settlement worker starts the terminal
  59. # write until the completion frame is published. The write runs outside ``event_stream.lock``
  60. # (a contended SQLite writer must not stall loop-side lock takers); readers holding that lock
  61. # still see the claim as started, so no snapshot shows a terminal row without its completion.
  62. settling: dict = field(default_factory=dict)
  63. # ``(revision key, payload)`` of the last ``session.context_breakdown``: Desktop re-polls it on
  64. # every idle mount and reconnect, and a rebuild walks the whole stored transcript (#140667).
  65. context_breakdown: tuple | None = None
  66. # A viewer attached while a guarded deferred pick awaits consent: the next boundary asks again.
  67. reask_pick: bool = False
  68. # (principal, effort) of a ``/reasoning`` command that arrived while a turn ran: applied in memory at
  69. # once, committed to the policy at the next boundary (gateway/session_setting_commands.py).
  70. pending_reasoning: tuple | None = None
  71. # The background-review guard of a cold ``/refine`` (gateway/session_refine.py): its side agent is
  72. # per call, so the run token lives here; a second /refine joins it and the next turn cancels it.
  73. review: SimpleNamespace = field(default_factory=SimpleNamespace)
  74. # Revision of the last ``session.control.update`` (gateway/session_control_actions.publish): a tool
  75. # completion republishes only when a tool actually moved the goal/loop/heartbeat state.
  76. control_revision: str = ""
  77. # Tickets ordering control publishes whose snapshot is read off the loop: the last one scheduled
  78. # wins, so a slow read cannot land over a newer state (gateway/session_control_actions.publish).
  79. control_seq: int = 0
  80. control_published_seq: int = 0
  81. # ``(execution stamp,)`` the last viewer close completed at: a drain alive then retires the closed
  82. # conversation at its idle exit unless a turn claimed since (gateway/session_retirement.py).
  83. closed_stamp: tuple | None = None
  84. # A conversation retirement let go that an RPC brought back: its re-retirement check
  85. # (``session_retirement.touch_restored``), pushed back by each authorize.
  86. retire_timer: object = None
  87. # Last-viewer detaches between losing their subscriber and recording the close (``detach``).
  88. detaching: int = 0
  89. def __post_init__(self):
  90. self.controls = PendingControls(self.event_stream)
  91. @property
  92. def detached(self):
  93. """A messaging conversation its chat moved past (``name_lane``): its turns and Stop act on
  94. it alone, on its own key, and reach its app viewers only (gateway/session_detached.py)."""
  95. from gateway.session_detached import detached_route_of
  96. return detached_route_of(self.source) is not None
  97. def _log_drain_failure(task):
  98. """A dead pump is the one failure this module must never swallow."""
  99. if task.cancelled() or task.exception() is None:
  100. return
  101. import logging
  102. logging.getLogger(__name__).error('Session drain task died: %r', task.exception())
  103. @contextlib.asynccontextmanager
  104. async def keyed_turn(table, key, *, queued=None):
  105. """One FIFO turn per *key*: ``table[key]`` is ``[asyncio.Lock(), holder + waiters]``, created on
  106. first use and deleted when the last one leaves. asyncio's Lock is FIFO and a cancelled waiter gives
  107. up only its own place. *queued* runs (before waiting) when an earlier turn holds or awaits the key."""
  108. entry = table.setdefault(key, [asyncio.Lock(), 0])
  109. entry[1] += 1
  110. if queued is not None and entry[1] > 1:
  111. queued()
  112. try:
  113. async with entry[0]:
  114. yield
  115. finally:
  116. entry[1] -= 1
  117. if not entry[1]:
  118. del table[key]
  119. class SessionAuthority:
  120. def __init__(self, runner, *, profile_id, instance_id, db, epoch):
  121. self.runner = runner
  122. self.profile_id = profile_id
  123. self.instance_id = instance_id
  124. self.db = db
  125. self.epoch = epoch
  126. self.sessions = {}
  127. self.waiters = {}
  128. self.events = {}
  129. self.native_waiters = set()
  130. self.pending_results = {}
  131. # Replies of recovered no-waiter turns, sent only once their terminal outcome commits.
  132. self.pending_deliveries = {}
  133. # Stops accepted for a running generation whose agent does not exist yet
  134. # (first-turn construction); consumed by adopt_agent, keyed session -> generation.
  135. self.pending_stops = {}
  136. # session -> (generation, reason): a gateway restart/shutdown Stop latched before that
  137. # generation's managed worker existed; its pre-bootstrap result carries the cause (GW-22).
  138. self.pending_stop_causes = {}
  139. # session -> (generation, [steer text]) accepted before that generation adopted an agent;
  140. # adopt_agent hands it to that agent (never a later generation's), the drain drops the rest.
  141. self.pending_steers = {}
  142. # session -> (generation, {admission_id: delivered}): queued admissions prompt.steer_queued handed
  143. # to that running claim (gateway/session_queued_controls.py); its settlement decides each one.
  144. self.steer_transfers = {}
  145. self.launch_sources = {} # session -> its frozen launch source (session_queued_controls)
  146. # Closed local conversations ``session_retirement`` let go (a restored one may linger here until
  147. # it is deleted or retired again): their source and frozen policy stay on the local adapter.
  148. self.retired_local = set()
  149. # Every conversation ``session_retirement`` let go and not yet deleted -> its route: a delete or
  150. # prune of one not in memory still evicts that route's cached agent.
  151. self.retired_routes = {}
  152. # session -> generation of the last Stop delivered to a live agent. A repeated
  153. # Stop can land on the session's reusable cached agent after that turn's finalizer
  154. # cleared it; the next generation's adopt_agent drops it instead of starting interrupted.
  155. self.delivered_stops = {}
  156. # session -> (generation, agent) its turn adopted. Until then the resident agent is the
  157. # previous turn's cached one, which pre-turn compression or a changed agent config
  158. # replaces, so a Stop that only reached it would let the rebuilt agent run the turn. After
  159. # it, a Stop goes to this agent, never through a cache that may have evicted it.
  160. self.adopted = {}
  161. # Queued admissions whose cancellation may have committed before its observers settled.
  162. self.cancel_obligations = set()
  163. # Set by profile retirement (unserve): the drain claims no successor after its running turn.
  164. self.retiring = False
  165. self.retired = False
  166. self._classic_evict_at = {} # route -> last leaked/reaped-slot eviction check (monotonic)
  167. self.executing = {} # session id -> generation whose execution is in progress (publish fence)
  168. # session id -> end_reason of a last-viewer close still waiting for an idle FIFO
  169. # (gateway/session_viewer_close); a viewer attaching again cancels it.
  170. self.pending_closes = {}
  171. self.orphan_closes = {} # session id -> grace timer of a session left with no viewer, never closed
  172. # Logical owner rotated away by the exhaustion auto-reset -> its session key: an admission
  173. # that reserved it before the rotation commits on that key's current route instead.
  174. self.rotated_owners = {}
  175. # Route key -> [lock, holder + waiters] of the native admissions in flight (``_route_turn``, ``keyed_turn``).
  176. self._route_turns = {}
  177. # Per route: how many messaging /stop and /new fenced it (``fence_route``). Kept for the
  178. # owner's life (one int per chat that ever got one): an input or command turn holding an older
  179. # value never commits, and a reset entry could read as an old epoch again.
  180. self._route_epochs = {}
  181. # route -> the compression-exhaustion auto-reset its LOCAL turn deferred: the store refuses a
  182. # reset while that turn's admission is started, so the drain runs it once the turn settles.
  183. self.deferred_resets = {}
  184. def authorize(self, actor, ref, capability):
  185. """Every handler calls this first, so a later ``self.sessions[ref.session_id]`` is
  186. safe: a deleted/evicted live entry surfaces here as ``not_found``, not as a KeyError
  187. deeper in the handler."""
  188. from gateway.session_authorities import require_serving
  189. require_serving(self)
  190. if actor.profile_id != self.profile_id or ref.profile_id != self.profile_id:
  191. raise RuntimeStoreError('profile_mismatch')
  192. if capability not in actor.capabilities:
  193. raise RuntimeStoreError('permission_denied')
  194. if ref.session_id not in self.sessions:
  195. row = self.db.get_session(ref.session_id)
  196. if row is None:
  197. raise RuntimeStoreError('not_found')
  198. from hermes_state_local import POLICY_PREFIX
  199. with self.db._read_ctx() as conn:
  200. from hermes_state_local_migration import LEGACY_PREFIX
  201. local = conn.execute('SELECT 1 FROM state_meta WHERE key IN (?,?)',
  202. (POLICY_PREFIX + ref.session_id, LEGACY_PREFIX + ref.session_id)).fetchone()
  203. if local or str(row.get('chat_id') or '').startswith('local-'):
  204. from gateway.session_local_recovery import restore_local_session
  205. restore_local_session(self, ref.session_id)
  206. else:
  207. from gateway.session_route_recovery import restore_routed_session
  208. from gateway.session_history_mount import mount_history
  209. if (restore_routed_session(self, ref.session_id) is None
  210. and not mount_history(self, actor, ref.session_id)):
  211. from gateway.session_api import restore_api_session
  212. restore_api_session(self, ref.session_id)
  213. from gateway.session_history_mount import is_history_mount, require_history_access
  214. if is_history_mount(self.sessions[ref.session_id]):
  215. require_history_access(self, actor, ref.session_id, capability)
  216. from gateway.config import Platform
  217. source = self.sessions[ref.session_id].source
  218. from gateway.owner_identity import same_owner
  219. if (source is not None and source.platform == Platform.LOCAL
  220. and not same_owner(source.user_id, actor.subject) and 'session:operator' not in actor.capabilities):
  221. raise RuntimeStoreError('permission_denied')
  222. from gateway.session_retirement import touch_restored
  223. touch_restored(self, ref.session_id) # a rename or a read brought a retired one back
  224. def _require_admission_open(self, *, queue_only=False):
  225. """``queue_only``: the caller only COMMITS a queued row (never claims or runs it). A restart's
  226. after-turn wait accepts those (#139041): the row is durable FIFO input the next owner runs
  227. exactly once, while this owner's claim path stays refused."""
  228. if self.retiring:
  229. raise RuntimeStoreError('runtime_draining')
  230. if self.runner._draining:
  231. from gateway.drain_control import restart_wait_open
  232. if not (queue_only and restart_wait_open(self.runner)):
  233. raise RuntimeStoreError('runtime_draining')
  234. def _admission_gate(self, authorize=None, *, queue_only=False):
  235. """The admission gate re-checked inside an off-loop write transaction (with the caller's
  236. own guard), so a drain that began while the write waited on the writer still refuses it."""
  237. def guard(conn):
  238. self._require_admission_open(queue_only=queue_only)
  239. if authorize is not None:
  240. authorize(conn)
  241. return guard
  242. def logical_owner(self, session_id):
  243. """The FIFO/admission identity of a route: the root of its compression lineage.
  244. Compression advances the physical transcript, never the admission identity. A session this
  245. store does not hold (another served profile's, under multiplex) keeps its own id."""
  246. if not session_id:
  247. return session_id
  248. lineage = self.db.get_compression_lineage(session_id)
  249. return lineage[0] if lineage else session_id
  250. def physical_target(self, ref):
  251. return self.db.get_compression_tip(ref.session_id) or ref.session_id
  252. def register(self, source):
  253. return self._publish_route(source, *self._reserve_route(source))
  254. def _reserve_route(self, source):
  255. """The durable half of ``register`` (routing entry + reserved row): thread-safe, so async
  256. callers run it off the owner loop and publish the live entry on the loop."""
  257. self._require_admission_open()
  258. entry = self.runner.session_store.get_or_create_session(source)
  259. sid = self.logical_owner(entry.session_id)
  260. # SessionStore reserves routing metadata before the first AIAgent exists.
  261. if self.db.get_session(sid) is None:
  262. self.db.create_session(sid, source=source.platform.value)
  263. return sid, entry.session_key
  264. def _publish_route(self, source, sid, route):
  265. self._require_admission_open()
  266. if sid not in self.sessions:
  267. self.sessions[sid] = LiveSession(source, route)
  268. self.name_lane(route, sid, source)
  269. return SessionRef(self.profile_id, sid)
  270. def name_lane(self, route, sid, source):
  271. """The messaging chat on *route* (*source*) now names conversation *sid* (its next inbound,
  272. ``/new``, ``/resume``, ``/branch``, ``/handoff``). Every other live session on the route (a
  273. view kept open, a cold mount, a local chat handed off earlier) detaches onto its own key: its
  274. app viewers keep using it, and nothing of it reaches the chat (gateway/session_detached.py).
  275. *sid*'s own mount moves onto the route: a detached one comes back, a local one ``/handoff``
  276. took over or one another chat held (a cross-chat ``/resume``) follows it, so its open viewers'
  277. turns and Stops reach the chat that holds it now (#143286)."""
  278. from gateway.session_detached import detach, forget_detached
  279. for other_sid, other in self.sessions.items():
  280. if other.route == route and other_sid != sid:
  281. detach(self, other, other_sid)
  282. if (live := self.sessions.get(sid)) is not None and live.route != route:
  283. left = live.route if live.detached else None
  284. live.source, live.route = source, route
  285. if left is not None:
  286. forget_detached(self, left)
  287. def agent(self, ref):
  288. from gateway.session_detached import running_key
  289. live = self.sessions[ref.session_id]
  290. lookup = getattr(self.runner, '_resident_agent_for', None) or self.runner._cached_agent_for
  291. return lookup(running_key(self.runner, live.route, ref.session_id))
  292. def _pending_rows(self, ref):
  293. """Pending admissions as the event stream presents them: a settlement committed but not yet
  294. published (``LiveSession.settling``) still reads as its started claim. Callers that pair the
  295. result with stream state hold ``event_stream.lock``, under which ``settling`` changes."""
  296. rows = list_session_admissions(self.db, session_id=ref.session_id)
  297. live = self.sessions.get(ref.session_id)
  298. masked = dict(live.settling) if live is not None else {}
  299. if not masked:
  300. return rows
  301. rows = {row['admission_id']: row for row in rows}
  302. rows.update(masked)
  303. return sorted(rows.values(), key=lambda row: row['seq'])
  304. def _handle(self, ref):
  305. row = self.db.get_session(ref.session_id)
  306. if row is None: # deleted after ``authorize``: the commit lands before the retirement runs
  307. raise RuntimeStoreError('not_found')
  308. pending = self._pending_rows(ref)
  309. state = 'unknown' if any(r['status'] == 'unknown' for r in pending) else (
  310. 'running' if any(r['status'] == 'started' for r in pending) else 'idle')
  311. return SessionHandle(ref, self.instance_id, self.epoch, row['runtime_revision'],
  312. row['runtime_generation'], state)
  313. async def resolve(self, actor, ref):
  314. self.authorize(actor, ref, 'session:read')
  315. return self._handle(ref)
  316. def unless_deleted(self, ref, read):
  317. """``read()``; a refusal because *ref* was deleted after ``authorize`` passed is ``not_found``.
  318. A delete commits off the loop, so its rows can be gone while the live entry still waits for its
  319. retirement, and readers meet the missing receipt instead."""
  320. try:
  321. return read()
  322. except RuntimeStoreError:
  323. if self.db.get_session(ref.session_id) is None:
  324. raise RuntimeStoreError('not_found') from None
  325. raise
  326. async def attach(self, actor, ref, *, history=True, replay_from=None):
  327. """Join *ref*'s event stream and snapshot it. ``history=False``: the snapshot carries no transcript
  328. (``history == ()``), only the physical ``transcript_target``, so the display-row lineage read is
  329. skipped; for a viewer that pages the transcript itself (Desktop over REST).
  330. *replay_from*: ``(epoch, sequence)`` this connection's reconnect replay answered (SessionEvents.catch_up)."""
  331. self.authorize(actor, ref, 'session:read')
  332. from gateway.session_local_recovery import local_history, prepare_history, transcript_target
  333. # The lineage read stays off the loop and the lock; without history the target is resolved under it.
  334. prepared = await prepare_history(self, ref) if history else None
  335. from gateway.session_onboarding import keep_from_discard
  336. # The last await before the viewer joins: a fresh launch's claim is taken back with no yield
  337. # before the join, and a delete already past it answers not_found below.
  338. await keep_from_discard(self, ref.session_id)
  339. # A delete or eviction can land during those awaits: re-check, so it answers not_found, never a KeyError.
  340. self.authorize(actor, ref, 'session:read')
  341. live = self.sessions[ref.session_id]
  342. from gateway.session_queued_controls import pending_rows
  343. with live.event_stream.lock:
  344. handle = self._handle(ref) # before the viewer joins: a delete committed meanwhile is not_found
  345. subscription = next((key for key, member in live.subscribers.items()
  346. if member == actor), None) or uuid.uuid4().hex
  347. live.subscribers[subscription] = actor
  348. self.pending_closes.pop(ref.session_id, None)
  349. from gateway.session_viewer_close import cancel_orphan_close
  350. cancel_orphan_close(self, ref.session_id)
  351. from tools.async_delegation import allow_for_session
  352. allow_for_session((self.profile_id, ref.session_id)) # a closed session's work may start again
  353. transport = self.events.get(actor.transport_id)
  354. if transport is not None:
  355. if replay_from is not None and not live.event_stream.fanout.contains(transport):
  356. live.event_stream.catch_up(transport, *replay_from)
  357. live.event_stream.fanout.attach(transport)
  358. live.event_stream.on_overflow = partial(self._retire_overflowed, ref.session_id)
  359. active_generation = handle.execution_generation if handle.execution_state == "running" else None
  360. prompts = live.controls.snapshot(ref.session_id, active_generation)
  361. epoch, sequence = live.event_stream.watermark()
  362. if history:
  363. history, target = self.unless_deleted(ref, lambda: tuple(local_history(self, ref, prepared))), None
  364. else:
  365. history, target = (), self.unless_deleted(ref, lambda: transcript_target(self, ref))
  366. snapshot = SubscriptionSnapshot(subscription, handle, epoch, sequence, history,
  367. tuple(self._pending_receipt(r) for r in pending_rows(self, ref)), prompts,
  368. transcript_target=target)
  369. from gateway.session_viewer_close import cancel_persisted_close
  370. await asyncio.to_thread(cancel_persisted_close, self, ref.session_id) # the viewer cancels it
  371. from gateway.session_deferred_pick import reask_on_attach
  372. await reask_on_attach(self, ref, live)
  373. return snapshot
  374. def _retire_overflowed(self, session_id, transport):
  375. """The fanout dropped this peer's backlog: its subscription is over even though the
  376. socket still answers RPCs. A later resume re-attaches it with a fresh snapshot."""
  377. live = self.sessions[session_id]
  378. for subscription, member in list(live.subscribers.items()):
  379. if self.events.get(member.transport_id) is transport:
  380. del live.subscribers[subscription]
  381. async def detach(self, actor, subscription_id, *, end_reason=None, orphaned=False):
  382. for session_id, live in self.sessions.items():
  383. if subscription_id in live.subscribers:
  384. if live.subscribers[subscription_id] != actor:
  385. raise RuntimeStoreError('permission_denied')
  386. del live.subscribers[subscription_id]
  387. transport = self.events.get(actor.transport_id)
  388. if transport is not None:
  389. live.event_stream.fanout.detach(transport)
  390. if not live.subscribers:
  391. from gateway.session_local_mcp import release_editor_mcp
  392. live.detaching += 1 # unwatched off the loop, its close not yet pending: not retirable
  393. try:
  394. await asyncio.to_thread(release_editor_mcp, self, [session_id])
  395. finally:
  396. live.detaching -= 1
  397. if end_reason:
  398. from gateway.session_viewer_close import close_viewer_session
  399. await close_viewer_session(self, session_id, end_reason)
  400. return
  401. from gateway.session_acp_lifecycle import end_idle_acp_session, reopen_if_reattached
  402. from gateway.session_runtime_workers import tracked_write
  403. # A viewer that attached while the close was off the loop reopens it.
  404. await tracked_write(self, partial(end_idle_acp_session, self, session_id),
  405. then=partial(reopen_if_reattached, self, session_id))
  406. if orphaned: # a plain session.detach is a live client handing the session back
  407. from gateway.session_viewer_close import arm_detached_orphan_close
  408. await arm_detached_orphan_close(self, session_id)
  409. return
  410. raise RuntimeStoreError('not_found')
  411. def _receipt(self, row):
  412. status, outcome = row['status'], row['outcome']
  413. if outcome == STEERING: # handed to the running claim: it reads as that execution until settled
  414. status, outcome = 'started', None
  415. return AdmissionReceipt(row['admission_id'], SessionRef(self.profile_id, row['target_session_id']),
  416. row['seq'], status, outcome, row['owner_epoch'] or self.epoch, row['generation'])
  417. def _pending_receipt(self, row):
  418. # Only public input text crosses the viewer boundary, never the
  419. # private native envelope's routing or authorization provenance.
  420. payload = row['payload']
  421. text = payload.get('text')
  422. if text is None:
  423. text = payload.get('native_text_v1', {}).get('event', {}).get('text', '')
  424. from gateway.session_queued_controls import queued_controls
  425. return PendingAdmission(**vars(self._receipt(row)), input_id=row['request_id'], text=text,
  426. controls=queued_controls(self, row), steering=row['outcome'] == STEERING)
  427. def _publish_pending(self, ref):
  428. from gateway.session_retirement import restore_retired
  429. restore_retired(self, ref.session_id) # input committed after its conversation retired
  430. live = self.sessions[ref.session_id]
  431. from gateway.session_queued_controls import pending_rows
  432. with live.event_stream.lock:
  433. handle = self._handle(ref)
  434. pending = [asdict(self._pending_receipt(row)) for row in pending_rows(self, ref)]
  435. # Turns bump runtime_revision without any session.updated event, so
  436. # this is the only place a viewer learns the CAS revision a later
  437. # prepared mutation must present.
  438. live.event_stream.publish(ref.session_id, {
  439. 'stored_session_id': ref.session_id, 'pending': pending, 'authority_epoch': self.epoch,
  440. 'desktop_protocol': CANONICAL_GATEWAY_PROTOCOL,
  441. 'running': handle.execution_state == 'running',
  442. 'execution_generation': handle.execution_generation,
  443. 'revision': handle.revision,
  444. }, event_type='session.info')
  445. from gateway.session_bot_mailbox import wake_bot_receipts
  446. wake_bot_receipts(self, ref.session_id)
  447. def _schedule(self, ref):
  448. from gateway.session_retirement import restore_retired
  449. restore_retired(self, ref.session_id)
  450. live = self.sessions[ref.session_id]
  451. live.rescan = True
  452. if live.task is None or live.task.done():
  453. live.task = asyncio.create_task(self._drain(ref))
  454. live.task.add_done_callback(_log_drain_failure)
  455. from gateway.session_runtime_workers import persist_work_count
  456. live.task.add_done_callback(lambda task: persist_work_count(self, task))
  457. def wake_after_worker(self, session_id):
  458. """A worker execution on ``session_id`` (a logical owner or its compression continuation)
  459. ended: a drain that parked behind it re-runs. A no-op while a drain is running."""
  460. for sid in dict.fromkeys((session_id, self.logical_owner(session_id))):
  461. if sid in self.sessions:
  462. self._schedule(SessionRef(self.profile_id, sid))
  463. def _pause(self, ref, reason):
  464. """The FIFO stopped without claiming its head. Committed rows stay queued for a later
  465. drain; process-local delivery/HTTP/producer waiters on this session are released,
  466. with the reason instead of a reply, so an adapter loop is never parked on a turn that
  467. will not run. The ingress turns that refusal into one user-facing notice per episode.
  468. ``session_busy`` is transient (a registered worker finishes and wakes the FIFO), so only
  469. the messaging delivery waiters are released for it; an API/webhook/Bot observer keeps
  470. waiting for the answer instead of reporting accepted work as a conflict."""
  471. for row in list_session_admissions(self.db, session_id=ref.session_id):
  472. admission_id = row['admission_id']
  473. if reason == 'session_busy' and admission_id not in self.native_waiters:
  474. continue
  475. self.native_waiters.discard(admission_id)
  476. waiter = self.waiters.pop(admission_id, None)
  477. if waiter is not None and not waiter.done():
  478. waiter.set_exception(RuntimeStoreError(reason))
  479. async def admit_automation(self, adapter, event, identity, replaces=None, fold=None):
  480. from gateway.session_automation import admit_automation
  481. return await admit_automation(self, adapter, event, identity, replaces, fold)
  482. async def admit_native(self, event):
  483. """Await current connector policy, then commit before ACK or scheduling execution."""
  484. from gateway.session_ingress_media import abandon_capture, capture_lease
  485. async with self._route_turn(event) as fence:
  486. with capture_lease(self) as scope: # captured bytes are held until commit or refusal
  487. try:
  488. return await self._admit_native(event, fence)
  489. except BaseException:
  490. # Before the shielded write starts, no row can own these bytes. A cancelled copy
  491. # arranges its own completion cleanup in prepare_native; other failures clean here.
  492. if not scope.write_started:
  493. await abandon_capture(scope)
  494. raise
  495. @contextlib.asynccontextmanager
  496. async def _route_turn(self, event):
  497. """One route's inputs commit in the order they entered ``admit_native``: a photo's off-loop
  498. capture must not let the chat's next text take the earlier FIFO position. asyncio's Lock is
  499. FIFO and a cancelled waiter gives up only its own place, so a later input never overtakes an
  500. earlier one still in flight. Other routes never wait; an unroutable event (refused by
  501. ``prepare_native``) is not held. Yields the messaging input's fence check (else None)."""
  502. ingress = messaging_admission.get()
  503. messaging_admission.set(None) # not inherited by the drain this admission schedules
  504. try:
  505. key = self.runner.session_store._generate_session_key(event.source)
  506. except AttributeError: # no source, or a duck-typed one without routing fields
  507. key = None
  508. if key is None:
  509. yield None
  510. return
  511. arrived = self.route_epoch(key)
  512. def fence(_conn=None):
  513. if self.route_epoch(key) != arrived:
  514. raise RuntimeStoreError('stale_generation', field='route',
  515. detail='a /stop or /new of this chat came after this message')
  516. def queued(): # its FIFO position is fixed behind the earlier input
  517. if isinstance(ingress, asyncio.Future) and not ingress.done():
  518. ingress.set_result(None)
  519. async with keyed_turn(self._route_turns, key, queued=queued):
  520. if ingress is not None:
  521. fence()
  522. yield fence if ingress is not None else None
  523. def route_epoch(self, route):
  524. """How many messaging /stop and /new fenced *route* (``fence_route``)."""
  525. return self._route_epochs.get(route, 0)
  526. def fence_route(self, route):
  527. """/stop and /new: the route's messaging input still capturing or waiting, and a slash command's
  528. turn read before it (``_stop_epoch``), never commit."""
  529. self._route_epochs[route] = self.route_epoch(route) + 1
  530. async def _admit_native(self, event, fence=None):
  531. import json
  532. from gateway.session_envelope import prepare_native, restore_native_source
  533. self._require_admission_open()
  534. payload = await prepare_native(self.runner, event)
  535. if fingerprint := getattr(event, '_ingress_sha256', None):
  536. payload['native_text_v1']['ingress_sha256'] = fingerprint # the delivery before a rewrite
  537. self._require_admission_open()
  538. source = restore_native_source(payload) # no re-hash of the just-captured media on the loop
  539. from gateway.session_runtime_workers import tracked_write
  540. ref = self._publish_route(source, *await tracked_write(self, partial(self._reserve_route, source)))
  541. identity = json.dumps([source.profile, source.platform.value, source.chat_id,
  542. source.thread_id, source.user_id], separators=(',', ':'))
  543. request_id = str(payload['native_text_v1']['event']['message_id'] or uuid.uuid4().hex)
  544. if revision := payload['native_text_v1']['event'].get('edit_revision'):
  545. request_id += ':edit:' + revision # an edit is its own input, not a changed replay of the send
  546. committed = [ref] # the row's target: the write may retarget a rotated-away reservation
  547. # A /retry re-send carries its transcript rewrite: it commits in this admission's own write.
  548. from gateway import session_retry_admission
  549. rewrite = session_retry_admission.take(event, self.db)
  550. def admitted(row):
  551. committed[0] = SessionRef(self.profile_id, row.get('target_session_id') or ref.session_id)
  552. if rewrite is not None and rewrite.applied:
  553. rewrite.on_commit()
  554. return self._admitted(committed[0], event, row)
  555. from gateway.session_ingress_media import _lease_scope
  556. if (scope := _lease_scope.get()) is not None:
  557. scope.write_started = True
  558. native_write_started.set(True) # shielded from here: a cancelled caller may still commit
  559. return await tracked_write(self, partial(
  560. self._admit_native_write, principal_id='messaging:' + identity, session_id=ref.session_id,
  561. request_id=request_id, payload=payload,
  562. authorize=session_retry_admission.compose(rewrite, self._admission_gate(fence)),
  563. raw=getattr(event, 'raw_message', None)),
  564. then=admitted, ordered=True, after=lambda _receipt: self._schedule_admitted(committed[0]))
  565. def _admit_native_write(self, *, principal_id, session_id, request_id, payload, authorize, raw=None):
  566. # Off the loop and in admission order: the redelivery reconcile reads the ledger it decides
  567. # against, so a concurrent first delivery of the same message cannot slip between them.
  568. from gateway.session_ingress_media import drop_capture_lease, reconcile_native_retry, release_unheld_media
  569. fresh = payload.get('native_text_v1', {}).get('media', [])
  570. session_id = self._current_owner(session_id)
  571. try:
  572. payload = reconcile_native_retry(self.db, principal_id=principal_id, session_id=session_id,
  573. request_id=request_id, payload=payload, raw=raw)
  574. return admit_session_input(self.db, epoch=self.epoch, principal_id=principal_id, session_id=session_id,
  575. request_id=request_id, payload=payload, _authorize_write=authorize)
  576. except Exception:
  577. # A refused admission owns nothing it captured (unless another admission holds the same bytes).
  578. drop_capture_lease()
  579. release_unheld_media(self.db, fresh, retain_history=False)
  580. raise
  581. def _current_owner(self, session_id):
  582. """``session_id`` unless the exhaustion auto-reset rotated it away after this admission
  583. reserved it: then the live fresh session its route now names (the moved rows' home)."""
  584. route = self.rotated_owners.get(session_id)
  585. if route is None:
  586. return session_id
  587. entry = self.runner.session_store.lookup_by_session_key(route)
  588. current = self.logical_owner(entry.session_id) if entry is not None else session_id
  589. return current if current in self.sessions else session_id
  590. def _admitted(self, ref, event, row):
  591. if event is not None:
  592. event._gateway_accepted = True
  593. if row['status'] == 'queued' and (committed := getattr(event, '_on_commit', None)) is not None:
  594. committed() # a queued slash command's ack follows its commit (``_queue_command_turn``)
  595. mark_producer_admission(self, event, row)
  596. self._publish_pending(ref)
  597. return self._receipt(row)
  598. def _schedule_admitted(self, ref, _receipt=None):
  599. # tracked_write's ``after``: in the admitting caller's own step (see tracked_write).
  600. self._schedule(ref)
  601. async def recover_native_sessions(self, bindings):
  602. """Bind only server-observed native routes; unknown work stays paused."""
  603. from collections import Counter
  604. from gateway.session_envelope import check_native_route
  605. bindings = list(bindings)
  606. counts = Counter(sid for sid, _, _ in bindings)
  607. results = {}
  608. for sid, available_source, adapter in bindings:
  609. try:
  610. if counts[sid] != 1:
  611. raise RuntimeStoreError('admission_conflict')
  612. rows = list_session_admissions(self.db, session_id=sid, pending_only=False)
  613. native = [row for row in rows if 'native_text_v1' in row['payload']]
  614. if not native:
  615. raise RuntimeStoreError('not_found')
  616. target = self.physical_target(SessionRef(self.profile_id, sid))
  617. source, route = await check_native_route(self.runner, native[-1]['payload'], target,
  618. available_source, adapter)
  619. for row in rows:
  620. if row['status'] != 'queued':
  621. continue
  622. if 'native_text_v1' in row['payload']:
  623. await check_native_route(self.runner, row['payload'], target, available_source, adapter)
  624. elif 'local_automation_v1' in row['payload']:
  625. from gateway.session_automation import check_local_automation
  626. check_local_automation(
  627. self, SessionRef(self.profile_id, sid), row, route=route)
  628. else:
  629. from gateway.session_operator import check_local_input
  630. check_local_input(
  631. self, SessionRef(self.profile_id, sid), row, source=source)
  632. self._require_admission_open()
  633. self.sessions.setdefault(sid, LiveSession(source, route))
  634. if any(row['status'] == 'unknown' for row in rows):
  635. raise RuntimeStoreError('unknown_execution')
  636. self._schedule(SessionRef(self.profile_id, sid))
  637. results[sid] = 'ready'
  638. except RuntimeStoreError as exc:
  639. results[sid] = exc.reason
  640. return results
  641. async def submit(self, actor: SessionActor, request: Submission, *, _authorize_write=None, queue_while_restarting=False,
  642. _replay_guard=None, _kanban_card=None):
  643. """``queue_while_restarting``: the caller is an interactive client attached to a restart's
  644. after-turn wait, which commits the input queued for the next owner. Machine callers (a2a,
  645. kanban, hosted rooms) keep the plain ``runtime_draining`` refusal and retry against the next
  646. owner with their own stable request id; none of them reads a released waiter as queued.
  647. ``_kanban_card`` is the Kanban dispatcher's trusted producer marker (``session_kanban.run_task``
  648. only): committed as ``kanban_card_v1``, it lets a card attempt lost across an owner restart
  649. settle interrupted (#134889 KB-8). Never a payload field: a client cannot supply it."""
  650. self.authorize(actor, request.ref, 'session:submit')
  651. async with self.sessions[request.ref.session_id].mutation_lock:
  652. # Deletion can retire this session while the caller waits on its writer.
  653. self.authorize(actor, request.ref, 'session:submit')
  654. from gateway.session_ingress_media import capture_lease
  655. with capture_lease(): # captured bytes are held from capture until commit or refusal
  656. return await self._submit(actor, request, _authorize_write=_authorize_write,
  657. queue_only=queue_while_restarting is True, _replay_guard=_replay_guard,
  658. _kanban_card=_kanban_card)
  659. async def _submit(self, actor, request, *, _authorize_write=None, queue_only=False, _replay_guard=None,
  660. _kanban_card=None):
  661. self._require_admission_open(queue_only=queue_only)
  662. if (request.intent != 'queue' or not {'text'} <= set(request.payload) <= {
  663. 'text', 'attachments', 'finite', 'unattended', 'surface', 'voice_context', 'interrupted',
  664. 'voice_turn', 'display_kind', 'title_preview', 'turn_author', 'api_text', 'model_override', 'clarify_batch'}
  665. or not isinstance(request.payload['text'], str)):
  666. raise RuntimeStoreError('invalid_params')
  667. from gateway.session_ingress_media import admit_attachments
  668. from gateway.session_finite import admit_finite
  669. from gateway.session_surface import admit_surface
  670. from gateway.session_display import admit_display
  671. from gateway.session_cli_turn import admit_cli_turn
  672. finite = admit_finite(request.payload)
  673. def admitted():
  674. # The exact durable identity (authenticated principal + target + request id); the
  675. # attachment fields it yields still pass the full payload-digest comparison below.
  676. from hermes_state_runtime import find_session_admission
  677. return find_session_admission(self.db, principal_id=actor.subject,
  678. session_id=request.ref.session_id, request_id=request.request_id)
  679. payload = {'text': request.payload['text'], **finite, **admit_surface(request.payload),
  680. **admit_display(request.payload), **admit_cli_turn(request.payload),
  681. **admit_attachments(request.payload.get('attachments'), admitted=admitted)}
  682. if _kanban_card is not None:
  683. payload['kanban_card_v1'] = _kanban_card
  684. source = self.sessions[request.ref.session_id].source
  685. from gateway.owner_identity import same_owner
  686. if source is not None and not same_owner(source.user_id, actor.subject):
  687. # Durable server authorization, not a client payload field. The original
  688. # principal remains the admission/retry identity across owner restarts.
  689. # Operator provenance is about who was authorized at this boundary, not
  690. # whether the bound session happens to use the LOCAL transport.
  691. payload['local_operator_v1'] = {
  692. 'profile_id': self.profile_id, 'session_id': request.ref.session_id,
  693. 'principal_id': actor.subject}
  694. def admit():
  695. try:
  696. return admit_session_input(self.db, epoch=self.epoch, principal_id=actor.subject,
  697. session_id=request.ref.session_id, request_id=request.request_id,
  698. payload=payload, intent=request.intent, _replay_guard=_replay_guard,
  699. _authorize_write=self._admission_gate(_authorize_write, queue_only=queue_only))
  700. except Exception:
  701. self._release_refused_capture(request, payload)
  702. raise
  703. from gateway.session_runtime_workers import tracked_write
  704. return await tracked_write(self, admit, then=partial(self._admitted, request.ref, None), ordered=True,
  705. after=partial(self._schedule_admitted, request.ref))
  706. def _release_refused_capture(self, request, payload):
  707. """A refused submission's captured bytes have no owner unless another admission holds the
  708. same bytes. They become durable retirement candidates (collected again by delete/prune if
  709. this pass fails), then a holder-aware release runs. Its failure is logged and never replaces
  710. the refusal the client must see (e.g. ``admission_conflict``)."""
  711. media = payload.get('attachments_v1', {}).get('media', [])
  712. if not media:
  713. return
  714. from gateway.session_ingress_media import drop_capture_lease
  715. drop_capture_lease() # its own lease must not keep the refused bytes from collection
  716. from hermes_state_media import collect_retired_media, retire_media
  717. try:
  718. self.db._execute_write(lambda conn: retire_media(conn, {'attachments_v1': {'media': media}}))
  719. collect_retired_media(self.db)
  720. except Exception:
  721. import logging
  722. logging.getLogger(__name__).exception(
  723. 'Releasing refused attachments of request %s failed', request.request_id)
  724. async def receipt(self, actor, ref, admission_id):
  725. self.authorize(actor, ref, 'session:submit')
  726. row = get_session_admission(self.db, admission_id=admission_id)
  727. if row is None or row['target_session_id'] != ref.session_id:
  728. raise RuntimeStoreError('not_found')
  729. from gateway.owner_identity import same_owner
  730. if not same_owner(row['principal_id'], actor.subject) and 'session:control' not in actor.capabilities:
  731. raise RuntimeStoreError('permission_denied')
  732. live = self.sessions[ref.session_id]
  733. with live.event_stream.lock:
  734. # Read after the row: a terminal row whose completion is not yet published reads as
  735. # its claim, so a receipt poller never replays events that lack the completion frame.
  736. row = live.settling.get(admission_id, row)
  737. return self._receipt(row)
  738. async def cancel_queued(self, actor, ref, admission_id):
  739. before = await self.receipt(actor, ref, admission_id)
  740. from gateway.session_runtime_workers import track_mutation
  741. # Write off the shared loop; commit and observer settlement are one tracked task the
  742. # caller cannot cancel between (an exact retry still pays a left-over obligation).
  743. row = await asyncio.shield(track_mutation(self, self._commit_cancel(ref, admission_id, before)))
  744. # Media collection is its own write txn after the committed, settled cancellation: its
  745. # failure surfaces to the caller, and an exact retry collects again without re-settling.
  746. from gateway.session_ingress_media import release_admission_media
  747. await asyncio.to_thread(release_admission_media, self.db, admission_id)
  748. return self._receipt(row)
  749. async def cancel_or_claimed(self, actor, ref, admission_id):
  750. """Cancel a row the caller read ``queued``: None once cancelled, else the started row the
  751. drain claimed between that read and the cancel commit (``stale_generation``), which the
  752. caller must stop instead of refusing a live execution."""
  753. try:
  754. await self.cancel_queued(actor, ref, admission_id)
  755. return None
  756. except RuntimeStoreError as exc:
  757. if exc.reason != 'stale_generation':
  758. raise
  759. row = get_session_admission(self.db, admission_id=admission_id)
  760. if row is None or row['status'] != 'started':
  761. raise RuntimeStoreError('stale_generation')
  762. return row
  763. async def _commit_cancel(self, ref, admission_id, before):
  764. if before.status == 'queued':
  765. # Owed from before the write: a commit that reports failure, or anything raising
  766. # between the commit and observer settlement, leaves it for an exact retry to pay.
  767. self.cancel_obligations.add(admission_id)
  768. try:
  769. row = await asyncio.to_thread(cancel_session_input, self.db, epoch=self.epoch, admission_id=admission_id)
  770. except RuntimeStoreError:
  771. self.cancel_obligations.discard(admission_id) # a definite refusal: nothing committed
  772. raise
  773. if (row['status'], row['outcome']) == ('terminal', 'cancelled') and admission_id in self.cancel_obligations:
  774. self._settle_cancelled(ref, admission_id)
  775. else:
  776. if row['status'] != 'queued':
  777. self.cancel_obligations.discard(admission_id)
  778. self._publish_pending(ref)
  779. return row
  780. def _settle_cancelled(self, ref, admission_id, outcome='cancelled'):
  781. # The only place a queued row becomes terminal: every observer kind that waits on
  782. # the admission (native delivery, API/webhook/hosted waiters, ACP and viewer streams)
  783. # settles here, or a cancelled row that never reaches _drain blocks them forever.
  784. live = self.sessions[ref.session_id]
  785. with live.event_stream.lock:
  786. self._publish_pending(ref)
  787. running = live.event_stream.execution
  788. # A queued row owns no execution generation; stamp its own identity so the
  789. # completion is not attributed to the turn currently running ahead of it.
  790. live.event_stream.execution = {'authority_epoch': self.epoch, 'admission_id': admission_id}
  791. try:
  792. live.event_stream.publish(ref.session_id, {
  793. 'text': '', 'content': '', 'admission_id': admission_id, 'outcome': outcome})
  794. finally:
  795. live.event_stream.execution = running
  796. # Published once: a later exact retry re-collects media but never repeats the completion.
  797. self.cancel_obligations.discard(admission_id)
  798. self.native_waiters.discard(admission_id)
  799. drop_producer_turn(self, admission_id)
  800. waiter = self.waiters.pop(admission_id, None)
  801. if waiter is not None and not waiter.done():
  802. waiter.set_result(None)
  803. # A paused drain (preclaim refusal on this head) ended its task; the successors
  804. # need a fresh drain that revalidates them on their own merits.
  805. self._schedule(ref)
  806. # Last: telemetry must never stand between a cancelled row and its waiter/successor drain.
  807. busy_row_settled(admission_id, outcome)
  808. async def resolve_unknown(self, actor, ref, admission_id, generation):
  809. """Operator acknowledgement that an ``unknown`` turn will not finish; the paused FIFO behind
  810. it resumes. Never requeues the input. A turn that finished in THIS owner but whose
  811. settlement write failed (its exact result is still in ``pending_results``) commits that
  812. result instead of discarding it: the answer was produced, only its receipt was lost."""
  813. self.authorize(actor, ref, 'session:control')
  814. await self.receipt(actor, ref, admission_id)
  815. return await self.resolve_unknown_as_owner(ref, admission_id, generation)
  816. async def resolve_unknown_as_owner(self, ref, admission_id, generation, *, notice=None):
  817. """``resolve_unknown`` past authorization: also the owner's own call when the last viewer
  818. closed a session whose turn recovered ``unknown`` (gateway/session_viewer_close), or when an
  819. unattended automation turn settles itself (``notice``: its transcript copy, GW-10)."""
  820. captured = self.pending_results.get(admission_id)
  821. before = get_session_admission(self.db, admission_id=admission_id)
  822. prepared = None
  823. if captured is not None and before is not None and before['status'] == 'unknown':
  824. from gateway.session_results import prepare_result
  825. row = get_session_admission(self.db, admission_id=admission_id)
  826. prepared = prepare_result(row, captured['result'].get('final_response') or '', 'completed', captured)
  827. # The transcript boundary commits WITH the terminal transition: a follower can never be
  828. # claimed with the discarded text left open to be merged into its request. With a captured
  829. # result the answer is normally already the transcript tail, so the closer is a no-op.
  830. from gateway.session_results import close_discarded_turn
  831. from gateway.session_runtime_workers import tracked_write
  832. return await tracked_write(self, partial(
  833. resolve_unknown_session_input, self.db, epoch=self.epoch, admission_id=admission_id,
  834. generation=generation, _captured=prepared,
  835. # ``notice`` says the turn was interrupted: only when that is the outcome, never beside a
  836. # captured result this resolution commits as the turn's own answer.
  837. _terminal_write=lambda conn, lost: close_discarded_turn(self.db, conn, lost, None if (
  838. prepared is not None and lost['owner_epoch'] == self.epoch) else notice)),
  839. then=partial(self._resolved_unknown, ref, admission_id, prepared, captured))
  840. async def _resolved_unknown(self, ref, admission_id, prepared, captured, row):
  841. self.pending_results.pop(admission_id, None)
  842. # A resolution racing recovery leaves publication, delivery and collection to the drain. A
  843. # conversation idle retirement let go between the commit and this follow-up has no drain:
  844. # bring it back first, so the captured answer reaches its viewers.
  845. live = self.sessions.get(ref.session_id)
  846. if live is None:
  847. from gateway.session_retirement import restore_retired
  848. restore_retired(self, ref.session_id)
  849. live = self.sessions.get(ref.session_id)
  850. drain_owns = live is not None and live.event_stream.execution.get('admission_id') == admission_id
  851. if not drain_owns:
  852. from gateway.session_ingress import deliver_resolved
  853. from gateway.session_settlement_recovery import _publish_completion, release_media_after_delivery
  854. try:
  855. if prepared is not None and row['owner_epoch'] == self.epoch:
  856. live = self.sessions[ref.session_id]
  857. with live.event_stream.lock:
  858. _publish_completion(self, live, ref, row, row,
  859. captured['result'].get('final_response') or '', captured)
  860. await deliver_resolved(self, admission_id, owed=prepared is not None and row['owner_epoch'] == self.epoch)
  861. finally:
  862. release_media_after_delivery(self, row)
  863. else:
  864. from gateway.session_settlement_recovery import hand_collection_to_drain
  865. hand_collection_to_drain(self, admission_id)
  866. self._publish_pending(ref)
  867. self._schedule(ref)
  868. return self._receipt(row)
  869. async def interrupt(self, actor, ref, generation):
  870. self.authorize(actor, ref, 'session:control')
  871. handle = self._handle(ref)
  872. if handle.execution_generation != generation:
  873. raise RuntimeStoreError('stale_generation')
  874. live, agent = self.sessions[ref.session_id], None
  875. if handle.execution_state == 'running':
  876. # Latches are written under the lock adopt_agent takes on the executor thread, BEFORE the
  877. # agent is called: an adoption racing the call consumes them. The agent is called outside it.
  878. with live.event_stream.lock:
  879. adopted_generation, agent = self.adopted.get(ref.session_id, (None, None))
  880. if adopted_generation != generation:
  881. # Its execution returned and only the settlement is left: nothing to stop, and the route's
  882. # agent may already be the NEXT turn's (own-agent runs install it before their claim).
  883. returned = (live.event_stream.execution.get('execution_generation') == generation
  884. and self.executing.get(ref.session_id) != generation)
  885. agent = None if returned else self.agent(ref)
  886. if not returned and not self._runs_own_agent(ref):
  887. # Accepted for this exact claim before its turn adopted an agent: whatever agent
  888. # the turn adopts for this generation must not run the work as if no Stop arrived.
  889. self.pending_stops[ref.session_id] = generation
  890. if agent is not None:
  891. self.delivered_stops[ref.session_id] = generation
  892. # Stop hard-cancels this turn (aborts compression too). Background delegations are not the
  893. # turn: they keep running and report back; `/stop` and session close/reset end them.
  894. from agent.interrupt_compat import request_hard_interrupt
  895. if agent is not None:
  896. request_hard_interrupt(agent)
  897. return self._handle(ref)
  898. def _classic_turn_holds(self, route):
  899. """The route's running slot is held by a turn this FIFO did not claim (the drain is serial:
  900. its own previous turn released the slot before this claim). Leaked and reaped slots are
  901. evicted first, so they never block the FIFO; the reaped check reads state.db at most 1/s."""
  902. runner = self.runner
  903. running = getattr(runner, '_is_session_running', None)
  904. if running is None or not running(route):
  905. return False
  906. import time
  907. now = time.monotonic()
  908. if now - self._classic_evict_at.get(route, 0.0) >= 1.0:
  909. self._classic_evict_at[route] = now
  910. runner._hm_evict_idle_stale_agent(route)
  911. if running(route):
  912. runner._hm_evict_reaped_agent(route)
  913. return running(route)
  914. def _runs_own_agent(self, ref):
  915. """The route's running slot holds a real agent (not the claim's pending sentinel), so the
  916. Stop above reached the turn's own agent, not a cached one it may still replace."""
  917. from gateway.run import _AGENT_PENDING_SENTINEL
  918. from gateway.session_detached import running_key
  919. live = self.sessions[ref.session_id]
  920. running = getattr(self.runner, '_running_agents', None) or {}
  921. agent = running.get(running_key(self.runner, live.route, ref.session_id))
  922. return agent is not None and agent is not _AGENT_PENDING_SENTINEL
  923. def steer_agent(self, ref, generation, text):
  924. """The agent a steer for this running generation reaches; the caller holds the session's
  925. event_stream.lock after check_approval_generation. Before the turn adopts its agent the
  926. resident one is the previous turn's cached agent, which this turn may replace, so *text* is
  927. latched for the agent this generation adopts (adopt_agent, under the same lock):
  928. ``(None, True)``. A claim with no steer seam (cron, managed worker) latches nothing."""
  929. adopted_generation, agent = self.adopted.get(ref.session_id, (None, None))
  930. if adopted_generation != generation:
  931. agent = self.agent(ref) if self._runs_own_agent(ref) else None
  932. from gateway.session_busy_controls import adopts_owner_agent
  933. if agent is not None or text is None or not adopts_owner_agent(self, ref):
  934. return agent, False
  935. latched, texts = self.pending_steers.get(ref.session_id, (None, []))
  936. self.pending_steers[ref.session_id] = (generation, [*texts, text] if latched == generation else [text])
  937. return None, True
  938. def adopt_agent(self, session_id, generation, agent):
  939. """The turn installs its agent for the running claim; a Stop latched while there
  940. was no agent to deliver it to fires now, never against a later generation. A Stop
  941. delivered to the cached agent for an earlier generation may have landed after that
  942. turn's finalizer cleared it, so it is dropped rather than cancelling this turn. A managed
  943. worker is a fresh process per turn and carries no earlier flag to drop."""
  944. live = self.sessions.get(session_id)
  945. # The turn's executor thread runs this; the stream lock orders it against session.steer's latch.
  946. with live.event_stream.lock if live is not None else contextlib.nullcontext():
  947. self.adopted[session_id] = (generation, agent)
  948. clear = getattr(agent, 'clear_interrupt', None)
  949. if self.delivered_stops.pop(session_id, generation) != generation and clear is not None:
  950. clear()
  951. if self.pending_stops.get(session_id) == generation:
  952. del self.pending_stops[session_id]
  953. from agent.interrupt_compat import request_hard_interrupt
  954. request_hard_interrupt(agent)
  955. latched, steers = self.pending_steers.pop(session_id, (None, ()))
  956. for text in steers if latched == generation else ():
  957. agent.steer(text)
  958. def check_approval_generation(self, session_id, generation):
  959. handle = self._handle(SessionRef(self.profile_id, session_id))
  960. if handle.execution_generation != generation or handle.execution_state != "running":
  961. raise RuntimeStoreError("stale_generation")
  962. def publish_execution(self, session_id, generation, event_type, payload):
  963. """Worker callbacks never outlive their exact running claim."""
  964. live = self.sessions[session_id]
  965. with live.event_stream.lock:
  966. # The executing generation (set with the claim's stamp, cleared the moment its execution
  967. # returns, before settlement): no per-token SQLite read under this lock, and a late
  968. # callback (an agent thread outliving a timed-out turn) never follows the completion.
  969. if self.executing.get(session_id) != generation:
  970. return False
  971. live.event_stream.note_progress(generation, event_type, payload)
  972. delivered = live.event_stream.publish(session_id, payload, event_type=event_type)
  973. from gateway.session_api_turn import publish_api_event
  974. observed = publish_api_event(self, session_id, event_type, payload)
  975. return bool(delivered or observed)
  976. def register_approval(self, session_id, generation, route, data, *, timeout_s=None):
  977. live = self.sessions[session_id]
  978. with live.event_stream.lock:
  979. self.check_approval_generation(session_id, generation)
  980. live.controls.register(session_id, route, generation, data, timeout_s=timeout_s)
  981. def register_clarify(self, session_id, generation, entry, *, timeout_s=None):
  982. live = self.sessions[session_id]
  983. with live.event_stream.lock:
  984. self.check_approval_generation(session_id, generation)
  985. live.controls.register_clarify(session_id, generation, entry, timeout_s=timeout_s)
  986. async def respond(self, actor, ref, generation, prompt_id, response, *, kind="approval"):
  987. capability = {"approval": "session:approve", "clarify": "session:respond"}.get(kind)
  988. if capability is None:
  989. raise RuntimeStoreError("invalid_params")
  990. self.authorize(actor, ref, capability)
  991. live = self.sessions[ref.session_id]
  992. with live.event_stream.lock:
  993. if actor not in live.subscribers.values():
  994. raise RuntimeStoreError("permission_denied")
  995. if type(generation) is not int:
  996. raise RuntimeStoreError("stale_generation")
  997. self.check_approval_generation(ref.session_id, generation)
  998. if not isinstance(prompt_id, str) or not prompt_id:
  999. raise RuntimeStoreError("invalid_params")
  1000. return live.controls.respond(ref.session_id, generation, prompt_id, response, kind=kind)
  1001. async def _claim_next(self, ref, live):
  1002. # False asks the pump to re-read after a queued head changed during preflight.
  1003. async with live.claim_gate, live.mutation_lock:
  1004. pending = list_session_admissions(self.db, session_id=ref.session_id)
  1005. if any(row['status'] == 'unknown' for row in pending):
  1006. raise RuntimeStoreError('unknown_execution')
  1007. first = next((row for row in pending if row['status'] == 'queued'), None)
  1008. from gateway.config import Platform
  1009. if live.source.platform == Platform.LOCAL:
  1010. # A reasoning command or a model pick made while the previous turn ran lands here, at the
  1011. # turn boundary.
  1012. from gateway.session_deferred_pick import apply_deferred_pick
  1013. from gateway.session_setting_commands import apply_pending_reasoning
  1014. await apply_pending_reasoning(self, ref, live)
  1015. await apply_deferred_pick(self, ref, live)
  1016. if first is not None and live.source.platform == Platform.LOCAL:
  1017. from gateway.session_local_recovery import restore_local_session
  1018. restore_local_session(self, ref.session_id)
  1019. if first['request_id'].startswith('hosted:'):
  1020. from gateway.session_hosted_transport import check_remote_hosted_admission
  1021. if not await asyncio.to_thread(check_remote_hosted_admission, self, ref, first):
  1022. service = getattr(self, 'hosted_room_service', None)
  1023. if service is None:
  1024. raise RuntimeStoreError('permission_denied')
  1025. await asyncio.to_thread(service.check_admission, ref, first)
  1026. # Cancellation may advance the FIFO while the source owner is awaited;
  1027. # the successor must earn its own reauthorization, not inherit this one.
  1028. current = get_session_admission(self.db, admission_id=first['admission_id'])
  1029. if current is None or current['status'] != 'queued':
  1030. return False, first
  1031. if 'local_automation_v1' in first['payload']:
  1032. from gateway.session_automation import check_local_automation
  1033. check_local_automation(self, ref, first)
  1034. else:
  1035. from gateway.session_operator import check_local_input
  1036. check_local_input(self, ref, first)
  1037. # Editor MCP schema drift re-freezes here, at this admission's start (a turn boundary),
  1038. # once the head is authorized: a refused head spawns no server and commits no change.
  1039. from gateway.session_local_mcp import refreeze_editor_mcp
  1040. await refreeze_editor_mcp(self, ref, live)
  1041. # The first real turn reopens a finalized row (#85303): mounts are reads, and
  1042. # SessionStore would route a stamped row as stale onto a FRESH session. A write: off-loop,
  1043. # tracked so retirement and the shutdown close gate count it.
  1044. from gateway.session_local_recovery import reopen_local_session
  1045. from gateway.session_runtime_workers import tracked_write
  1046. from gateway.session_completion_notes import noted_completion
  1047. if not noted_completion(first): # a note leaves a closed row closed (session_completion_notes)
  1048. await tracked_write(self, partial(reopen_local_session, self, ref))
  1049. if first is not None and 'native_text_v1' in first['payload']:
  1050. from gateway.session_envelope import check_native_route
  1051. await check_native_route(self.runner, first['payload'], self.physical_target(ref), live.source,
  1052. self.runner._adapter_for_source(live.source))
  1053. # Cancellation may advance FIFO while the connector is awaited.
  1054. # Never let the successor inherit this row's fresh verdict.
  1055. current = get_session_admission(self.db, admission_id=first['admission_id'])
  1056. if current is None or current['status'] != 'queued':
  1057. return False, first
  1058. if first is not None and live.source.platform == Platform.API_SERVER:
  1059. from gateway.session_api import reopen_api_session
  1060. from gateway.session_api_turn import check_api_turn
  1061. from gateway.session_runtime_workers import tracked_write
  1062. check_api_turn(self, ref, first['payload'])
  1063. await tracked_write(self, partial(reopen_api_session, self, ref))
  1064. elif first is not None and live.detached:
  1065. # The chat's /new ended its row; the app's turn makes it live again, as an API row's
  1066. # does. A note leaves it closed, as a local one's does (session_completion_notes).
  1067. from gateway.session_completion_notes import noted_completion, reopen_for_turn
  1068. if not noted_completion(first):
  1069. await reopen_for_turn(self, ref)
  1070. if first is not None and self._classic_turn_holds(live.route):
  1071. # A classic turn (fallback wake, CLI handoff, legacy resume) holds the route's slot:
  1072. # a claim now would hand this row to its busy path and settle it unrun. The head
  1073. # waits queued, as main's pending slot did; /stop and /new still cancel it.
  1074. return 'classic_busy', first
  1075. self._require_admission_open()
  1076. from gateway.session_runtime_workers import tracked_write
  1077. live.rescan = False # a _schedule from here on may not be visible to this claim's read
  1078. # The row these checks validated, never whichever is first when the write runs: a
  1079. # cancellation commits from a worker thread and can advance the FIFO at any point.
  1080. row = await tracked_write(self, partial(claim_session_input, self.db, epoch=self.epoch,
  1081. session_id=ref.session_id, _guard=self._admission_gate(),
  1082. head=first['admission_id'] if first is not None else None),
  1083. then=partial(self._stamp_claim, ref, live))
  1084. # Input scheduled while this claim's transaction ran may postdate its read: re-read
  1085. # instead of letting the drain exit idle over a committed admission.
  1086. return (False if row is None and live.rescan else row), first
  1087. def _stamp_claim(self, ref, live, row):
  1088. """The claim's execution stamp, in the same tracked task as its commit: no observer (or
  1089. retirement's join) ever sees a committed claim that no execution stamp names."""
  1090. if row: # None or False (the head changed under the write) claimed nothing
  1091. with live.event_stream.lock:
  1092. live.event_stream.execution = {
  1093. 'authority_epoch': self.epoch, 'execution_generation': row['generation'],
  1094. 'admission_id': row['admission_id']}
  1095. self.executing[ref.session_id] = row['generation']
  1096. from gateway.session_automation import admission_display
  1097. live.event_stream.publish(ref.session_id, admission_display(row.get('payload') or {}),
  1098. event_type='message.start')
  1099. self._publish_pending(ref)
  1100. return row
  1101. def _drain_stopping(self, ref, live):
  1102. """A drain about to wait (storage retry, classic slot) exits instead when the runtime drains,
  1103. the profile retires or the live entry was replaced; committed rows stay queued."""
  1104. if self.runner._draining or self.retiring or self.sessions.get(ref.session_id) is not live:
  1105. self._pause(ref, 'runtime_draining')
  1106. return True
  1107. return False
  1108. async def _refuse_head(self, ref, admission_id, reason):
  1109. """Settle the queued head a preclaim check refused as cancelled: its waiters get *reason* (the
  1110. ingress notice, the API's refusal) and its successors revalidate on their own merits."""
  1111. from gateway.session_runtime_workers import tracked_write
  1112. before = get_session_admission(self.db, admission_id=admission_id)
  1113. if before is not None and before['status'] == 'queued':
  1114. # Owed from before the write, as in _commit_cancel: storage failing after the commit leaves
  1115. # it for the drain's retry of this exact refusal (or an explicit cancel) to pay.
  1116. self.cancel_obligations.add(admission_id)
  1117. try:
  1118. row = await tracked_write(self, partial(cancel_session_input, self.db, epoch=self.epoch,
  1119. admission_id=admission_id))
  1120. except RuntimeStoreError:
  1121. self.cancel_obligations.discard(admission_id)
  1122. return False
  1123. if (row['status'], row['outcome']) != ('terminal', 'cancelled'):
  1124. self.cancel_obligations.discard(admission_id)
  1125. return False
  1126. if admission_id in self.cancel_obligations:
  1127. waiter = self.waiters.pop(admission_id, None)
  1128. if waiter is not None and not waiter.done():
  1129. waiter.set_exception(RuntimeStoreError(reason))
  1130. self._settle_cancelled(ref, admission_id)
  1131. from gateway.session_ingress_media import release_admission_media
  1132. await asyncio.to_thread(release_admission_media, self.db, admission_id)
  1133. return True
  1134. async def _claim_with_retry(self, ref, live):
  1135. """The drain's next ``(row, first)``, waiting out transient storage and a classic turn on the
  1136. route; None when the drain must exit (paused, or stopping with committed rows left queued)."""
  1137. backoff, conflicted, refusing = 0.0, None, None
  1138. while True:
  1139. # The refusal handling's own reads and writes share the claim's storage retry, which finishes
  1140. # that exact refusal first: once its cancellation commits, the claim no longer sees the row.
  1141. try:
  1142. if refusing is not None:
  1143. await self._refuse_head(ref, *refusing)
  1144. refusing = None
  1145. try:
  1146. row, first = await self._claim_next(ref, live)
  1147. except RuntimeStoreError as exc:
  1148. if exc.reason == 'unknown_execution':
  1149. # An automation turn nobody is waiting on settles interrupted (GW-10); only a
  1150. # person's lost turn keeps the FIFO paused for them to resolve.
  1151. from gateway.session_unattended_unknown import settle_unattended_unknown
  1152. if await settle_unattended_unknown(self, ref):
  1153. continue
  1154. if exc.reason == 'admission_conflict':
  1155. # A verdict on the head alone (its route or binding moved under it) that every drain
  1156. # repeats: pausing behind it strands the FIFO for good. Refused only once the same
  1157. # head conflicted twice, read after each: never a successor a cancel advanced to.
  1158. head = next((row['admission_id'] for row in list_session_admissions(
  1159. self.db, session_id=ref.session_id) if row['status'] == 'queued'), None)
  1160. if head is not None and head != conflicted:
  1161. conflicted = head
  1162. continue
  1163. if head is not None:
  1164. refusing = head, exc.reason
  1165. refused = await self._refuse_head(ref, *refusing)
  1166. refusing = None
  1167. if refused:
  1168. continue
  1169. import logging
  1170. logging.getLogger(__name__).warning('Session %s paused: %s', ref.session_id, exc.reason)
  1171. self._pause(ref, exc.reason)
  1172. return None
  1173. except (OSError, sqlite3.OperationalError) as exc:
  1174. # A writer held past its patience or a full disk is transient, and every _schedule
  1175. # caller is event-driven: a dead drain would strand the committed head and hang its
  1176. # waiters until the next submit. Retry in place (like recover_failed_settlement),
  1177. # keeping waiters: releasing them fails requests that would run a moment later.
  1178. if self._drain_stopping(ref, live):
  1179. return None
  1180. backoff = min(max(backoff * 2, _CLAIM_RETRY_MIN_S), _CLAIM_RETRY_MAX_S)
  1181. import logging
  1182. logging.getLogger(__name__).warning(
  1183. 'Session %s claim deferred by storage (%r); retrying in %.1fs', ref.session_id, exc, backoff)
  1184. await asyncio.sleep(backoff)
  1185. continue
  1186. backoff = 0.0
  1187. if row != 'classic_busy':
  1188. return row, first
  1189. if self._drain_stopping(ref, live):
  1190. return None
  1191. await asyncio.sleep(_CLAIM_RETRY_MIN_S)
  1192. async def _execute_claimed(self, ref, live, row):
  1193. """Run the claimed admission: ``(response, outcome)``. The executing stamp clears the moment
  1194. execution returns; a lost managed worker settles an automation turn (``settled_unknown``) or pauses
  1195. the FIFO and re-raises ManagedExecutionUnknown."""
  1196. from gateway.session_finite import execute_finite_admission
  1197. from gateway.session_managed_worker import ManagedExecutionUnknown
  1198. review = getattr(live.review, '_background_review_run', None)
  1199. if review is not None and not review.request_done.is_set():
  1200. # A cold /refine's review shares the session id, as the resident agent's does: cancel it
  1201. # before the turn (its bounded wait runs off-loop).
  1202. from agent.background_review import cancel_background_review_for_live_turn
  1203. await asyncio.to_thread(cancel_background_review_for_live_turn, live.review)
  1204. try:
  1205. try:
  1206. return await execute_finite_admission(self, ref, row), 'completed'
  1207. finally:
  1208. with live.event_stream.lock:
  1209. self.executing.pop(ref.session_id, None)
  1210. except ManagedExecutionUnknown:
  1211. with live.event_stream.lock:
  1212. live.event_stream.execution = {}
  1213. self.pending_stops.pop(ref.session_id, None)
  1214. self.pending_stop_causes.pop(ref.session_id, None)
  1215. self.pending_steers.pop(ref.session_id, None)
  1216. self.adopted.pop(ref.session_id, None)
  1217. # An automation turn nobody waits on settles here, at the exit that fenced it (GW-10),
  1218. # so the followers already queued run in this drain; a person's turn still pauses.
  1219. from gateway.session_unattended_unknown import settle_unattended_unknown
  1220. if await settle_unattended_unknown(self, ref):
  1221. return None, 'settled_unknown'
  1222. self._pause(ref, 'unknown_execution')
  1223. raise
  1224. except Exception:
  1225. import logging
  1226. logging.getLogger(__name__).exception('Admitted turn %s failed', row['admission_id'])
  1227. return 'The admitted turn failed.', 'failed'
  1228. async def _drain(self, ref):
  1229. from gateway.session_managed_worker import ManagedExecutionUnknown
  1230. live = self.sessions[ref.session_id]
  1231. while True:
  1232. claimed = await self._claim_with_retry(ref, live)
  1233. if claimed is None:
  1234. return
  1235. row, first = claimed
  1236. if row is False:
  1237. continue
  1238. if row is None and first is not None:
  1239. # A live worker (or a claim this drain does not own) blocks the head: the same
  1240. # pause episode, so its notice is not repeated for every message queued behind it.
  1241. self._pause(ref, 'session_busy')
  1242. return
  1243. # The FIFO is moving again (or empty): the next pause is a new episode.
  1244. live.pause_notified = False
  1245. if row is None:
  1246. from gateway.session_acp_lifecycle import end_idle_acp_session
  1247. # A write (idle check + stamp in one txn): off-loop. Input scheduled meanwhile set
  1248. # ``rescan`` on this still-live drain, which must claim it rather than exit.
  1249. from gateway.session_runtime_workers import tracked_write
  1250. await tracked_write(self, partial(end_idle_acp_session, self, ref.session_id))
  1251. if live.rescan:
  1252. continue
  1253. from gateway.session_viewer_close import settle_viewer_close
  1254. await settle_viewer_close(self, ref.session_id) # a close that arrived mid-turn
  1255. if live.rescan:
  1256. continue
  1257. from gateway.session_retirement import retire_at_idle
  1258. retire_at_idle(self, ref, live)
  1259. return
  1260. admission_id = row['admission_id']
  1261. try:
  1262. response, outcome = await self._execute_claimed(ref, live, row)
  1263. if outcome == 'settled_unknown':
  1264. self.deferred_resets.pop(live.route, None) # no settlement follows, as below
  1265. continue
  1266. except BaseException as exc:
  1267. # No settlement follows (a lost managed worker, a cancelled drain): the turn is never
  1268. # replayed, so its deferred reset is dropped (the next turn re-detects the overflow),
  1269. # never left to fire after a later settled turn with a stale entry.
  1270. self.deferred_resets.pop(live.route, None)
  1271. from gateway.session_ingress import fire_post_delivery
  1272. await fire_post_delivery(self, admission_id) # cancelled mid-turn: as an adapter task's finally
  1273. if isinstance(exc, ManagedExecutionUnknown):
  1274. return
  1275. raise
  1276. from gateway.session_runtime_workers import track_mutation
  1277. from gateway.session_settlement_recovery import settle_and_deliver
  1278. delivery = track_mutation(self, settle_and_deliver(self, live, ref, row, response, outcome))
  1279. try:
  1280. advance = await asyncio.shield(delivery)
  1281. except asyncio.CancelledError:
  1282. # Keep this FIFO's drain live until the committed reply has been handed off.
  1283. # Cleanup itself is separately tracked and must not delay the next turn.
  1284. try:
  1285. await asyncio.shield(delivery)
  1286. except Exception:
  1287. import logging
  1288. logging.getLogger(__name__).exception(
  1289. 'Delivery of admission %s failed after its drain was cancelled', row['admission_id'])
  1290. raise
  1291. if not advance:
  1292. return
  1293. @staticmethod
  1294. async def _run_deferred_reset(route, reset):
  1295. """The auto-reset the settled turn deferred (``_hmwa_defer_exhaustion_reset``), before its
  1296. waiter wakes: a delivery retry or the next DM then reads the fresh target."""
  1297. if reset is not None:
  1298. try:
  1299. await reset()
  1300. except Exception:
  1301. import logging
  1302. logging.getLogger(__name__).exception('Deferred auto-reset of %s failed', route)
  1303. async def _resolve_settled_waiter(self, admission_id, response):
  1304. """Hand the settled reply to its waiter. The adapter sending it adopted the turn's crash marker
  1305. and releases it once the reply is ledgered (seconds with auto-TTS); a follower claimed before
  1306. that replaces the one-slot marker, so a kill in between would lose the persisted reply: the
  1307. drain waits for that release (bounded; never while draining or retiring)."""
  1308. adopter = getattr(self, 'marker_adopters', {}).get(admission_id)
  1309. released = None
  1310. if adopter is not None and getattr(adopter, '_gateway_active_turn_token', None):
  1311. released = adopter._gateway_marker_released = asyncio.Event()
  1312. waiter = self.waiters.pop(admission_id, None)
  1313. if waiter is None or waiter.done():
  1314. return
  1315. waiter.set_result(response)
  1316. if released is not None and not (self.runner._draining or self.retiring):
  1317. with contextlib.suppress(TimeoutError):
  1318. await asyncio.wait_for(released.wait(), _MARKER_RELEASE_WAIT_S)
  1319. def backfill_legacy_session_owners(db):
  1320. """Stamp this store's own profile onto pre-ownership NULL rows (#94724).
  1321. The owner is the only writer while it holds the profile, so the offline REST backfill is
  1322. refused for as long as it runs; doing it here is what migrates an upgraded install. Idempotent
  1323. and never overwrites an owner, so every start runs it. A store outside the profile tree has
  1324. no derivable owner and keeps NULL.
  1325. """
  1326. import logging
  1327. stamp = db._own_profile_name()
  1328. if not stamp:
  1329. return 0
  1330. try:
  1331. stamped = db.backfill_null_session_profiles(stamp)
  1332. except Exception:
  1333. logging.getLogger(__name__).warning('Legacy session owner backfill failed for %s', db.db_path, exc_info=True)
  1334. return 0
  1335. if stamped:
  1336. logging.getLogger(__name__).info('Stamped %d legacy session row(s) with owner profile %r', stamped, stamp)
  1337. return stamped
  1338. def backfill_compression_created_sources(db):
  1339. """List conversations compressed before write-time stamping under their surface again: a chain
  1340. root without a created_source takes its nearest stamped continuation's. Idempotent and never
  1341. overwrites a known value, so every start runs it, like the owner backfill above."""
  1342. import logging
  1343. try:
  1344. stamped = db.stamp_legacy_compression_chains()
  1345. except Exception:
  1346. logging.getLogger(__name__).warning('Compression created_source repair failed for %s', db.db_path, exc_info=True)
  1347. return 0
  1348. if stamped:
  1349. logging.getLogger(__name__).info('Stamped created_source on %d compressed session row(s)', stamped)
  1350. return stamped
  1351. async def initialize_session_authority(runner, *, profile_id, instance_id, db=None, register=True):
  1352. """Call after exclusive profile ownership, before connecting adapters/API.
  1353. ``register=False`` builds a served secondary's authority without making it the runner's
  1354. launch authority (``runner.session_authority``); the per-home registry owns the lookup.
  1355. """
  1356. if db is None:
  1357. db = getattr(runner._session_db, '_db', runner._session_db)
  1358. # Resolved off-loop before any state is published: the rest of the bring-up runs without
  1359. # yielding, so no other task observes a half-registered authority.
  1360. from pathlib import Path
  1361. db_key = await asyncio.to_thread(Path(db.db_path).resolve)
  1362. epoch = await asyncio.to_thread(begin_runtime_epoch, db, instance_id=instance_id)
  1363. # Hot-serve builds a profile while the loop serves the others: both writes stay off it.
  1364. await asyncio.to_thread(recover_session_inputs, db, epoch=epoch)
  1365. # A lost automation turn has nobody to resolve it: it settles interrupted before any FIFO runs.
  1366. from gateway.session_unattended_unknown import settle_recovered_unattended
  1367. await asyncio.to_thread(settle_recovered_unattended, db, epoch)
  1368. await asyncio.to_thread(backfill_legacy_session_owners, db)
  1369. await asyncio.to_thread(backfill_compression_created_sources, db)
  1370. from gateway.session_local_recovery import release_handed_off_receipts
  1371. await asyncio.to_thread(release_handed_off_receipts, db, epoch=epoch)
  1372. from gateway.session_bot import release_superseded_bot_chat_at_boot
  1373. await asyncio.to_thread(release_superseded_bot_chat_at_boot, db, runner.session_store)
  1374. authority = SessionAuthority(runner, profile_id=profile_id, instance_id=instance_id, db=db, epoch=epoch)
  1375. # A receipt's compression catch-up (#140060) writes: it commits off the loop too, before the
  1376. # authority is published, so the recovery below only reads.
  1377. from gateway.session_local_adoption import adopt_local_compressions
  1378. await adopt_local_compressions(authority)
  1379. if register:
  1380. runner.session_authority = authority
  1381. from gateway.session_cron import bind_owner
  1382. bind_owner(authority)
  1383. store = runner.session_store
  1384. if register:
  1385. store._local_authority_epoch = epoch
  1386. # Local resets write the owning profile's store; the epoch fence must be that store's.
  1387. epochs = getattr(store, '_local_authority_epochs', None)
  1388. if epochs is None:
  1389. epochs = store._local_authority_epochs = {}
  1390. epochs[db_key] = epoch
  1391. from gateway.session_detached import watch_routes
  1392. watch_routes(runner, store)
  1393. from gateway.session_local_recovery import recover_local_sessions
  1394. recover_local_sessions(authority)
  1395. return authority