session_viewer_close.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396
  1. """The last viewer closing a session ends it on the owner, lifecycle included (#134889 LC-1/LC-2/LC-6).
  2. Main's classic CLI and TUI ran the end of a conversation in their own process: ``end_session``,
  3. the memory providers' ``on_session_end`` extraction and plugin ``on_session_finalize`` (which also
  4. closes the Relay conversation scope). Under the gateway those clients are viewers: the agent, its
  5. memory manager and the Relay scope live in the owner, so ``session.detach {end_reason}`` from the
  6. last viewer is where the owner runs them. A close that lands while a turn is still queued or running
  7. is remembered and completed when the FIFO goes idle, so exiting mid-turn still ends the row.
  8. The pending close is also kept in ``state_meta`` (``CLOSE_PREFIX + session_id``): a gateway stop or
  9. restart before the FIFO goes idle would otherwise drop it, and nothing else ends a local row. Boot
  10. recovery re-arms it (``rearm_persisted_closes``), so the restarted owner's first idle point ends it.
  11. Each stored close carries its own operation id, and every delete is compare-and-delete on that exact
  12. record under the owner-epoch fence: a cancelled older close, a reattach or a stale owner can only
  13. remove the record it saw, never a newer close stored since.
  14. """
  15. import asyncio
  16. import contextlib
  17. import json
  18. import logging
  19. import time
  20. import uuid
  21. from functools import partial
  22. from pathlib import Path
  23. from gateway.session_history_mount import is_history_mount
  24. from hermes_state_local import local_receipt
  25. from hermes_state_mutation_retirement import VIEWER_CLOSE_PREFIX
  26. from hermes_state_runtime import RuntimeStoreError
  27. logger = logging.getLogger(__name__)
  28. CLOSE_PREFIX = VIEWER_CLOSE_PREFIX # route retirement drops it too
  29. _BUSY_SQL = ("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status IN ('queued','started','unknown') "
  30. "UNION ALL SELECT 1 FROM worker_executions WHERE session_id IN (?,?) "
  31. "AND status IN ('registered','running','unknown') LIMIT 1")
  32. def _physical_target(authority, session_id):
  33. try:
  34. return local_receipt(authority.db, session_id)['entry']['session_id']
  35. except (RuntimeStoreError, KeyError, TypeError):
  36. return authority.db.get_compression_tip(session_id) or session_id
  37. def _busy(authority, session_id, target):
  38. with authority.db._read_ctx() as conn:
  39. return conn.execute(_BUSY_SQL, (session_id, session_id, target)).fetchone() is not None
  40. # The viewer's /new: the classic CLI's ``new_session``; Ink's ``tui_close`` (/new or a resume switch,
  41. # both of which main's TUI followed with on_session_reset for the session it opened next).
  42. _RESET_REASONS = frozenset({'new_session', 'tui_close'})
  43. # Ends nobody asked for (#143547 LC-35/LC-37): the last viewer vanished without closing (a killed
  44. # terminal, a dropped socket), or the previous gateway stopped under it. Both stay resumable.
  45. ORPHAN_REASONS = frozenset({'ws_orphan_reap', 'startup_orphan_reap'})
  46. # The legacy startup sweep's session TTL (tui_gateway ``_SESSION_TTL_S``): see arm_startup_orphan_closes.
  47. STARTUP_IDLE_S = 6 * 3600.0
  48. class _PendingClose:
  49. """One viewer close; identity, not the reason, tells a settler its close is still the pending one.
  50. ``record`` is its ``state_meta`` value: a unique operation id plus the reason."""
  51. __slots__ = ('reason', 'record')
  52. def __init__(self, reason, record=None):
  53. self.reason = reason
  54. self.record = record or json.dumps({'op': uuid.uuid4().hex, 'reason': reason}, sort_keys=True)
  55. @classmethod
  56. def from_record(cls, record):
  57. try:
  58. reason = json.loads(record)['reason']
  59. except (ValueError, TypeError, KeyError):
  60. reason = record # a bare-reason record from before operation ids
  61. return cls(reason, record)
  62. async def close_viewer_session(authority, session_id, reason):
  63. """``session.detach {end_reason}`` left *session_id* without viewers: close it now, or once idle.
  64. Its own async delegations end now, mid-turn too, as main's CLI exit and TUI close ended them
  65. (#134889 LC-7); each returns as an interrupted completion, and ones its still-running turn starts
  66. later are refused until a viewer attaches again. Other sessions' work is untouched."""
  67. live = authority.sessions[session_id]
  68. if live.subscribers:
  69. return # a viewer attached while the detach was off the loop: nothing closes or ends
  70. if is_history_mount(live):
  71. return # stored history: no route, agent or delegation of its own
  72. from tools.async_delegation import interrupt_for_session, refuse_for_session
  73. route = live.route
  74. parents = tuple(dict.fromkeys((session_id, _physical_target(authority, session_id))))
  75. refuse_for_session((authority.profile_id, session_id), session_key=route, parent_session_ids=parents)
  76. for parent in parents:
  77. interrupt_for_session(session_key=route, parent_session_id=parent, reason=f'viewer_close:{reason}')
  78. authority.pending_closes[session_id] = close = _PendingClose(reason)
  79. await _store_and_settle(authority, session_id, live, close)
  80. async def _store_and_settle(authority, session_id, live, close):
  81. """Persist *close* (already pending in memory), then complete it at once if the FIFO is idle."""
  82. await asyncio.to_thread(_persist_close, authority, session_id, close.record)
  83. if not _still_pending(authority, session_id, live, close):
  84. # A viewer attached during the write: its own clear may have run before this commit. Only
  85. # this close's record goes; a newer close stored since keeps its own.
  86. await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)
  87. return
  88. await settle_viewer_close(authority, session_id)
  89. def _persist_close(authority, session_id, record):
  90. from hermes_state_runtime import _epoch
  91. def write(conn):
  92. _epoch(conn, authority.epoch)
  93. conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
  94. (CLOSE_PREFIX + session_id, record))
  95. authority.db._execute_write(write)
  96. def forget_persisted_close(authority, session_id, record):
  97. """Blocking: drop *session_id*'s stored close only if it is still exactly *record* (that close
  98. completed or was cancelled), and only while this authority owns the store. Returns whether a
  99. row went."""
  100. from hermes_state_runtime import _epoch
  101. def write(conn):
  102. _epoch(conn, authority.epoch)
  103. return conn.execute('DELETE FROM state_meta WHERE key=? AND value=?',
  104. (CLOSE_PREFIX + session_id, record)).rowcount
  105. try:
  106. return bool(authority.db._execute_write(write))
  107. except RuntimeStoreError as exc:
  108. logger.info('Stored close for %s left in place: %s', session_id, exc)
  109. return False
  110. def cancel_persisted_close(authority, session_id):
  111. """Blocking, for a viewer attach: drop the stored close unless it is the one now pending in
  112. memory (a detach that ran after the attach stored it; it is that close's own to settle)."""
  113. record = authority.db.get_meta(CLOSE_PREFIX + session_id)
  114. if record is None:
  115. return
  116. pending = authority.pending_closes.get(session_id)
  117. if pending is not None and pending.record == record:
  118. return
  119. forget_persisted_close(authority, session_id, record)
  120. def rearm_persisted_closes(authority):
  121. """Boot/serve recovery, after the local sessions are restored: every stored close whose session
  122. is mounted with no viewer goes back to ``pending_closes``. The caller schedules the drain, whose
  123. idle point (after any queued turn; never past an ``unknown`` one) completes it. Returns the ids."""
  124. with authority.db._read_ctx() as conn:
  125. stored = conn.execute('SELECT key, value FROM state_meta WHERE key LIKE ?', (CLOSE_PREFIX + '%',)).fetchall()
  126. armed = []
  127. for key, record in stored:
  128. session_id = key[len(CLOSE_PREFIX):]
  129. live = authority.sessions.get(session_id)
  130. if live is None or is_history_mount(live):
  131. continue # not restored (paused, retired): left stored for a later recovery
  132. if live.subscribers:
  133. forget_persisted_close(authority, session_id, record)
  134. continue
  135. authority.pending_closes.setdefault(session_id, _PendingClose.from_record(record))
  136. armed.append(session_id)
  137. return armed
  138. def _seconds(raw, default):
  139. with contextlib.suppress(ValueError, TypeError):
  140. return max(0.0, float(raw))
  141. return default
  142. def _section(config, key):
  143. value = config.get(key)
  144. return value if isinstance(value, dict) else {}
  145. def orphan_close_grace(authority, *, startup=False):
  146. """Blocking: ``sessions.orphan_end_grace_s`` of the profile that owns *authority*'s store (0 = never).
  147. Unset, the legacy backend's switches for the same reaping still opt out: ``dashboard.
  148. ws_orphan_reap_grace_s: 0`` (park forever) and, for *startup* arms, ``startup_orphan_sweep: false``."""
  149. from hermes_cli.config_defaults import DEFAULT_CONFIG
  150. from hermes_cli.config_effective import load_user_config_effective
  151. from gateway.session_local import profile_env_scope
  152. from utils import is_truthy_value
  153. default = DEFAULT_CONFIG['sessions']['orphan_end_grace_s']
  154. home = Path(authority.db.db_path).resolve().parent
  155. # A dropped socket detaches outside the RPC's owner scope: expand ``${VAR}`` against this profile.
  156. with profile_env_scope(home):
  157. config = load_user_config_effective(home / 'config.yaml')
  158. sessions, dashboard = _section(config, 'sessions'), _section(config, 'dashboard')
  159. if 'orphan_end_grace_s' in sessions:
  160. return _seconds(sessions['orphan_end_grace_s'], default)
  161. legacy_off = _seconds(dashboard.get('ws_orphan_reap_grace_s'), default) == 0 or (
  162. startup and not is_truthy_value(dashboard.get('startup_orphan_sweep'), default=True))
  163. return 0.0 if legacy_off else default
  164. def orphan_candidate(authority, session_id):
  165. """Blocking: *session_id* is a local session with an open row. ACP ends on its own at detach."""
  166. try:
  167. receipt = local_receipt(authority.db, session_id)
  168. row = authority.db.get_session(receipt['entry']['session_id'])
  169. except (RuntimeStoreError, KeyError, TypeError):
  170. return False # not a local session
  171. return receipt.get('policy', {}).get('source') != 'acp' and row is not None and row['ended_at'] is None
  172. def _call_later(loop, delay, callback):
  173. # The loop's monotonic clock: time the host spends asleep does not count toward the grace.
  174. return loop.call_later(delay, callback)
  175. def _stopping(authority):
  176. return authority.retiring or authority.retired or authority.runner._draining
  177. def arm_orphan_close(authority, session_id, reason, grace):
  178. """*session_id* has no viewer and nobody closed it: unless one attaches within *grace* seconds,
  179. close it as a viewer would, without stopping its turn or its async delegations (#106742). A
  180. stopping owner arms nothing; the next boot does (``recover_local_sessions``)."""
  181. live = authority.sessions.get(session_id)
  182. if not grace or live is None or live.subscribers or is_history_mount(live) or _stopping(authority):
  183. return
  184. cancel_orphan_close(authority, session_id)
  185. authority.orphan_closes[session_id] = _call_later(
  186. asyncio.get_running_loop(), grace, partial(_fire_orphan_close, authority, session_id, reason))
  187. async def arm_detached_orphan_close(authority, session_id):
  188. """``session.detach`` with no ``end_reason`` (a killed terminal, a dropped socket) left no viewer."""
  189. def grace():
  190. return orphan_close_grace(authority) if orphan_candidate(authority, session_id) else 0
  191. arm_orphan_close(authority, session_id, 'ws_orphan_reap', await asyncio.to_thread(grace))
  192. def _idle_s(authority, session_id):
  193. from hermes_state_common import _sql_session_last_active
  194. # Resolve the target BEFORE opening a read context: ``_physical_target`` reads the receipt, and
  195. # a read that lands on the degraded path (non-WAL, or every read permit spent) waits on the
  196. # writer's non-reentrant lock. Nested inside this block it would wait on the lock this thread
  197. # already holds — a boot-time self-deadlock the watchdog can only answer by killing the gateway.
  198. target = _physical_target(authority, session_id)
  199. with authority.db._read_ctx() as conn:
  200. row = conn.execute(f'SELECT {_sql_session_last_active("s")} FROM sessions s WHERE s.id = ?',
  201. (target,)).fetchone()
  202. return time.time() - row[0] if row and row[0] else 0.0
  203. def arm_startup_orphan_closes(authority, session_ids):
  204. """Boot, after the stored closes are re-armed: the previous owner stopped under these sessions
  205. (idle, or with input it left queued), so no viewer is left to close them (#143547 LC-37). One
  206. that reattaches keeps it open; a fired close still waits for the queued input to settle.
  207. An idle classic CLI only redials on its next call, so a session idle less than the legacy
  208. sweep's TTL waits out the rest of it, as that sweep did, before its grace starts to count."""
  209. grace = orphan_close_grace(authority, startup=True)
  210. for session_id in session_ids:
  211. if grace and session_id not in authority.pending_closes and orphan_candidate(authority, session_id):
  212. delay = max(grace, STARTUP_IDLE_S - _idle_s(authority, session_id))
  213. arm_orphan_close(authority, session_id, 'startup_orphan_reap', delay)
  214. def cancel_orphan_close(authority, session_id):
  215. handle = authority.orphan_closes.pop(session_id, None)
  216. if handle is not None:
  217. handle.cancel()
  218. def _fire_orphan_close(authority, session_id, reason):
  219. authority.orphan_closes.pop(session_id, None)
  220. live = authority.sessions.get(session_id)
  221. if live is None or live.subscribers or session_id in authority.pending_closes or _stopping(authority):
  222. return
  223. # The persisted pending close a viewer's own close uses: it completes once the turn settles, and
  224. # survives a restart. Never ``close_viewer_session``: that would interrupt the delegations.
  225. authority.pending_closes[session_id] = close = _PendingClose(reason)
  226. from gateway.session_runtime_workers import track_mutation
  227. track_mutation(authority, _store_and_settle(authority, session_id, live, close))
  228. async def settle_viewer_close(authority, session_id):
  229. """Complete a pending viewer close once *session_id* has no viewer and an idle FIFO: stamp the
  230. row with the viewer's reason, commit the cached agent's memory and fire ``on_session_finalize``.
  231. The drain calls this at every idle point; a viewer attaching meanwhile cancels the close."""
  232. close = authority.pending_closes.get(session_id)
  233. live = authority.sessions.get(session_id)
  234. if close is None or live is None or live.subscribers:
  235. authority.pending_closes.pop(session_id, None)
  236. return
  237. if not await _resolve_abandoned_unknown(authority, session_id, live, close):
  238. return # left paused (a viewer came back, or the resolution was refused)
  239. target = _physical_target(authority, session_id)
  240. # Every await below can let a viewer reattach (attach drops the pending close) or another
  241. # settler run, so the close is re-checked after each one.
  242. if await asyncio.to_thread(_busy, authority, session_id, target):
  243. return # the drain's next idle point completes it
  244. idle_stamp = live.event_stream.execution # each claim stamps a new one (``_stamp_claim``)
  245. row = await asyncio.to_thread(authority.db.get_session, target)
  246. if not _still_pending(authority, session_id, live, close):
  247. return
  248. # Claimed with no await since the check: exactly one settler stamps the row and runs the
  249. # lifecycle, and a close cancelled before its claim never wrote ``ended_at``.
  250. del authority.pending_closes[session_id]
  251. from gateway.session_acp_lifecycle import end_idle_viewer_session, reopen_if_reattached
  252. from gateway.session_runtime_workers import tracked_write
  253. ended = await tracked_write(authority, partial(end_idle_viewer_session, authority, session_id, close.reason),
  254. then=partial(reopen_if_reattached, authority, session_id))
  255. if not ended:
  256. await _requeue_if_busy(authority, session_id, live, target, close)
  257. return
  258. await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record)
  259. if live.subscribers:
  260. return # a viewer reattached during the stamp; reopen_if_reattached reopened the row
  261. platform = (row or {}).get('source') or (live.source.platform.value if live.source else '')
  262. from gateway.session_vault_prompt import release_unlocks
  263. release_unlocks(session_id)
  264. await _run_close_lifecycle(authority, live, target, platform, close.reason)
  265. if not live.subscribers: # ended: no viewer is left to replay to; a resume refetches history
  266. live.event_stream.release()
  267. from gateway.session_retirement import retire_closed
  268. retire_closed(authority, session_id, live, target, idle_stamp)
  269. async def _resolve_abandoned_unknown(authority, session_id, live, close):
  270. """A turn the previous owner lost (hard kill) recovers ``unknown`` and pauses the FIFO until
  271. someone Discards it, but the last viewer closed the session: nobody is left to. The viewer's
  272. close is that decision, so the owner resolves it as Discard does (interrupted, never replayed,
  273. transcript boundary written) and the drain moves on; a queued successor still runs before the
  274. close. A viewer back first cancels the close and keeps the decision. Returns whether the
  275. close may go on to its idle check (no ``unknown`` turn is left)."""
  276. from hermes_state_runtime import list_session_admissions
  277. rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=session_id)
  278. unknown = [row for row in rows if row['status'] == 'unknown']
  279. if not unknown:
  280. return True
  281. if close.reason in ORPHAN_REASONS:
  282. return False # nobody closed it: the turn waits for its user's own Discard
  283. from gateway.session_contract import SessionRef
  284. ref = SessionRef(authority.profile_id, session_id)
  285. for row in unknown:
  286. if not _still_pending(authority, session_id, live, close):
  287. return False
  288. try:
  289. await authority.resolve_unknown_as_owner(ref, row['admission_id'], row['generation'])
  290. except RuntimeStoreError as exc:
  291. logger.warning('Unknown turn %s on closed session %s left paused: %s',
  292. row['admission_id'], session_id, exc.reason)
  293. return False
  294. logger.info('Unknown turn %s resolved interrupted: its session %s was closed by its last viewer',
  295. row['admission_id'], session_id)
  296. # The resolution also rescheduled the drain; the idle check below and its next idle point
  297. # are both safe (the close is claimed exactly once).
  298. return _still_pending(authority, session_id, live, close)
  299. def _still_pending(authority, session_id, live, close):
  300. return not live.subscribers and authority.pending_closes.get(session_id) is close
  301. async def _requeue_if_busy(authority, session_id, live, target, close):
  302. """The stamp was refused. If input was admitted after the idle check, the close goes back to the
  303. drain's next idle point; any other refusal (row already ended) leaves nothing to finish."""
  304. if not await asyncio.to_thread(_busy, authority, session_id, target):
  305. if session_id not in authority.pending_closes:
  306. await asyncio.to_thread(forget_persisted_close, authority, session_id, close.record) # already ended
  307. return
  308. if live.subscribers or session_id in authority.pending_closes:
  309. return # a reattach cancelled it, or a newer close replaced it
  310. authority.pending_closes[session_id] = close
  311. # The drain may have passed its idle point while the close was out of the map.
  312. await settle_viewer_close(authority, session_id)
  313. async def _run_close_lifecycle(authority, live, target, platform, reason):
  314. """What main's client ran at exit and ``/new``, in the process that owns the session: the
  315. memory providers' end-of-session commit (the cached agent is evicted at a boundary, as a
  316. gateway ``/new`` does) and the finalize hooks under the owning profile's scope."""
  317. runner = authority.runner
  318. evict = getattr(runner, '_evict_cached_agent_at_boundary', None)
  319. if evict is not None:
  320. evict(live.route)
  321. finalize = getattr(runner, '_finalize_session_off_loop', None)
  322. if finalize is None:
  323. return
  324. messages = await finalize(session_id=target, platform=platform, reason=reason, session_key=live.route)
  325. if messages:
  326. # No viewer is left to show them (session.detach carries no messages yet).
  327. logger.info('Session-end plugin message(s) for %s dropped: no viewer', target)
  328. reset = getattr(runner, '_reset_session_off_loop', None)
  329. if reason in _RESET_REASONS and reset is not None:
  330. # The gateway's own /new arguments. The viewer opens its next session only after this
  331. # detach, so the new id is not known here.
  332. await reset(session_key=live.route, session_id=None, reason='new_session', platform=platform,
  333. old_session_id=target, new_session_id=None)