session_local_recovery.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. """Restore only private, profile-bound local policy into the existing authority."""
  2. import asyncio
  3. import contextlib
  4. import json
  5. import logging
  6. from pathlib import Path
  7. from gateway.config import Platform
  8. from gateway.session import SessionEntry, SessionSource
  9. from gateway.session_contract import SessionRef
  10. from hermes_state_keys import local_identity
  11. from hermes_state_local import POLICY_PREFIX, local_receipt
  12. from hermes_state_runtime import RuntimeStoreError, list_session_admissions
  13. def local_adapter_map(authority):
  14. """The adapter map a LOCAL session of *authority* lives in: ``runner.adapters`` for the launch
  15. profile, ``runner._profile_adapters[name]`` for a served secondary (its routed home, so
  16. ``_adapter_for_source`` / ``_adapters_for_profile`` resolve the right transport)."""
  17. runner = authority.runner
  18. registry = getattr(runner, 'session_authorities', None)
  19. name = registry.profile_name(authority) if registry is not None else None
  20. if name is None:
  21. return runner.adapters
  22. profile_adapters = getattr(runner, '_profile_adapters', None)
  23. if profile_adapters is None:
  24. profile_adapters = runner._profile_adapters = {}
  25. return profile_adapters.setdefault(name, {})
  26. def local_adapter(authority):
  27. """The LOCAL adapter *authority*'s local sessions, and the conversations its chats moved past
  28. (gateway/session_detached.py), deliver through: created on first use."""
  29. adapters = local_adapter_map(authority)
  30. if (adapter := adapters.get(Platform.LOCAL)) is None:
  31. from gateway.session_local import LocalSessionAdapter
  32. adapter = adapters[Platform.LOCAL] = LocalSessionAdapter(authority)
  33. return adapter
  34. def local_source(authority, chat_id, user_id):
  35. """LOCAL SessionSource for a session owned by *authority*. A served secondary's local sessions
  36. carry its profile so ``build_session_key`` namespaces them (``agent:<name>:local:...``) and the
  37. SessionStore writes their rows to that profile's ``state.db``, not the launch store."""
  38. source = SessionSource(platform=Platform.LOCAL, chat_id=chat_id, user_id=user_id, chat_type='dm')
  39. registry = getattr(authority.runner, 'session_authorities', None)
  40. name = registry.profile_name(authority) if registry is not None else None
  41. if name is not None:
  42. source.profile = name
  43. return source
  44. def _refused_check(exc):
  45. """``<reason or error> at <function>:<line>``: where a receipt check raised, and why, followed
  46. through every chained ``__cause__`` (``restore_policy`` and ``local_receipt`` wrap the check
  47. that failed into a bare ``storage_unavailable``)."""
  48. import traceback
  49. frames = traceback.extract_tb(exc.__traceback__)
  50. cause = exc.reason if isinstance(exc, RuntimeStoreError) else f'{type(exc).__name__}: {exc}'
  51. where = f' at {frames[-1].name}:{frames[-1].lineno}' if frames else ''
  52. chained = f', caused by {_refused_check(exc.__cause__)}' if exc.__cause__ is not None else ''
  53. return f'{cause}{where}{chained}'
  54. class _ReceiptRefused(RuntimeStoreError):
  55. """A refused receipt ``restore_local_session`` already logged with its failing check."""
  56. class _CatchUpPending(RuntimeStoreError):
  57. """The receipt's compression catch-up is committing in a worker; this restore is refused."""
  58. def _with_untracked_compression(authority, sid, receipt):
  59. """*receipt*, after recording compression steps an older build published past its tip (#140060).
  60. The catch-up writes, so it never runs on the owner loop: a contended writer there would stall every
  61. profile the loop serves. The async restore paths commit it in a worker first
  62. (``session_local_adoption``); a sync restore on the loop that still finds it hands it to a tracked
  63. worker, which publishes the session once it commits, and refuses this read.
  64. """
  65. from hermes_state_local_lineage import adopt_untracked_compression, adoptable_compression
  66. with authority.db._read_ctx() as conn:
  67. if adoptable_compression(conn, receipt) is None:
  68. return receipt
  69. from gateway.session_local_adoption import on_owner_loop, schedule_adoption
  70. if on_owner_loop():
  71. schedule_adoption(authority, sid, receipt)
  72. raise _CatchUpPending('storage_unavailable')
  73. adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)
  74. return local_receipt(authority.db, sid)
  75. def _checked_entry(authority, receipt, source, route):
  76. """*receipt*'s entry, once its route, origin and stored identity are this authority's."""
  77. if receipt['route'] != route:
  78. raise ValueError('route mismatch')
  79. entry = SessionEntry.from_dict(receipt['entry'])
  80. # Older local receipts omitted the served profile from the origin. The receipt's
  81. # home and route already matched this authority; fill only that absent field,
  82. # never accept an explicitly different profile or change the durable route.
  83. if 'profile' not in receipt['entry']['origin']:
  84. entry.origin.profile = source.profile
  85. if (entry.session_key != route
  86. or entry.origin.to_dict() != source.to_dict()):
  87. raise ValueError('entry mismatch')
  88. row = authority.db.get_session(entry.session_id)
  89. if row is None or (row['session_key'], row['chat_id'], row['user_id']) != (route, source.chat_id, source.user_id):
  90. raise ValueError('stored identity mismatch')
  91. return entry
  92. def owned_receipt(authority, sid):
  93. """*sid*'s stored receipt with its LOCAL source, route and entry, once all are this owner's."""
  94. receipt = local_receipt(authority.db, sid)
  95. if receipt['profile_id'] != authority.profile_id:
  96. raise RuntimeStoreError('profile_mismatch')
  97. chat_id = (receipt['legacy_session_id'] if 'legacy_session_id' in receipt else
  98. local_identity(receipt['profile_id'], receipt['principal_id'], receipt['request_id']))
  99. if receipt['session_id'] != sid or sid != chat_id:
  100. raise ValueError('identity mismatch')
  101. source = local_source(authority, chat_id, receipt['principal_id'])
  102. route = authority.runner.session_store._generate_session_key(source)
  103. return receipt, source, route, _checked_entry(authority, receipt, source, route)
  104. def unused_session(conn, session_id):
  105. """*session_id* holds nothing: no transcript, ledger rows or children (what the SessionStore's
  106. stale-route heal, #54878, mints as a route's replacement before any turn runs on it)."""
  107. return conn.execute(
  108. 'SELECT 1 FROM messages WHERE session_id=? UNION ALL SELECT 1 FROM session_admissions '
  109. 'WHERE target_session_id=? UNION ALL SELECT 1 FROM worker_executions WHERE session_id=? '
  110. 'UNION ALL SELECT 1 FROM sessions WHERE parent_session_id=? LIMIT 1',
  111. (session_id, session_id, session_id, session_id)).fetchone() is None
  112. def _stray_route_target(authority, receipt, session_id):
  113. """*session_id* is a replacement the SessionStore's stale-route self-heal minted on this local
  114. route before #140642: outside the receipt's lineage, never admitted, no transcript. The receipt
  115. is the route's truth, so restoring it drops that entry instead of refusing the conversation
  116. forever (``admission_conflict`` on every resume). Anything with history or ledger rows still
  117. refuses: it is not provably a stray."""
  118. if not session_id or session_id in receipt.get('lineage', [receipt['session_id']]):
  119. return False
  120. with authority.db._read_ctx() as conn:
  121. if not unused_session(conn, session_id):
  122. return False
  123. logging.getLogger(__name__).warning(
  124. 'Local route for %s named an empty replacement %s outside its lineage; restoring the receipt target',
  125. receipt['session_id'], session_id)
  126. return True
  127. def restore_local_session(authority, sid):
  128. from gateway.session_authority import LiveSession
  129. from gateway.session_local import LocalSessionAdapter
  130. from gateway.session_policy import restore_policy
  131. try:
  132. # The catch-up writes, so the receipt as stored must already be this owner's (#140060).
  133. receipt, source, route, entry = owned_receipt(authority, sid)
  134. chat_id, store = source.chat_id, authority.runner.session_store
  135. adopted = _with_untracked_compression(authority, sid, receipt)
  136. policy = restore_policy(adopted['policy'])
  137. if adopted is not receipt: # re-read after a catch-up: check the copy that will be served
  138. receipt, entry = adopted, _checked_entry(authority, adopted, source, route)
  139. target = entry.session_id
  140. from hermes_state_local_lineage import validate_local_lineage
  141. with authority.db._read_ctx() as conn:
  142. validate_local_lineage(conn, receipt)
  143. except _CatchUpPending:
  144. raise
  145. except (KeyError, TypeError, ValueError, AttributeError) as exc:
  146. # The client only sees the bounded code: gateway.log names the check that refused (#140099).
  147. logging.getLogger(__name__).warning('Local session %s receipt refused: %s', sid, _refused_check(exc))
  148. if isinstance(exc, RuntimeStoreError):
  149. raise _ReceiptRefused(exc.reason, field=exc.field, detail=exc.detail) from exc
  150. raise _ReceiptRefused('storage_unavailable') from exc
  151. adapter = local_adapter(authority)
  152. if not isinstance(adapter, LocalSessionAdapter) or adapter.authority is not authority:
  153. raise RuntimeStoreError('runtime_draining')
  154. live = authority.sessions.get(sid)
  155. from gateway.session_history_mount import is_history_mount
  156. if live is None or is_history_mount(live):
  157. source = adapter.kept_source(source)
  158. with store._lock:
  159. store._ensure_loaded_locked()
  160. current = store._entries.get(route)
  161. if current is not None and current.session_id != target:
  162. if not _stray_route_target(authority, receipt, current.session_id):
  163. raise RuntimeStoreError('admission_conflict')
  164. current = None
  165. # The receipt is the creation copy; a persisted entry for the same target keeps the
  166. # route's explicit ``/yolo`` toggle (an OFF over a ``--yolo`` launch must survive a
  167. # cold restore). An untoggled entry stays None and keeps following the launch policy.
  168. if current is not None:
  169. entry.yolo = current.yolo
  170. entry.origin = source
  171. store._entries[route] = entry
  172. adapter.policies[chat_id] = policy
  173. if live is None:
  174. authority.sessions[sid] = LiveSession(source, route)
  175. authority.retired_local.discard(sid) # back in memory (session_retirement)
  176. else:
  177. # A read-only view opened before the adoption (a metadata read, another viewer) becomes the
  178. # adopted conversation in place, so its viewers keep their subscriptions (as restore_api_session).
  179. live.source, live.route = source, route
  180. adapter.register_source(source)
  181. elif (not adapter.authorize_source(live.source)
  182. or live.source.to_dict() != source.to_dict()
  183. or adapter.policies.get(chat_id) != policy):
  184. raise RuntimeStoreError('storage_unavailable')
  185. elif _route_already_restored(store, route, target, receipt.get('lineage', [sid])):
  186. pass # every claim of a live session lands here: no write, so no wait on the store lock
  187. else:
  188. with store._lock:
  189. current = store._entry_locked(route)
  190. if current is None:
  191. # A route the loaded store cannot see (an older release's scope, a pruned row) is the
  192. # receipt's to rebuild, as on a cold restore: refusing wedges every claim until a restart.
  193. entry.origin = source
  194. store._entries[route] = entry
  195. elif (current.session_id not in receipt.get('lineage', [sid])
  196. and not _stray_route_target(authority, receipt, current.session_id)):
  197. raise RuntimeStoreError('admission_conflict')
  198. else:
  199. current.session_id = target
  200. return SessionRef(authority.profile_id, sid)
  201. def _route_already_restored(store, route, target, lineage):
  202. """The loaded route already names *target*, a lineage member: the locked branch would only
  203. re-assign the same id. Read without ``store._lock`` (one dict lookup, atomic under the GIL),
  204. because worker threads hold that lock across state.db writes and the restore runs on the loop.
  205. Any other shape takes the lock and is re-checked there."""
  206. current = store._entries.get(route)
  207. return current is not None and current.session_id == target and target in lineage
  208. def transcript_target(authority, ref):
  209. """The current physical transcript of logical *ref* (a local receipt's entry, else the tip)."""
  210. live = authority.sessions[ref.session_id]
  211. if live.source is not None and live.source.platform == Platform.LOCAL:
  212. restore_local_session(authority, ref.session_id)
  213. return local_receipt(authority.db, ref.session_id)['entry']['session_id']
  214. return authority.physical_target(ref)
  215. # A resume reply's rows reach Ink twice (``messages`` and ``display_messages``). Archived rows are display-only
  216. # and fill only what the live rows leave of this budget, so they keep the reply well under the gateway clients'
  217. # 8 MiB WebSocket frame cap (``hermes_cli/gateway_client.py``) and never push a reply main could send past it.
  218. # Live rows are never dropped.
  219. RESUME_HISTORY_BYTES = 3 * 1024 * 1024
  220. def display_rows(db, target):
  221. """``(live row ids, display rows, truncated)`` of physical *target*, the live ids read first
  222. (``local_history``'s fence). The display rows are the compression lineage as main's resume and the
  223. REST transcript show it: every live row, and the newest compaction-archived rows that fit what the
  224. live rows leave of ``RESUME_HISTORY_BYTES`` (``truncated`` when older ones were left out). A lineage
  225. over the resume cap shows only the live rows."""
  226. from hermes_state import SessionResumeTooLargeError
  227. live_rows = db.get_messages_as_conversation(target, include_row_ids=True)
  228. live = {row['_row_id'] for row in live_rows}
  229. try:
  230. db.assert_resume_safe(target)
  231. except SessionResumeTooLargeError:
  232. return live, [], False
  233. kept, truncated = _fit_archive(live_rows, db.get_messages_as_conversation(
  234. target, include_ancestors=True, include_compacted=True, include_row_ids=True))
  235. return live, kept, truncated
  236. def _fit_archive(live_rows, display):
  237. """``(display rows kept, truncated)``: every live row of *display*, and its newest archived rows that fit
  238. what *live_rows* leave of ``RESUME_HISTORY_BYTES``."""
  239. live = {row['_row_id'] for row in live_rows}
  240. kept, used, truncated = [], sum(len(json.dumps(row, default=str)) for row in live_rows), False
  241. for row in reversed(display):
  242. if row['_row_id'] not in live:
  243. used += len(json.dumps(row, default=str))
  244. truncated = truncated or used > RESUME_HISTORY_BYTES
  245. if truncated:
  246. continue
  247. kept.append(row)
  248. return kept[::-1], truncated
  249. async def prepare_history(authority, ref):
  250. """``local_history``'s lineage read, off the owner loop; the locked part re-reads only the live tip."""
  251. target = transcript_target(authority, ref)
  252. return target, await asyncio.to_thread(display_rows, authority.db, target)
  253. def omitted_history_state(db, target):
  254. """``(todo_state, message_count)`` of a resume that omits its transcript: the physical *target*'s stored
  255. count and its todo plan. Skips the display-row lineage read; the todo lookup still walks the lineage's
  256. todo rows (removed ones included, so an undone plan stays gone)."""
  257. from tui_gateway.tool_progress import _todo_state_from_db
  258. row = db.get_session(target)
  259. if row is None: # deleted since the attach resolved it
  260. raise RuntimeStoreError('not_found')
  261. return _todo_state_from_db(db, target), int(row.get('message_count') or 0)
  262. def _merge_live_in_stored_order(display, live):
  263. """The live rows in stored (``row_id``) order, each archived display row placed before the live
  264. row it preceded in display order.
  265. Display deduplication keeps a carried message at its FIRST copy's position, so after a rotated
  266. compression the copied tail sits before the new summary. Clients take the live rows of a resume
  267. as the transcript (the CLI's ``conversation_history``, ``/undo``'s prefix, rollback, ACP replay),
  268. so the live subsequence must be the stored one; archived rows only anchor between live rows."""
  269. current = {row['_row_id'] for row in live}
  270. groups, pending, shown = [], [], 0
  271. for row in display:
  272. if row['_row_id'] in current:
  273. groups.append(pending)
  274. pending, shown = [], shown + 1
  275. else:
  276. pending.append({**row, 'archived': True})
  277. shown_ids = {row['_row_id'] for row in display} & current
  278. rows, seen = [], 0
  279. for row in sorted(live, key=lambda r: r['_row_id']):
  280. if row['_row_id'] in shown_ids:
  281. rows.extend(groups[seen])
  282. seen += 1
  283. rows.append(row)
  284. return rows + pending
  285. def local_history(authority, ref, prepared=None):
  286. """Display history of logical *ref*: every stored row the user saw, in display order.
  287. A row outside the current physical transcript's active set (summarized away by a compression)
  288. carries ``archived: true``: it is shown, never live, so no undo, retry or rewind may target it.
  289. The rows without it are exactly the live transcript's, so a client choosing among live rows sees
  290. what it saw before. That includes a micro-compaction merge (``display_metadata.model_only``): its
  291. originals are the archived rows renderers show, so renderers skip the merge itself.
  292. Each row carries its durable ``messages.id`` as ``row_id`` (the ``TranscriptMessage`` field
  293. Desktop hydration and the legacy ``session.history`` already use): a ``session.mutate`` rewind
  294. addresses its ``target_message_id`` by it, so a snapshot without it leaves no way to undo/retry.
  295. *prepared* (``prepare_history``) is kept while every row it read as live still is; a rewind or
  296. compaction since then re-reads the lineage here. When the byte budget left older archived rows out
  297. (``display_rows``), the first row carries ``archived_truncated: true``.
  298. """
  299. db = authority.db
  300. target = transcript_target(authority, ref)
  301. read_live, display, truncated = (prepared[1] if prepared is not None and prepared[0] == target
  302. else display_rows(db, target))
  303. live = db.get_messages_as_conversation(target, include_row_ids=True)
  304. live_ids = {row['_row_id'] for row in live}
  305. if not read_live <= live_ids:
  306. read_live, display, truncated = display_rows(db, target)
  307. live = db.get_messages_as_conversation(target, include_row_ids=True)
  308. elif live_ids - read_live:
  309. # Rows appended since the prepare count against the archive budget too (no lineage re-read).
  310. display, cut = _fit_archive(live, display)
  311. truncated = truncated or cut
  312. rows = [{**{k: v for k, v in row.items() if k != '_row_id'}, 'row_id': row['_row_id']}
  313. for row in _merge_live_in_stored_order(display, live)]
  314. if truncated and rows:
  315. rows[0]['archived_truncated'] = True # earlier archived history was left out (the byte budget)
  316. return rows
  317. def reopen_local_session(authority, ref):
  318. """The first admitted turn makes a finalized local session live again (#85303).
  319. Mounting (``session.resume``) is a read and leaves ``ended_at``/``end_reason`` alone; the drain
  320. calls this after the turn's local preflight, before execution routes it. A TUI shutdown / WS
  321. disconnect / idle eviction stamps ``end_reason`` on the physical row. ``SessionStore`` reads any
  322. stamped row as a stale route (#54878) and, since those reasons are not recoverable, answers the
  323. turn with a FRESH session — the client keeps waiting on an id that never emits again. The classic
  324. surfaces clear the stamp on the first real turn (tui_gateway ``_reopen_if_finalized``); the
  325. authority must too. Only the lineage tip is reopened: a
  326. compression parent or reset predecessor is never the physical target.
  327. """
  328. live = authority.sessions.get(ref.session_id)
  329. if live is None or live.source is None or live.source.platform != Platform.LOCAL:
  330. return
  331. target = local_receipt(authority.db, ref.session_id)['entry']['session_id']
  332. row = authority.db.get_session(target)
  333. if row is not None and row.get('end_reason') is not None:
  334. authority.db.reopen_session(target)
  335. def reset_local_session(store, old_entry, session_id, now, display_name):
  336. from hermes_state_local_lineage import reset_local_target
  337. entry = SessionEntry(old_entry.session_key, session_id, now, now,
  338. origin=old_entry.origin, platform=old_entry.platform, chat_type=old_entry.chat_type,
  339. display_name=display_name if display_name is not None else old_entry.display_name,
  340. is_fresh_reset=True)
  341. db = store._db_for_key(old_entry.session_key)
  342. epochs = getattr(store, '_local_authority_epochs', None) or {}
  343. epoch = epochs.get(Path(db.db_path).resolve(), store._local_authority_epoch) if db is not None else store._local_authority_epoch
  344. reset_local_target(db, epoch=epoch,
  345. parent_session_id=old_entry.session_id, entry=entry.to_dict())
  346. store._entries[old_entry.session_key] = entry
  347. return entry
  348. # The receipts boot restores: an open row (or one compressed away, for its catch-up), unsettled input
  349. # or a stored close. An ended session keeps its receipt and stays resumable; ``authorize`` restores it
  350. # on first touch, so the live registry no longer grows with every session ever opened (#143547 LC-36).
  351. _BOOT_RECEIPTS_SQL = (
  352. "SELECT m.key FROM state_meta m LEFT JOIN sessions s "
  353. "ON s.id = CASE WHEN json_valid(m.value) THEN json_extract(m.value, '$.entry.session_id') END "
  354. "WHERE m.key LIKE ? AND (s.ended_at IS NULL OR s.end_reason = 'compression' "
  355. "OR EXISTS (SELECT 1 FROM session_admissions a WHERE a.target_session_id = substr(m.key, ?) "
  356. "AND a.status IN ('queued','started','unknown')) "
  357. "OR EXISTS (SELECT 1 FROM state_meta c WHERE c.key = ? || substr(m.key, ?)))")
  358. def restore_completion_route(runner, session_key):
  359. """Background work that outlived its local session reports after a restart: boot left that ended
  360. session unrestored, so restore it first, as ``authorize`` does on a viewer's first touch, and its
  361. completion is noted on it as before. Anything else is left to the delivery's own checks."""
  362. from gateway.run import _parse_session_key
  363. from gateway.session_authorities import active_authority
  364. authority, parsed = active_authority(runner), _parse_session_key(session_key) or {}
  365. if authority is None or parsed.get('platform') != Platform.LOCAL.value or parsed['chat_id'] in authority.sessions:
  366. return
  367. with contextlib.suppress(RuntimeStoreError):
  368. restore_local_session(authority, parsed['chat_id'])
  369. def release_handed_off_receipts(db, *, epoch):
  370. """Boot heal for #143286: an older build's ``/handoff`` (or a platform ``/resume``) moved a local
  371. session onto a messaging lane but kept its local receipt, so every open, pin or rename refused it
  372. (``stored identity mismatch``) and its title never resolved. Retire each such receipt, so the row
  373. restores from its route like any other platform session, and move the rest of its conversation
  374. there with it (``hand_off_conversations``). Only receipts whose CURRENT target a messaging lane
  375. now holds: any other refused receipt stays as it is, logged when it is touched."""
  376. from gateway.session_route_recovery import _messaging_platform
  377. from hermes_state_local import hand_off_conversations, moved_receipts
  378. from hermes_state_runtime import _epoch
  379. def write(conn):
  380. _epoch(conn, epoch)
  381. moved = {owner: (target, source) for owner, (target, source) in moved_receipts(conn).items()
  382. if _messaging_platform(source) is not None}
  383. return hand_off_conversations(conn, moved)
  384. for owner, (target, source) in db._execute_write(write).items():
  385. logging.getLogger(__name__).info(
  386. 'Local session %s was handed off to %s (row %s); released its local receipt', owner, source, target)
  387. def local_receipt_ids(authority):
  388. from gateway.session_viewer_close import CLOSE_PREFIX
  389. start = len(POLICY_PREFIX) + 1
  390. with authority.db._read_ctx() as conn:
  391. return [row[0][len(POLICY_PREFIX):] for row in conn.execute(
  392. _BOOT_RECEIPTS_SQL, (POLICY_PREFIX + '%', start, CLOSE_PREFIX, start))]
  393. def recover_local_sessions(authority, *, schedule=False):
  394. orphans = []
  395. for sid in local_receipt_ids(authority):
  396. try:
  397. ref = restore_local_session(authority, sid)
  398. pending = list_session_admissions(authority.db, session_id=sid)
  399. if any(row['status'] == 'unknown' for row in pending):
  400. continue # the drain pauses on it until its user's Discard
  401. if schedule and pending:
  402. authority._schedule(ref)
  403. orphans.append(sid)
  404. except (_ReceiptRefused, _CatchUpPending):
  405. continue # logged: the check that refused it, or the catch-up handed to a worker
  406. except RuntimeStoreError as exc:
  407. logging.getLogger(__name__).warning('Local session %s paused: %s', sid, exc.reason)
  408. if schedule:
  409. # A last-viewer close the previous owner stopped before completing (#134889 LC-6b).
  410. from gateway.session_viewer_close import rearm_persisted_closes, settle_viewer_close
  411. from gateway.session_runtime_workers import track_mutation
  412. for sid in rearm_persisted_closes(authority):
  413. authority._schedule(SessionRef(authority.profile_id, sid))
  414. if any(row['status'] == 'unknown' for row in list_session_admissions(authority.db, session_id=sid)):
  415. # A hard-killed owner's turn: the drain pauses on it, so the close resolves it.
  416. track_mutation(authority, settle_viewer_close(authority, sid))
  417. from gateway.session_viewer_close import arm_startup_orphan_closes
  418. arm_startup_orphan_closes(authority, orphans)