run_runtime.py 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789
  1. """Session-runtime lifecycle owned by the ordinary gateway bootstrap.
  2. One SessionAuthority per reserved profile home. The launch home always has one; under
  3. ``gateway.multiplex_profiles`` every served secondary gets its own, built under that
  4. profile's runtime scope against that profile's ``state.db``. ``runner.session_authority``
  5. stays the launch profile's authority so single-profile behaviour is byte-identical.
  6. The served set is not frozen at boot: ``serve_profile_runtime`` / ``unserve_profile_runtime``
  7. grow and shrink it for the hot-serve reconcile, and a secondary whose store cannot be opened
  8. is parked (logged, left out of ``served_profiles``) instead of aborting every other profile.
  9. """
  10. from __future__ import annotations
  11. import asyncio
  12. from contextlib import suppress
  13. import logging
  14. from pathlib import Path
  15. import threading
  16. import uuid
  17. logger = logging.getLogger(__name__)
  18. def reserved_profile_homes(runner):
  19. """``(name, home)`` pairs this process reserved: launch home first, then served secondaries."""
  20. from hermes_constants import get_hermes_home
  21. launch = get_hermes_home().resolve()
  22. homes = [(getattr(runner, '_primary_profile_name', None) or 'default', launch)]
  23. if getattr(runner.config, 'multiplex_profiles', False):
  24. reserved = getattr(runner.config, '_runtime_profile_homes', None) or ()
  25. for name, home in reserved:
  26. canonical = Path(home).resolve()
  27. if canonical != launch and canonical not in {h for _, h in homes}:
  28. homes.append((name, canonical))
  29. return homes
  30. async def _build_profile_authority(runner, name, home, *, register):
  31. """One profile's authority against its own ``state.db``; raises when that store is unusable."""
  32. from gateway.run import _profile_runtime_scope
  33. from gateway.session_authority import initialize_session_authority
  34. registry = runner.session_authorities
  35. instance_id = runner.session_runtime_descriptor['instance_id']
  36. # Each home's store resolves through the runner's scope-following handle cache, exactly
  37. # the handle every later scoped read of that profile uses (one writer per state.db).
  38. db = await asyncio.to_thread(open_profile_store, runner, home)
  39. with _profile_runtime_scope(home, hydrate_secrets=False):
  40. if db is None:
  41. raise RuntimeError(_store_open_failure(runner, home))
  42. if (await asyncio.to_thread(Path(db.db_path).resolve)).parent != home:
  43. raise RuntimeError(f'session authority database does not belong to the reserved profile {home}')
  44. registry.add(home, None, name=name)
  45. try:
  46. authority = await initialize_session_authority(
  47. runner, profile_id=str(home), instance_id=instance_id, db=db, register=register)
  48. except BaseException:
  49. registry.remove(home)
  50. raise
  51. registry.replace(home, authority)
  52. return authority
  53. def open_profile_store(runner, home):
  54. """*home*'s SessionDB through the runner's scoped handle cache, opened in the CALLING thread
  55. under that profile's scope (bound here: a worker thread inherits no scope it was not given).
  56. Callers run it in a worker: a first open runs the schema / FTS setup, about a second per store
  57. on a loaded host, and run on the loop it froze every connected client for the whole
  58. multi-profile boot. Later opens of the same path are cache hits."""
  59. from gateway.run import _profile_runtime_scope
  60. with _profile_runtime_scope(home, hydrate_secrets=False):
  61. db = runner._session_db
  62. return getattr(db, '_db', db)
  63. async def _prewarm_profile_stores(runner, homes):
  64. """Open every secondary's store concurrently off the loop before the authorities are built in
  65. order. A failed open is not reported here: the cache records it and the authority build that
  66. follows parks that profile with the recorded reason."""
  67. await asyncio.gather(*(asyncio.to_thread(open_profile_store, runner, home) for _name, home in homes),
  68. return_exceptions=True)
  69. def pin_launch_store():
  70. """Open the launch profile's state.db in the calling (worker) thread and hold one registry
  71. reference, so the synchronous ``SessionStore`` open inside ``GatewayRunner()`` on the loop is a
  72. registry hit instead of the schema setup. None when the open fails: the store's own open then
  73. reports and backs off exactly as before."""
  74. from hermes_state_registry import acquire
  75. try:
  76. return acquire()
  77. except Exception:
  78. logger.debug('launch state.db prewarm failed; the store open reports it', exc_info=True)
  79. return None
  80. def unpin_launch_store(db):
  81. if db is not None:
  82. from hermes_state_registry import release
  83. release(db)
  84. def _store_open_failure(runner, home):
  85. """Why *home*'s ``state.db`` has no handle: the scoped handle caches answer None for a failed open
  86. (and while they back off), so the park reason named an ownership mismatch over the real cause, a
  87. legacy store that crashed its migration on every open (#141467). The reason is the exception the
  88. failed open recorded; reopening here to learn it could create or migrate the store, and would skip
  89. the caches' backoff."""
  90. path = home / 'state.db'
  91. caches = (getattr(getattr(runner, 'session_store', None), '_db_handle_cache', None),
  92. getattr(runner, '_session_db_handle_cache', None))
  93. error = next((e for e in (c.last_error(path) for c in caches if c is not None) if e), '')
  94. if error:
  95. return f'{path} did not open: {error}'
  96. return f'{path} is unavailable (its last open failed; retry pending)'
  97. def _park_reserved_profile(runner, name, home, exc):
  98. """A secondary whose store is unusable is parked: logged, unreserved, left out of
  99. ``served_profiles`` and named under ``parked_profiles`` in the runtime descriptor. Boot
  100. continues for every other profile (a runtime hot-add parks the same way)."""
  101. logger.error("[MULTIPLEX] Profile '%s' not served: its session store is unusable (%s): %s",
  102. name, home, exc)
  103. release_profile_home(runner, home)
  104. park_profile(runner, name, f'session store unusable: {exc}')
  105. def park_profile(runner, name, reason):
  106. """Publish *name* under ``parked_profiles`` (name -> reason) in the runtime descriptor and the
  107. root's ``gateway_state.json``: a client of that profile then gets a terminal ``profile_parked``
  108. verdict from ``ensure`` instead of waiting its whole deadline for a service that never comes."""
  109. parked = parked_profile_map(runner)
  110. parked[name] = str(reason)[:500]
  111. _record_parked_profiles(parked)
  112. def unpark_profile(runner, name):
  113. parked = parked_profile_map(runner)
  114. if parked.pop(name, None) is not None:
  115. _record_parked_profiles(parked)
  116. def profile_parked(runner, name):
  117. """True when *name* is parked: not served now, but a rescan serves it again, so its ingress is
  118. temporarily unavailable (retry it), never an invalid route (refuse it for good)."""
  119. descriptor = getattr(runner, 'session_runtime_descriptor', None) or {}
  120. return bool(name) and name in (descriptor.get('parked_profiles') or {})
  121. def refuse_parked_scope(runner):
  122. """An API request under a parked profile's ``/p/<profile>/`` prefix finds no authority: refuse it
  123. retryably (``runtime_draining``, a 503 with ``Retry-After``), not with the final
  124. ``profile_mismatch`` of a profile this gateway does not serve."""
  125. from gateway.session_authorities import served_profile_name
  126. from hermes_constants import get_hermes_home
  127. name = served_profile_name(get_hermes_home())
  128. if profile_parked(runner, name):
  129. from gateway.parked_route_refusal import ParkedProfileRefusal
  130. raise ParkedProfileRefusal(name)
  131. def parked_profile_map(runner):
  132. descriptor = getattr(runner, 'session_runtime_descriptor', None)
  133. if descriptor is None:
  134. descriptor = runner.session_runtime_descriptor = {}
  135. return descriptor.setdefault('parked_profiles', {})
  136. def _record_parked_profiles(parked):
  137. try:
  138. from gateway.status import write_runtime_status
  139. write_runtime_status(parked_profiles=dict(parked))
  140. except Exception:
  141. logger.debug('could not record parked_profiles', exc_info=True)
  142. async def initialize_gateway_runtime(runner):
  143. from gateway.runtime_bootstrap import TicketStore
  144. from gateway.runtime_ownership import process_ownership
  145. from gateway.session_authorities import SessionAuthorities
  146. homes = reserved_profile_homes(runner)
  147. for _name, home in homes:
  148. if not process_ownership.owns(home):
  149. raise RuntimeError(f'session authority requires reserved profile ownership: {home}')
  150. instance_id = uuid.uuid4().hex
  151. descriptor = {
  152. 'instance_id': instance_id, 'runtime_protocol': 1,
  153. 'state': 'starting', 'capabilities': [],
  154. 'served_profiles': [],
  155. }
  156. runner.session_runtime_descriptor = descriptor
  157. registry = SessionAuthorities(homes[0][1], multiplexed=getattr(runner.config, 'multiplex_profiles', False))
  158. runner.session_authorities = registry
  159. await _prewarm_profile_stores(runner, homes[1:])
  160. for index, (name, home) in enumerate(homes):
  161. try:
  162. await _build_profile_authority(runner, name, home, register=index == 0)
  163. except Exception as exc:
  164. if index == 0:
  165. raise # the launch profile's store is the process's own; nothing to park it behind
  166. _park_reserved_profile(runner, name, home, exc)
  167. descriptor['authority_epoch'] = registry.launch.epoch
  168. descriptor['served_profiles'] = registry.served_profiles()
  169. # Publish this boot's verdict (an empty map clears a previous run's parked set).
  170. _record_parked_profiles(parked_profile_map(runner))
  171. runner.session_ticket_store = TicketStore(instance_id, registry.profile_ids())
  172. release_unknown_turn_markers(runner)
  173. def release_unknown_turn_markers(runner):
  174. """Before the unclean-start pass reads crash-left turn markers: a marked turn whose canonical
  175. admission recovered ``unknown`` (the owner died before its terminal commit) owes no reply, so
  176. its marker is cleared rather than ledgered as a delivery the FIFO never committed. A terminal
  177. admission keeps its marker: its persisted reply is ledgered and sent once, never re-run."""
  178. from gateway.session_authorities import admission_owner, owner_scope
  179. store = getattr(runner, 'session_store', None)
  180. if store is None:
  181. return
  182. for authority in _authorities(runner):
  183. with owner_scope(authority):
  184. unknown = {row['target_session_id'] for row in authority.db._read_all(
  185. "SELECT DISTINCT target_session_id FROM session_admissions WHERE status='unknown'")}
  186. if not unknown:
  187. continue
  188. for entry in store.list_sessions():
  189. if entry.active_turn_token and admission_owner(authority, entry.session_id) in unknown:
  190. store.clear_turn_active(entry.session_key, entry.active_turn_token)
  191. def _publish_served_set(runner):
  192. registry = runner.session_authorities
  193. runner.session_runtime_descriptor['served_profiles'] = registry.served_profiles()
  194. runner.session_ticket_store.profile_ids = registry.profile_ids()
  195. def reserve_profile_home(runner, name, home, cancel=None):
  196. """Grow the process reservation by one profile (hot-serve): take its ``gateway.lock`` and add it
  197. to the frozen boot set, so every reader of the reservation — and the next restart's
  198. all-or-nothing reserve — sees it. Waits out a transient holder within boot's window (blocking:
  199. the event loop calls :func:`reserve_profile_home_in_worker`). ``OwnershipConflict`` when another
  200. gateway owns the home; *cancel* set ends the wait with nothing reserved."""
  201. from gateway.run_bootstrap import _browse_reconcile_reserve_wait_s
  202. from gateway.runtime_ownership import process_ownership, reserve_waiting_out_transient_holders
  203. home = Path(home).resolve()
  204. reserve_waiting_out_transient_holders(process_ownership, [home], wait_s=_browse_reconcile_reserve_wait_s(),
  205. cancel=cancel)
  206. reserved = getattr(runner.config, '_runtime_profile_homes', None)
  207. if reserved is not None and all(Path(h).resolve() != home for _n, h in reserved):
  208. runner.config._runtime_profile_homes = (*reserved, (name, home))
  209. async def reserve_profile_home_in_worker(runner, name, home):
  210. """:func:`reserve_profile_home` off the event loop, settled on cancellation. A cancelled bare
  211. ``to_thread`` await leaves its worker retrying, free to take the flock after the canceller's
  212. ownership cleanup with no coroutine left to serve or release it. So a cancel stops the worker,
  213. waits for it to finish (each further cancel is absorbed until then), undoes a reservation it
  214. completed first, and only then propagates."""
  215. from gateway.runtime_ownership import process_ownership
  216. home = await asyncio.to_thread(Path(home).resolve)
  217. owned_before = process_ownership.owns(home)
  218. cancel = threading.Event()
  219. worker = asyncio.ensure_future(asyncio.to_thread(reserve_profile_home, runner, name, home, cancel))
  220. try:
  221. await asyncio.shield(worker)
  222. except asyncio.CancelledError:
  223. cancel.set()
  224. while not worker.done():
  225. with suppress(asyncio.CancelledError):
  226. await asyncio.wait({worker})
  227. if not worker.cancelled() and worker.exception() is None and not owned_before:
  228. release_profile_home(runner, home)
  229. raise
  230. def release_profile_home(runner, home):
  231. """Shrink the reservation by one profile (deleted, or parked because it cannot be served)."""
  232. from gateway.runtime_ownership import process_ownership
  233. home = Path(home).resolve()
  234. registry = getattr(runner, 'session_authorities', None)
  235. if registry is not None and registry.for_home(home) is not None:
  236. raise RuntimeError('Profile authority must retire before ownership is released')
  237. _withdraw_profile_home(runner, home)
  238. process_ownership.release(home)
  239. def _withdraw_profile_home(runner, home):
  240. """Drop *home* from the served set every startup consumer reads (adapters, served status, cron
  241. ticks, ingress, spool recovery) without releasing its ``gateway.lock``."""
  242. reserved = getattr(runner.config, '_runtime_profile_homes', None)
  243. if reserved is not None:
  244. runner.config._runtime_profile_homes = tuple(
  245. entry for entry in reserved if Path(entry[1]).resolve() != home)
  246. def _authority_tasks(authority):
  247. """Snapshot every async task family owned directly by one session authority."""
  248. tasks = [live.task for live in authority.sessions.values() if live.task is not None]
  249. tasks.extend(getattr(authority, '_bot_receipt_tasks', ()))
  250. from gateway.session_runtime_workers import mutation_tasks
  251. tasks.extend(mutation_tasks(authority))
  252. return list(dict.fromkeys(tasks))
  253. # How long a Stopped turn gets to settle durably before shutdown refuses to release ownership: a managed worker
  254. # acknowledges Stop within STOP_ACK_SECONDS (30) or is terminated, then closed (<=10).
  255. TURN_SETTLE_SECONDS = 45.0
  256. # What a stop keeps of its watchdog leash for the phases after the settle (adapter teardown, executor
  257. # quiesce, SessionDB close, exit-state writes): the settle never spends it.
  258. STOP_TAIL_RESERVE_SECONDS = 10.0
  259. def managed_turn_count(runner):
  260. """Unmapped claims, managed workers and owner mutations across the served authorities."""
  261. from gateway.session_runtime_workers import uncounted_runtime_work
  262. return uncounted_runtime_work(runner)
  263. def stop_authority_turns(authority, *, in_process=False, reason=None):
  264. """Cooperative Stop for every turn *authority* is executing: the control a user's Stop sends to a
  265. managed worker (acknowledged, or terminated within STOP_ACK_SECONDS, and the admission settles
  266. through its own fenced path). ``in_process`` also hard-interrupts the running agent of each
  267. executing in-process turn, or latches the generation ``adopt_agent`` consumes before it exists.
  268. ``reason`` (a gateway restart/shutdown) is recorded on each worker it signals, so the turn settles
  269. with that cause; a worker already stopped (the user's Stop came first) keeps none. A worker
  270. still being spawned gets the same Stop and cause through latches (``_stop_spawning_managed``)."""
  271. from hermes_state_runtime import RuntimeStoreError
  272. signalled = 0
  273. workers = getattr(authority, '_managed_workers', {})
  274. for worker in list(workers.values()):
  275. if worker.closed.is_set() or (getattr(worker, 'stop', None) is not None and worker.stop.is_set()):
  276. continue
  277. worker.stop_cause = reason
  278. try:
  279. worker.control({'type': 'stop'})
  280. except RuntimeStoreError as exc:
  281. # A full control queue: control() latched worker.stop first, and the owner's read
  282. # loop escalates on that latch alone (terminate after STOP_ACK_SECONDS).
  283. logger.debug('managed Stop not queued (%s); the latch escalates it', exc.reason)
  284. signalled += 1
  285. signalled += _stop_spawning_managed(authority, workers, reason)
  286. if not in_process:
  287. return signalled
  288. from agent.interrupt_compat import request_hard_interrupt
  289. from gateway.run import _AGENT_PENDING_SENTINEL
  290. running = getattr(authority.runner, '_running_agents', {})
  291. for sid, live in list(authority.sessions.items()):
  292. generation = live.event_stream.execution.get('execution_generation')
  293. if generation is None or sid in workers or sid in getattr(authority, '_managed_spawning', {}):
  294. continue
  295. adopted_generation, agent = authority.adopted.get(sid, (None, None))
  296. if adopted_generation != generation:
  297. agent = running.get(live.route)
  298. if agent is None or agent is _AGENT_PENDING_SENTINEL:
  299. authority.pending_stops[sid] = generation
  300. else:
  301. request_hard_interrupt(agent, 'Profile stopping', tool_reason='gateway shutdown')
  302. signalled += 1
  303. return signalled
  304. def _stop_spawning_managed(authority, workers, reason):
  305. """A managed turn whose worker is still being spawned has nothing to signal yet: latch the
  306. Stop for its generation (``adopt_agent`` delivers it to the worker, which is never
  307. bootstrapped) and, for a gateway restart/shutdown, the cause its pre-bootstrap interrupted
  308. result carries (#134889 GW-22). A user's Stop latched first keeps no cause."""
  309. stopped = 0
  310. for sid, generation in list(getattr(authority, '_managed_spawning', {}).items()):
  311. if sid in workers:
  312. continue
  313. stops = authority.pending_stops
  314. if reason and stops.get(sid) != generation:
  315. causes = getattr(authority, 'pending_stop_causes', None)
  316. if causes is None:
  317. causes = authority.pending_stop_causes = {}
  318. causes[sid] = (generation, reason)
  319. stops[sid] = generation
  320. stopped += 1
  321. return stopped
  322. def stop_managed_turns(runner, reason=None):
  323. """``stop_authority_turns`` across every served authority (whole-runtime shutdown)."""
  324. return sum(stop_authority_turns(authority, reason=reason) for authority in _authorities(runner))
  325. def interrupt_side_agents(runner, reason):
  326. """Whole-runtime shutdown: interrupt every counted side agent (``session_runtime_workers``)."""
  327. from gateway.session_runtime_workers import interrupt_side_agents as interrupt
  328. return interrupt(runner, reason)
  329. async def _retire_profile_authority(authority, timeout=None):
  330. """Stop profile-local services and turns before its ownership is released.
  331. Claims are refused from here on; every executing turn gets a cooperative Stop, and its session
  332. task is joined (an in-process turn's task ends only after its executor thread returned and the
  333. admission settled). The hosted-room stop runs alongside that join under the same deadline: it
  334. never interrupts accepted turns and its room threads wait for the member turns they observe, so
  335. those turns must be signalled first or the room stop cannot finish. False when a turn or room
  336. thread misses *timeout* (default TURN_SETTLE_SECONDS): it may still call tools and write
  337. history, so the caller must keep the profile's reservation and store handles."""
  338. from gateway.session_cron import unbind_owner
  339. from gateway.session_runtime_workers import join_authority_work, stop_authority_work
  340. timeout = TURN_SETTLE_SECONDS if timeout is None else timeout
  341. authority.retiring = True
  342. if getattr(authority.runner, 'session_ticket_store', None) is not None:
  343. _publish_served_set(authority.runner) # discovery and tickets stop naming it now
  344. stop_authority_work(authority)
  345. from gateway.session_llm_oneshot import stop as stop_oneshots
  346. stop_oneshots(authority)
  347. service = getattr(authority, 'hosted_room_service', None)
  348. rooms = None if service is None else asyncio.ensure_future(asyncio.to_thread(service.stop, timeout=timeout))
  349. try:
  350. await join_authority_work(authority, timeout)
  351. workers_stopped = True
  352. except TimeoutError:
  353. logger.error('Profile %s workers did not stop; ownership retained', authority.profile_id)
  354. workers_stopped = False
  355. rooms_stopped = rooms is None or await rooms is not False
  356. if not rooms_stopped:
  357. logger.error('Profile %s hosted rooms did not stop; ownership retained', authority.profile_id)
  358. if not (workers_stopped and rooms_stopped):
  359. return False
  360. # Idle drains and receipt watchers wait on admissions this profile will no longer run.
  361. rest = _authority_tasks(authority)
  362. for task in rest:
  363. task.cancel()
  364. if rest:
  365. await asyncio.gather(*rest, return_exceptions=True)
  366. unbind_owner(authority)
  367. # Ownership is released next; until a successor bumps the epoch, A's epoch still fences its
  368. # writes as current, so a stale connection to A must find it retired instead.
  369. authority.retired = True
  370. # No drain is left to answer an observer (a session_busy pause keeps API/webhook/Bot ones
  371. # waiting): release them all. Their rows stay queued for the next owner, so nothing is resent.
  372. from hermes_state_runtime import RuntimeStoreError
  373. authority.native_waiters.clear()
  374. for waiter in authority.waiters.values():
  375. if not waiter.done():
  376. waiter.set_exception(RuntimeStoreError('runtime_draining'))
  377. authority.waiters.clear()
  378. return True
  379. class ProfileOwnershipRetained(RuntimeError):
  380. """A retired attempt's writer is still live: the profile keeps its reservation and authority."""
  381. async def serve_profile_runtime(runner, name, home):
  382. """Hot-serve one reserved profile's runtime: build its authority, recover its durable state and
  383. publish it in the descriptor/ticket store — the steps boot performs per secondary. Raises when
  384. the profile's store is unusable; the caller parks it (and releases the reservation). Its
  385. queued native rows are recovered once its adapters connect (``recover_served_native_rows``)."""
  386. from gateway.session_authorities import owner_scope
  387. from gateway.session_hosted_service import _ensure_hosted_service, start_ready_hosted_services
  388. from gateway.session_local_recovery import recover_local_sessions
  389. from gateway.platforms.webhook_ingress import recover_webhook_finalizations
  390. home = await asyncio.to_thread(Path(home).resolve)
  391. registry = runner.session_authorities
  392. existing = registry.for_home(home)
  393. if existing is not None and getattr(existing, 'retiring', False):
  394. # An earlier failed attempt kept this authority because its writer outlived Stop. It refuses
  395. # every admission, so serving it would start adapters on a dead profile: finish its
  396. # retirement without waiting, or refuse (the caller parks and keeps the reservation).
  397. if not await _retire_profile_authority(existing, timeout=0):
  398. raise ProfileOwnershipRetained(f'an earlier serve of {home} is still stopping')
  399. registry.remove(home)
  400. elif existing is not None:
  401. return existing
  402. authority = await _build_profile_authority(runner, name, home, register=False)
  403. try:
  404. with owner_scope(authority):
  405. _drop_retired_local_adapter(authority)
  406. await recover_bot_mailbox(authority)
  407. recover_local_sessions(authority, schedule=True)
  408. await recover_webhook_finalizations(authority)
  409. _recover_published_api_turns(runner, authority)
  410. if getattr(runner, 'session_control_server', None) is not None:
  411. await _ensure_hosted_service(runner, authority)
  412. except BaseException:
  413. if await _retire_profile_authority(authority):
  414. registry.remove(home)
  415. raise
  416. _publish_served_set(runner)
  417. start_ready_hosted_services(runner)
  418. return authority
  419. def _drop_retired_local_adapter(authority):
  420. """A LOCAL adapter still bound to this profile's previous authority, once that authority fully
  421. retired, gives way to *authority*'s own: restore and create otherwise refuse every local session
  422. of the served profile as ``runtime_draining``. A retained writer never gets here (the serve is
  423. refused while the previous authority is still retiring)."""
  424. from gateway.config import Platform
  425. from gateway.session_local_recovery import local_adapter_map
  426. adapters = local_adapter_map(authority)
  427. previous = getattr(adapters.get(Platform.LOCAL), 'authority', None)
  428. if previous is not None and previous is not authority and getattr(previous, 'retired', False):
  429. del adapters[Platform.LOCAL]
  430. def _recover_published_api_turns(runner, authority):
  431. """Boot recovers queued API turns when the launch API adapter is published
  432. (``recover_api_turns``); a profile served after that runs the same pass on its own store."""
  433. from gateway.config import Platform
  434. api = (getattr(runner, 'adapters', None) or {}).get(Platform.API_SERVER)
  435. if api is not None:
  436. from gateway.session_api_turn import recover_authority_api_turns
  437. recover_authority_api_turns(api, authority)
  438. async def recover_served_native_rows(runner):
  439. """Boot's native-row recovery once hot-served profiles' adapters connected (boot runs it after
  440. every adapter connects), in every serving ledger: a ``bot_profile`` route runs a bot's messages
  441. on another profile, so the new bots can carry rows queued in any ledger. A session already
  442. draining keeps its one drain. A store that cannot answer leaves its rows queued for the next
  443. start and never stops the other ledgers or the reconcile."""
  444. if getattr(runner, 'session_authorities', None) is None:
  445. return
  446. from gateway.session_authorities import serving_authorities
  447. for authority in serving_authorities(runner):
  448. try:
  449. await recover_profile_native_sessions(runner, authority)
  450. except Exception: # health: allow BLE001 -- logged; the rows stay queued for the next start
  451. logger.error('Native session recovery failed for %s', authority.profile_id, exc_info=True)
  452. async def unserve_profile_runtime(runner, home):
  453. """Retire one profile's authority (deleted while running) and shrink the published set.
  454. False when one of its turns outlived the Stop deadline (keep the reservation); True otherwise,
  455. including for a profile this process never served."""
  456. home = await asyncio.to_thread(Path(home).resolve)
  457. registry = runner.session_authorities
  458. authority = registry.for_home(home)
  459. if authority is None:
  460. return True
  461. retired = await _retire_profile_authority(authority)
  462. if not retired:
  463. return False
  464. registry.remove(home)
  465. _publish_served_set(runner)
  466. return retired
  467. def _authorities(runner):
  468. from gateway.session_authorities import all_authorities
  469. return all_authorities(runner)
  470. async def recover_bot_mailbox(authority):
  471. """Owner-startup Bot Chat recovery, skipped when this token cannot use the mailbox dir.
  472. An elevated install/update can leave ``runtime/bot_live_delivery`` owned by Administrators with
  473. no entry for the user (#139485, #138872): the gateway cannot even open its ``.lock``. Bot Chat
  474. deliveries pause (``_locked`` already logged the dir, its owner and the remedy); every other
  475. client keeps working instead of the whole gateway refusing to start. Any other failure still
  476. aborts startup."""
  477. from gateway.session_bot import recover_bot_deliveries
  478. from tools.bot_live_delivery import MailboxUnusable
  479. try:
  480. await recover_bot_deliveries(authority)
  481. except MailboxUnusable as exc:
  482. logger.warning('Bot Chat recovery skipped for %s: %s', authority.db.db_path, exc)
  483. async def start_gateway_runtime_api(runner):
  484. from gateway import run_api
  485. from gateway.webapp_listener import (
  486. STARTING, BrowserCredentialError, browser_session_token, dial_origin, listener_plan, register_control_verbs,
  487. )
  488. from hermes_constants import get_hermes_home
  489. home = get_hermes_home()
  490. # The bootstrap registered these before its control socket answered; a runner started without it
  491. # (an embedded gateway) gets them here, before the listener's slow start.
  492. register_control_verbs(runner, home)
  493. host, port, webapp, unusable = listener_plan(home)
  494. # A webapp: block that asks for the Webapp but cannot be used is reported with its reason.
  495. runner.webapp_status = ({"state": "unavailable", "url": None, "reason": unusable} if unusable
  496. else {"state": "off", "url": None, "reason": None} if webapp is None
  497. else dict(STARTING))
  498. try:
  499. # Without the Webapp the listener is main's: the same call, a private ephemeral port.
  500. runner.session_api = await (run_api.start_gateway_api(runner, host=host, port=port, webapp=webapp)
  501. if webapp is not None else run_api.start_gateway_api(runner))
  502. except (Exception, SystemExit) as exc:
  503. if webapp is None:
  504. raise
  505. # The Webapp's listener could not start (port taken, address unavailable, its browser credential
  506. # not storable in the home, or the dashboard's auth gate exiting on a wildcard bind with no
  507. # sign-in provider): native clients still need this gateway, so it keeps main's private
  508. # listener and reports why the Webapp is not served.
  509. if isinstance(exc, SystemExit):
  510. reason = str(exc.code).strip() # the gate's own message names the fix
  511. elif isinstance(exc, BrowserCredentialError):
  512. reason = str(exc) # names the file and why it cannot be stored
  513. elif isinstance(exc, OSError) and exc.filename: # the browser credential's file, not the port
  514. reason = f"could not use {exc.filename} ({exc.strerror or exc})"
  515. else:
  516. cause = (exc.strerror or exc) if isinstance(exc, OSError) else exc
  517. reason = f"could not listen on {host}:{port} ({cause})"
  518. logger.error("Webapp unavailable: %s", reason)
  519. runner.webapp_status = {"state": "unavailable", "url": None, "reason": reason}
  520. # Open tabs keep sending their credential to whatever answers on the Webapp's port, and that
  521. # is not this gateway now. Replace it, so one captured there never opens this gateway; the
  522. # tabs sign in again once the Webapp is back.
  523. try:
  524. browser_session_token(home, rotate=True)
  525. except OSError as write_error:
  526. logger.warning("Could not replace the Webapp's browser credential: %s", write_error)
  527. runner.session_api = await run_api.start_gateway_api(runner)
  528. else:
  529. if webapp is not None:
  530. runner.webapp_status = {"state": "serving", "url": dial_origin(host, port), "reason": None}
  531. runner.session_runtime_descriptor['api_origin'] = runner.session_api.api_origin
  532. for authority in _authorities(runner):
  533. await _recover_profile_bot_mailbox(runner, authority)
  534. async def _recover_profile_bot_mailbox(runner, authority):
  535. """Startup Bot Chat recovery of one served profile. A secondary whose durable leftovers cannot
  536. be recovered is parked like one whose store cannot be opened (retired, unreserved, named under
  537. ``parked_profiles``): under multiplex it would otherwise take every other profile down with it,
  538. on every restart (#140018). The launch profile's failure still aborts the start."""
  539. from gateway.session_authorities import owner_scope
  540. try:
  541. with owner_scope(authority):
  542. await recover_bot_mailbox(authority)
  543. except Exception as exc:
  544. registry = getattr(runner, 'session_authorities', None)
  545. name = None if registry is None else registry.profile_name(authority)
  546. if name is None:
  547. raise # the launch profile's store is the process's own; nothing to park it behind
  548. logger.error("[MULTIPLEX] Profile '%s' not served: startup recovery failed: %s", name, exc,
  549. exc_info=True)
  550. home = await asyncio.to_thread(Path(authority.profile_id).resolve)
  551. if not await _retire_profile_authority(authority):
  552. # A writer outlived Stop: the lock and the retiring authority stay for shutdown to
  553. # settle, but the profile leaves the served set, so later boot never starts it.
  554. _withdraw_profile_home(runner, home)
  555. park_profile(runner, name, f'startup recovery failed; ownership retained (a control-socket rescan, '
  556. f'`hermes -p {name} gateway run`, serves it once its writer stops): {exc}')
  557. return
  558. registry.remove(home)
  559. _publish_served_set(runner)
  560. release_profile_home(runner, home)
  561. park_profile(runner, name, f'startup recovery failed (a control-socket rescan, '
  562. f'`hermes -p {name} gateway run`, serves it again): {exc}')
  563. async def recover_gateway_native_sessions(runner):
  564. """Recover against the published routing index and currently connected adapters.
  565. Stored envelopes are input, not authority to create a route or reconnect a
  566. transport. The authority preflights every queued sender before any claim.
  567. """
  568. from gateway.session_authorities import owner_scope, serving_authorities
  569. authorities = serving_authorities(runner)
  570. if not authorities:
  571. return {}
  572. from gateway.session_hosted_service import ensure_hosted_service
  573. await ensure_hosted_service(runner)
  574. from gateway.session_local_recovery import recover_local_sessions
  575. from gateway.platforms.webhook_ingress import recover_webhook_finalizations
  576. results = {}
  577. for authority in authorities:
  578. with owner_scope(authority):
  579. recover_local_sessions(authority, schedule=True)
  580. await recover_webhook_finalizations(authority)
  581. results.update(await recover_profile_native_sessions(runner, authority))
  582. return results
  583. async def recover_profile_native_sessions(runner, authority):
  584. """Bind one authority's queued native rows to the adapters connected now."""
  585. from gateway.session_authorities import owner_scope
  586. with owner_scope(authority):
  587. pending = {row['target_session_id'] for row in authority.db._read_all(
  588. "SELECT DISTINCT target_session_id FROM session_admissions WHERE status IN ('queued','unknown')")}
  589. bindings = [(owner, entry.origin, runner._adapter_for_source(entry.origin))
  590. for entry in runner.session_store.list_sessions() if entry.origin is not None
  591. for owner in [authority.logical_owner(entry.session_id)] if owner in pending]
  592. outcome = await authority.recover_native_sessions(bindings)
  593. from gateway.session_detached import recover_detached_sessions
  594. outcome.update(dict.fromkeys(recover_detached_sessions(authority, pending - set(outcome)), 'detached'))
  595. for sid, verdict in outcome.items():
  596. logger.info('Native session startup recovery %s (%s): %s', sid, authority.profile_id, verdict)
  597. return outcome
  598. def publish_gateway_runtime_ready(runner):
  599. descriptor = runner.session_runtime_descriptor
  600. if runner.session_api.task.done() or not runner._running or runner._draining:
  601. raise RuntimeError('gateway stopped before session API readiness')
  602. descriptor.update(state='ready', capabilities=[
  603. 'session-authority-v1', 'durable-admission-v1', 'event-replay-v1'])
  604. from gateway.session_hosted_service import start_ready_hosted_services
  605. start_ready_hosted_services(runner)
  606. async def wait_gateway_runtime(runner):
  607. """A vanished interactive listener is fatal, not a healthy headless runtime."""
  608. shutdown = asyncio.create_task(runner.wait_for_shutdown())
  609. listener = runner.session_api.task
  610. try:
  611. done, _ = await asyncio.wait({shutdown, listener}, return_when=asyncio.FIRST_COMPLETED)
  612. if listener in done and not runner._draining:
  613. runner.session_runtime_descriptor.update(state='failed', capabilities=[])
  614. error = None if listener.cancelled() else listener.exception()
  615. raise RuntimeError('gateway session API stopped unexpectedly') from error
  616. await shutdown
  617. finally:
  618. if not shutdown.done():
  619. shutdown.cancel()
  620. await asyncio.gather(shutdown, return_exceptions=True)
  621. async def drain_gateway_runtime(runner):
  622. """Withdraw admission before any await; close sockets before DB teardown."""
  623. descriptor = getattr(runner, 'session_runtime_descriptor', None)
  624. if descriptor is None:
  625. return
  626. runner._draining = True
  627. descriptor.update(state='draining', capabilities=[])
  628. from gateway.drain_control import RESTART_WAIT_REASON
  629. if descriptor.get('drain_reason') == RESTART_WAIT_REASON:
  630. del descriptor['drain_reason'] # stopping now: no client attaches to answer anymore
  631. from gateway.session_hosted_service import stop_hosted_service
  632. await stop_hosted_service(runner)
  633. # Withdraw the public ingress callback without disconnecting egress needed
  634. # by already admitted work. Base adapters refuse before stamping acceptance.
  635. for adapter in runner.adapters.values():
  636. adapter.set_message_handler(None)
  637. for adapters in getattr(runner, '_profile_adapters', {}).values():
  638. for adapter in adapters.values():
  639. adapter.set_message_handler(None)
  640. async def settle_gateway_runtime(runner, budget=None, reason=None):
  641. """Keep authority tasks alive until their last durable settlement write, within a bound.
  642. A managed turn still running here gets its Stop first (its worker acknowledges or is
  643. terminated, and the admission settles interrupted or ``unknown``); a task that still misses
  644. TURN_SETTLE_SECONDS (clamped to ``budget``, what the stop's watchdog leash has left, minus
  645. STOP_TAIL_RESERVE_SECONDS) is logged and left running for recovery. The stop sequence continues:
  646. the executor quiesce skips the DB close while a writer is live, and process exit releases
  647. the ownership lock, so raising here would only skip teardown and disarm the watchdog."""
  648. authorities = _authorities(runner)
  649. stop_managed_turns(runner, reason)
  650. from gateway.session_llm_oneshot import stop as stop_oneshots
  651. for authority in authorities:
  652. stop_oneshots(authority) # an answer is no durable settlement: never waited for
  653. tasks = [task for authority in authorities for task in _authority_tasks(authority)]
  654. wait = TURN_SETTLE_SECONDS if budget is None else max(0.0, min(TURN_SETTLE_SECONDS,
  655. budget - STOP_TAIL_RESERVE_SECONDS))
  656. deadline = asyncio.get_running_loop().time() + wait
  657. if tasks:
  658. done, pending = await asyncio.wait(tasks, timeout=wait) if wait > 0 else (set(), set(tasks))
  659. for task in done:
  660. if not task.cancelled():
  661. task.exception()
  662. if pending:
  663. logger.warning('%d authority task(s) did not settle within %.0fs; left for recovery',
  664. len(pending), wait)
  665. from gateway.session_runtime_workers import join_authority_work
  666. for authority in authorities:
  667. try:
  668. await join_authority_work(authority, max(0.0, deadline - asyncio.get_running_loop().time()))
  669. except TimeoutError:
  670. logger.warning('Profile %s still has live writers at shutdown; its admissions stay for recovery',
  671. getattr(authority, 'profile_id', '?'))
  672. # Work settled above. ACP has no per-session destroy,
  673. # so the stop is the end of every ACP session nobody is viewing (#118216).
  674. from gateway.session_acp_lifecycle import end_idle_acp_sessions
  675. for authority in _authorities(runner):
  676. try:
  677. end_idle_acp_sessions(authority)
  678. except Exception:
  679. # Best-effort bookkeeping: a store that cannot answer must not abort the stop sequence.
  680. import logging
  681. logging.getLogger(__name__).warning('ACP sessions of %s not ended at shutdown',
  682. getattr(authority, 'profile_id', '?'), exc_info=True)
  683. from gateway.run_api import stop_gateway_api
  684. store = getattr(runner, 'session_ticket_store', None)
  685. if store is not None:
  686. store.revoke()
  687. handle = getattr(runner, 'session_api', None)
  688. if handle is not None:
  689. await stop_gateway_api(handle)