run_startup.py 114 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975
  1. """Startup sequence, resume/restore and handoff methods for GatewayRunner.
  2. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
  3. ``gateway.run`` internals are imported lazily inside method bodies (import cycle),
  4. so ``patch("gateway.run.X")`` keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import dataclasses
  9. import faulthandler
  10. import logging
  11. import os
  12. import signal
  13. import time
  14. from contextlib import nullcontext, suppress
  15. from contextvars import copy_context
  16. from pathlib import Path
  17. from agent.i18n import t
  18. from gateway.config import Platform
  19. from gateway.delivery import looks_like_telegram_private_chat_id
  20. from gateway.platforms.base import BasePlatformAdapter
  21. from gateway.platforms.event import MessageEvent, MessageType
  22. from gateway.session import SessionSource, build_session_key
  23. from gateway.restart import (
  24. DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, is_global_startup_conflict
  25. )
  26. from gateway.run_shutdown import _log_suppressed, _send_error
  27. from hermes_cli.observability.shared_metrics_busy import note_running_chat_owner
  28. from gateway.shutdown_watchdog import (
  29. DEFAULT_HEARTBEAT_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_INTERVAL_S,
  30. DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, loop_heartbeat_forever,
  31. )
  32. from typing import Any, Dict, Optional, Tuple
  33. # Log-record parity with the origin module.
  34. logger = logging.getLogger("gateway.run")
  35. def recover_left_core_in(home: Path, *, hydrate_secrets: bool = True) -> None:
  36. """Install the catalog plugin of every feature that left core (Home Assistant) *home* uses, in
  37. *home*'s runtime scope (its config, secrets and allow_lazy_installs); once per process per home.
  38. Blocking: call it off the event loop."""
  39. from gateway.run import _profile_runtime_scope
  40. from hermes_cli.left_core_migration import recover_at_startup
  41. with _profile_runtime_scope(Path(home), hydrate_secrets=hydrate_secrets):
  42. recover_at_startup(gateway_start=True)
  43. def recover_left_core_at_gateway_start() -> None:
  44. """Left-core migration for the homes whose platform config the runner loads first: the launch
  45. home and the default root (the multiplex primary; a named launcher is then served as a
  46. secondary). Every secondary runs :func:`recover_left_core_in` before its plugins are discovered,
  47. so a platform plugin it gets starts its adapter in the same gateway start."""
  48. from hermes_cli.left_core_migration import recover_at_startup
  49. from hermes_constants import get_default_hermes_root, get_hermes_home
  50. recover_at_startup(gateway_start=True)
  51. root = Path(get_default_hermes_root())
  52. if root.resolve() != Path(get_hermes_home()).resolve():
  53. recover_left_core_in(root)
  54. def _own_policy_envs(platform) -> Optional[tuple]:
  55. """``(dm_policy_env, group_policy_env, allow_all_env)`` for a platform that declares
  56. ``own_policy_env``, else None. The live entry wins; a bundled platform whose entry is absent
  57. (plugin disabled, safe mode) or lacks the field (a user plugin overriding it) falls back to the
  58. names its ``gateway_env.AUTHZ`` declares, so the startup refusal never weakens."""
  59. from gateway.platform_env import authz_field
  60. from gateway.platform_registry import platform_registry
  61. entry = platform_registry.get(platform.value)
  62. own = authz_field(platform.value, "own_policy_env", entry)
  63. if not own:
  64. return None
  65. return (*own, authz_field(platform.value, "allow_all_env", entry))
  66. class GatewayStartupMixin:
  67. """Startup sequence, resume/restore and handoff methods for GatewayRunner."""
  68. @staticmethod
  69. def _log_agent_budget() -> None:
  70. """Report the ENFORCED per-turn budget: ``agent.max_turns`` is bridged into
  71. ``HERMES_MAX_ITERATIONS`` before this runs, so the env slot already carries the config value;
  72. resolve it the same way the turn loop does (``none``/``unlimited`` spellings included) instead of
  73. ``int()`` on the raw string with an invented ``500`` default (#116888)."""
  74. from hermes_cli.config import TURN_LIMIT_UNLIMITED, resolve_turn_limit
  75. limit = resolve_turn_limit(os.getenv("HERMES_MAX_ITERATIONS"))
  76. logger.info("Agent budget: max_iterations=%s (agent.max_turns from config.yaml, else the HERMES_MAX_ITERATIONS bridge)",
  77. "unlimited" if limit == TURN_LIMIT_UNLIMITED else limit)
  78. # A configured platform failed non-retryably this boot and is parked: every "we are serving"
  79. # status stamp (startup, drain release, scale-to-zero wake) must say ``degraded``, not ``running``.
  80. _startup_parked_platforms: bool = False
  81. # A startup platform failure was a live foreign token/profile holder (exit 78 when nothing else runs).
  82. _startup_ownership_conflict: bool = False
  83. def _serving_state(self) -> str:
  84. return "degraded" if self._startup_parked_platforms else "running"
  85. def _park_last_fatal_platform(self, adapter) -> bool:
  86. """Mid-run twin of the all-fatal startup case: the last platform failed non-retryably, so park it
  87. and stay up degraded for local clients and cron. Return False when the gateway must stop instead
  88. (retryable, or another gateway owns the token/profile)."""
  89. if adapter.fatal_error_retryable or is_global_startup_conflict(adapter.fatal_error_code):
  90. return False
  91. self._startup_parked_platforms = True
  92. # An active drain or shutdown owns the state; the drain release stamps _serving_state() itself.
  93. if not (self._external_drain_active or self._draining or not self._running):
  94. self._update_runtime_status(self._serving_state())
  95. logger.error(
  96. "%s is parked after a non-retryable error (fix it, then `hermes gateway restart`). The gateway "
  97. "is DEGRADED — it serves local clients and cron with no messaging platform. %s: %s",
  98. adapter.platform.value, adapter.fatal_error_code or "unknown",
  99. adapter.fatal_error_message or "unknown error",
  100. )
  101. return True
  102. async def _run_startup_resume_event(
  103. self, adapter: BasePlatformAdapter, event: MessageEvent, session_key: str,
  104. ) -> None:
  105. """Dispatch one synthetic startup resume and wait for its agent turn (inbound stays queued
  106. until it finishes, else a user message can race it)."""
  107. from gateway.run import _AGENT_PENDING_SENTINEL
  108. try:
  109. await adapter.handle_message(event)
  110. session_tasks = getattr(adapter, "_session_tasks", {})
  111. task = session_tasks.get(session_key) if isinstance(session_tasks, dict) else None
  112. if task is not None:
  113. await asyncio.shield(task)
  114. finally:
  115. # Release the pre-claimed slot if handle_message raised before _handle_message took ownership.
  116. _pre_state = self._peek_session_state(session_key)
  117. if (_pre_state.turn.agent if _pre_state else None) is _AGENT_PENDING_SENTINEL:
  118. self._release_running_agent_state(session_key)
  119. def _queue_startup_restore_event(self, event: MessageEvent) -> None:
  120. queue = getattr(self, "_startup_restore_queue", None)
  121. if queue is None:
  122. queue = self._startup_restore_queue = []
  123. queue.append(event)
  124. with suppress(Exception):
  125. source = event.source
  126. logger.info(
  127. "Queued inbound message during gateway startup restore: platform=%s chat=%s",
  128. source.platform.value if source and source.platform else "unknown",
  129. source.chat_id if source else "unknown",
  130. )
  131. async def _drain_startup_restore_queue(self) -> int:
  132. """Replay inbound messages queued while startup auto-resume ran."""
  133. drained = 0
  134. queue = getattr(self, "_startup_restore_queue", None) or []
  135. while queue:
  136. event = queue.pop(0)
  137. try:
  138. source = getattr(event, "source", None)
  139. adapter = self._intake_adapter_for(source)
  140. if adapter is None:
  141. logger.warning(
  142. "Dropping startup-restore queued message: adapter unavailable for %s",
  143. getattr(getattr(source, "platform", None), "value", None),
  144. )
  145. continue
  146. # Mark the replay so _handle_message does not re-queue it while the restore gate is closed.
  147. with suppress(Exception):
  148. event._hermes_startup_restore_replay = True
  149. await adapter.handle_message(event)
  150. except Exception:
  151. # One bad replay must not abort the drain: the remaining queued
  152. # events still deserve their turn, and a raise here used to skip
  153. # the gate release in _finish_startup_restore entirely.
  154. logger.warning("Startup-restore queued replay failed; continuing drain", exc_info=True)
  155. continue
  156. drained += 1
  157. return drained
  158. def _start_startup_warmup(self) -> None:
  159. """Kick off the boot turn-machinery warm-up so it overlaps the network-bound platform
  160. connects; ``_finish_startup_restore`` awaits it (bounded)."""
  161. from gateway.run import _startup_warmup_timeout_secs
  162. if _startup_warmup_timeout_secs() <= 0:
  163. self._startup_warmup_task = None
  164. return
  165. self._startup_warmup_task = asyncio.ensure_future(self._warm_turn_prerequisites())
  166. async def _run_boot_probe_in_launch_scope(self, fn):
  167. """Run a boot-time probe on the default executor under the LAUNCH profile's scope when
  168. multiplexing (the ``_discover_gateway_mcp_tools`` shape). Boot probes (``check_fn``s) read profile-scoped secrets; an unscoped read under multiplex fails
  169. closed, so routing overrides resolve as absent and default routing applies. ``copy_context``
  170. carries the contextvars across the executor hop; single-profile keeps environ semantics."""
  171. loop = asyncio.get_running_loop()
  172. if getattr(self.config, "multiplex_profiles", False):
  173. from gateway.run import _async_profile_runtime_scope
  174. from hermes_constants import get_hermes_home
  175. try:
  176. async with _async_profile_runtime_scope(get_hermes_home()):
  177. return await loop.run_in_executor(None, copy_context().run, fn)
  178. except Exception:
  179. # Same fallback as load_gateway_config_for_runner: a scope that cannot be built must not
  180. # abort startup; the probe runs unscoped as it did before.
  181. logger.debug("multiplex launch-scope entry failed for boot probe %s; running unscoped",
  182. getattr(fn, "__name__", fn), exc_info=True)
  183. return await loop.run_in_executor(None, copy_context().run, fn)
  184. async def _warm_turn_prerequisites(self) -> None:
  185. """Initialize turn machinery on an executor thread before the gate opens. Never raises: a
  186. failed warm-up degrades to lazy init and must not block startup."""
  187. from gateway.run import _warm_turn_machinery_sync
  188. with _log_suppressed(
  189. logging.WARNING, "Turn-machinery warm-up failed; first inbound turn will initialize lazily",
  190. exc_info=True,
  191. ):
  192. t0 = time.monotonic()
  193. tool_count = await self._run_boot_probe_in_launch_scope(_warm_turn_machinery_sync)
  194. logger.info(
  195. "Turn machinery warmed in %.1fs (%d tool schema(s) materialized)",
  196. time.monotonic() - t0, tool_count,
  197. )
  198. async def _await_startup_warmup(self) -> None:
  199. """Bounded wait for the boot warm-up. On timeout the gate opens anyway (availability outranks
  200. prompt completeness for a WEDGED init); the warm-up continues and a late failure is logged."""
  201. from gateway.run import _startup_warmup_timeout_secs
  202. task = getattr(self, "_startup_warmup_task", None)
  203. if task is None or task.done():
  204. return
  205. timeout = _startup_warmup_timeout_secs()
  206. if timeout <= 0:
  207. return
  208. await self._wait_bounded_or_release(
  209. {task}, timeout,
  210. "Turn-machinery warm-up still running after %.0fs; opening inbound gate anyway — the "
  211. "first turn may see lazily initialized machinery (#99373). Warm-up continues in the background.",
  212. "boot turn-machinery warm-up failed after gate release", level=logging.DEBUG,
  213. )
  214. async def _wait_bounded_or_release(
  215. self, tasks: set, timeout: float, warn_fmt: str, late_msg: str, *,
  216. level: int = logging.WARNING, track: bool = False,
  217. ) -> set:
  218. """``asyncio.wait`` (which, unlike wait_for/gather+timeout, does NOT cancel on timeout) with
  219. the gate-release warning + late-failure callback every boot gate uses. Returns ``done``.
  220. ``warn_fmt`` takes ``%.0fs`` (timeout) and optionally ``%d`` (pending count). A non-positive
  221. ``timeout`` opts out of the bound ("wait forever")."""
  222. if timeout <= 0:
  223. await asyncio.gather(*tasks, return_exceptions=True)
  224. return set(tasks)
  225. done, pending = await asyncio.wait(tasks, timeout=timeout)
  226. if pending:
  227. args = (timeout, len(pending)) if warn_fmt.count("%") > 1 else (timeout,)
  228. await asyncio.to_thread(logger.warning, warn_fmt, *args)
  229. late = self._late_failure_callback(late_msg, level=level)
  230. for task in pending:
  231. task.add_done_callback(late)
  232. if track:
  233. self._retain_background_task(task)
  234. return done
  235. async def _finish_startup_restore(self) -> None:
  236. """Wait (BOUNDED by ``_startup_restore_drain_timeout_secs``) for startup auto-resume, then
  237. release + drain inbound. On timeout the gate opens and resume turns finish in the background
  238. (NOT cancelled) — safe because ``_schedule_resume_pending_sessions`` claims each
  239. ``_running_agents`` slot SYNCHRONOUSLY first, so drained inbound queues behind."""
  240. from gateway.run import _startup_restore_drain_timeout_secs
  241. drained = 0
  242. try:
  243. tasks = list(getattr(self, "_startup_restore_tasks", []) or [])
  244. if tasks:
  245. # Tasks outliving the gate get a late-failure callback (their done-callback only discards them).
  246. done = await self._wait_bounded_or_release(
  247. set(tasks), _startup_restore_drain_timeout_secs(),
  248. "Startup-restore gate released after %.0fs with %d boot auto-resume turn(s) "
  249. "still running; draining inbound queue now (resume slots already claimed, so no "
  250. "duplicate agents). Slow turn(s) continue in the background.",
  251. "background startup auto-resume task failed after gate release", level=logging.DEBUG,
  252. )
  253. report = self._late_failure_callback("startup auto-resume task failed", level=logging.DEBUG)
  254. for task in done:
  255. report(task)
  256. self._startup_restore_tasks = []
  257. # Warm the turn machinery BEFORE the queue drains: inbound turns must not build skeleton prompts.
  258. await self._await_startup_warmup()
  259. drained = await self._drain_startup_restore_queue()
  260. finally:
  261. # The inbound gate must open no matter what raised above it (bounded wait, warm-up,
  262. # drain): a stuck _startup_restore_in_progress would queue every non-internal inbound
  263. # forever (run_inbound.py reads this flag first). See #116514.
  264. self._startup_restore_in_progress = False
  265. if drained:
  266. logger.info("Drained %d inbound message(s) queued during startup restore", drained)
  267. @staticmethod
  268. def _late_failure_callback(message: str, *, level: int = logging.WARNING):
  269. """Done-callback for boot-path tasks that outlive the startup-restore gate: surface a late
  270. failure otherwise swallowed once the task leaves ``_background_tasks``. Cancellation is
  271. expected (shutdown), not an error."""
  272. def _report(task: asyncio.Task) -> None:
  273. if task.cancelled():
  274. return
  275. exc = task.exception()
  276. if exc is not None:
  277. logger.log(level, message, exc_info=(type(exc), exc, exc.__traceback__))
  278. return _report
  279. async def _await_startup_boot_sends(self, *, planned_restart_notification_pending: bool) -> None:
  280. """Run boot-path sends without letting them pin the inbound restore gate (one Telegram
  281. flood-control sleep must not freeze inbound on every platform): same bounded wait as the
  282. resume gate, sends finish in the background on timeout. The ledger claim + ``resume_pending``
  283. clear run INLINE before the send task exists — deferring it let a hung notification expire the
  284. gate with zero rows claimed, so answered turns were replayed AND redelivered.
  285. ``_send_restart_notification`` and ``_redeliver_pending_obligations`` used to be awaited inline
  286. *before* ``_finish_startup_restore`` released the gate. See #91969.
  287. """
  288. from gateway.run import _startup_restore_drain_timeout_secs
  289. claimed = await self._claim_pending_obligations()
  290. async def _boot_sends() -> None:
  291. await self._send_restart_notification()
  292. if planned_restart_notification_pending:
  293. await self._replay_pending_planned_restart_notification()
  294. await self._redeliver_claimed_obligations(claimed)
  295. boot_task = asyncio.create_task(_boot_sends())
  296. timeout = _startup_restore_drain_timeout_secs()
  297. if timeout <= 0:
  298. await boot_task # unbounded: a failing send surfaces here (unlike the gate path)
  299. return
  300. await self._wait_bounded_or_release(
  301. {boot_task}, timeout,
  302. "Boot-path sends still running after %.0fs; releasing inbound gate so other platforms are not "
  303. "frozen. Restart notification / obligation redelivery continue in the background.",
  304. "background boot-path send failed after gate release: see traceback", track=True,
  305. )
  306. async def _clear_resume_pending_for_claimed_obligations(
  307. self, claimed: list, *, require_success: bool = False
  308. ) -> list:
  309. """Clear resume flags and return rows safe to redeliver. Startup recovery is best-effort;
  310. runtime reconnect recovery (``require_success``) is stricter: if the session-store write
  311. fails the response must not be sent, or the turn could be resumed too."""
  312. sendable = []
  313. for row in claimed:
  314. session_key = row.get("session_key") or ""
  315. if not session_key:
  316. sendable.append(row)
  317. continue
  318. try:
  319. await self.async_session_store.clear_resume_pending(session_key)
  320. except Exception:
  321. logger.debug("clear_resume_pending failed for %s", session_key, exc_info=True)
  322. if require_success:
  323. continue
  324. sendable.append(row)
  325. return sendable
  326. async def _claim_pending_obligations(self) -> list:
  327. """Claim recoverable delivery-ledger rows and clear their ``resume_pending`` flags (pure DB
  328. work, no sends). Must run INLINE BEFORE ``_schedule_resume_pending_sessions`` and the
  329. abandonable boot-send task: these sessions already produced their answer, so the resume path
  330. must not re-run (re-pay for) the turn however long the sends take. Mid-send / rejected rows
  331. carry a visible recovered-reply marker (gateway/delivery_ledger.py). Returns the rows.
  332. A session with a recoverable obligation already produced its answer — the turn completed and only
  333. delivery is owed — so clearing ``resume_pending`` here prevents the resume path from re-running (and
  334. re-paying for) a turn whose output we hold, regardless of how long the sends ahead of redelivery
  335. take (#91969). Flood-refused rows adopted inside their wait come back flagged ``adopted``: their
  336. flags are cleared here too, and ``_redeliver_claimed_obligations`` leaves them to the timer.
  337. """
  338. try:
  339. from gateway.delivery_ledger import ledger_enabled, sweep_recoverable
  340. if not await asyncio.to_thread(ledger_enabled):
  341. return []
  342. # Claim only rows whose exact transport owner is connected: platform-only filtering would spend
  343. # a disconnected bot's retry budget because another bot on that platform is online.
  344. _profile_adapters = getattr(self, "_profile_adapters", None) or {}
  345. _pval = lambda p: getattr(p, "value", str(p))
  346. _deliverable_targets = {(_pval(p), "default") for p in self.adapters}
  347. # Legacy rows (no adapter_profile) are unambiguous only without multiplexing; else fail closed.
  348. if not _profile_adapters:
  349. _deliverable_targets.update((_pval(p), None) for p in self.adapters)
  350. for _profile, _adapters in _profile_adapters.items():
  351. _deliverable_targets.update((_pval(p), _profile) for p in _adapters)
  352. claimed = await asyncio.to_thread(
  353. sweep_recoverable, None,
  354. deliverable_platforms={platform for platform, _ in _deliverable_targets},
  355. deliverable_targets=_deliverable_targets,
  356. )
  357. except Exception:
  358. logger.debug("delivery ledger sweep failed", exc_info=True)
  359. return []
  360. if not claimed:
  361. return []
  362. # Clear resume_pending for EVERY claimed row before any send: the answer is in the ledger.
  363. # Claiming already spent one of the row's redelivery attempts — the answer is in the ledger, so the
  364. # resume path must never re-run these turns (#91969).
  365. await self._clear_resume_pending_for_claimed_obligations(claimed)
  366. return claimed
  367. @staticmethod
  368. async def _release_runtime_claim_quiet(obligation_id, log_fmt: str, error: str = "send_path_degraded") -> None:
  369. """Release an unsent runtime delivery-ledger claim; log-only on failure. ``error`` is what the row
  370. goes back to ``failed`` with: the claim's own pre-claim error when the caller knows it, so a
  371. flood-refused row keeps its ``flood_control:<seconds>`` and stays on the flood timer's list (the
  372. release re-stamps ``updated_at``, so it waits the platform's figure once more); else
  373. ``send_path_degraded``."""
  374. from gateway.delivery_ledger import release_runtime_claim
  375. try:
  376. await asyncio.to_thread(release_runtime_claim, obligation_id, error)
  377. except Exception:
  378. logger.debug(log_fmt, obligation_id, exc_info=True)
  379. def _schedule_flood_redelivery(self, platform, *, profile: Optional[str] = None) -> None:
  380. """Wake one deadline-driven ledger worker per bot identity, never sleep in a send."""
  381. from gateway.delivery_ledger import flood_retry_delay, pending_retries
  382. target = platform if isinstance(platform, Platform) else Platform(str(platform))
  383. key = (target.value, profile or "default")
  384. pending = getattr(self, "_flood_redelivery_tasks", None)
  385. if pending is None:
  386. pending = self._flood_redelivery_tasks = {}
  387. self._flood_redelivery_wakes = {}
  388. wakes = self._flood_redelivery_wakes
  389. if key in pending:
  390. # Remember refusals arriving during a threaded SELECT or a send. An empty stale
  391. # snapshot cannot retire this worker until it has observed the wake.
  392. wakes[key].set()
  393. return
  394. wake = wakes[key] = asyncio.Event()
  395. async def _redeliver_after_wait():
  396. try:
  397. while getattr(self, "_running", False):
  398. wake.clear()
  399. waiting = await asyncio.to_thread(pending_retries)
  400. deadlines = [r["not_before"] for r in waiting
  401. if (r["platform"], r["profile"]) == key]
  402. if not deadlines:
  403. if wake.is_set():
  404. continue
  405. return
  406. delay = flood_retry_delay(min(deadlines) - time.time())
  407. try:
  408. await asyncio.wait_for(wake.wait(), timeout=delay)
  409. continue # A shorter sibling may now be due first.
  410. except TimeoutError:
  411. pass
  412. if getattr(self, "_running", False):
  413. await self._redeliver_failed_obligations_for_platform(target, profile=profile)
  414. finally:
  415. pending.pop(key, None)
  416. wakes.pop(key, None)
  417. task = asyncio.create_task(_redeliver_after_wait(), name="flood-redelivery:{}:{}".format(*key))
  418. pending[key] = task
  419. # The gateway's ordinary shutdown drain must cancel sleeping timers too.
  420. background = getattr(self, "_background_tasks", None)
  421. if background is not None:
  422. self._track_task_in(background, task)
  423. async def _arm_flood_timers_for_waiting_rows(self) -> None:
  424. """Recover adopted, newly refused/rejected and unsent released rows without blocking the loop."""
  425. from gateway.delivery_ledger import pending_retries
  426. for row in await asyncio.to_thread(pending_retries):
  427. self._schedule_flood_redelivery(row["platform"], profile=row["profile"])
  428. async def _redeliver_claimed_obligations(self, claimed: list) -> int:
  429. """Redeliver final responses for claimed rows (network half of the split): runs inside the
  430. bounded boot-send task, so a flood-limited send can be abandoned by the restore gate without
  431. reopening the turn-replay window. Returns the redelivered count."""
  432. # No early return on an empty claim: the boot sweep may have ADOPTED flood-refused rows that are
  433. # not due yet, and those still need their timer armed below.
  434. try:
  435. from gateway.delivery_ledger import RECOVERED_MARKER, mark_delivered, mark_failed
  436. except Exception:
  437. logger.debug("delivery ledger import failed", exc_info=True)
  438. return 0
  439. redelivered = 0
  440. for row in claimed:
  441. if row.get("adopted"):
  442. # Adopted at boot inside its flood wait: its resume flag is cleared with the others, and
  443. # the timer armed below sends it once the platform's deadline has passed.
  444. continue
  445. adapter = await self._obligation_adapter(row)
  446. if adapter is None:
  447. continue
  448. content = row["content"]
  449. if row.get("needs_marker"):
  450. content = row.get("marker", RECOVERED_MARKER) + content
  451. metadata = {"thread_id": row["thread_id"]} if row.get("thread_id") else None
  452. try:
  453. result = await adapter.send(chat_id=row["chat_id"], content=content, metadata=metadata)
  454. except Exception as send_err:
  455. logger.warning("obligation %s: redelivery send raised: %s", row["obligation_id"], send_err)
  456. result = None
  457. with _log_suppressed(logging.DEBUG, "delivery ledger update failed", exc_info=True):
  458. if result is not None and getattr(result, "success", False):
  459. await asyncio.to_thread(mark_delivered, row["obligation_id"])
  460. redelivered += 1
  461. logger.info(
  462. "Redelivered recovered final response to %s:%s (obligation %s, attempt %d)",
  463. row["platform"], row["chat_id"], row["obligation_id"], row["attempts"],
  464. )
  465. else:
  466. await asyncio.to_thread(
  467. mark_failed, row["obligation_id"], str(getattr(result, "error", "") or "send failed")
  468. )
  469. # Whatever is still waiting on a flood penalty or a retry backoff (adopted at boot, skipped as not
  470. # yet due, refused again just now) gets a timer, so no rejected reply waits for the next restart.
  471. with _log_suppressed(logging.DEBUG, "arming flood redelivery timers failed", exc_info=True):
  472. await self._arm_flood_timers_for_waiting_rows()
  473. return redelivered
  474. async def _obligation_adapter(self, row: dict):
  475. """Resolve the adapter for a claimed ledger row, or None when it cannot be delivered now."""
  476. try:
  477. platform = Platform(row["platform"])
  478. except Exception:
  479. logger.debug("obligation %s: unknown platform %r", row["obligation_id"], row.get("platform"))
  480. return None
  481. if "profile" in row:
  482. adapter = self._authorization_adapter(platform, row.get("profile"))
  483. else:
  484. # Startup rows preserve the historical default-adapter route.
  485. adapter = self.adapters.get(platform)
  486. # A claim (runtime or boot) whose adapter vanished before dispatch was never sent: release it
  487. # without spending an attempt so the reconnect sweep, which only takes 'failed' rows, delivers it
  488. # instead of it waiting 'attempting' for the next restart.
  489. # Only a flood row keeps its error (the platform's wait must be honoured); any other row
  490. # becomes reconnect-only, or the redelivery timer would claim and release it until the
  491. # adapter is back.
  492. if adapter is None:
  493. from gateway.delivery_ledger import is_flood_error
  494. last_error = row.get("last_error")
  495. await self._release_runtime_claim_quiet(
  496. row["obligation_id"], "failed to release undispatched runtime obligation %s",
  497. error=last_error if is_flood_error(last_error) else "send_path_degraded",
  498. )
  499. return adapter
  500. async def _redeliver_pending_obligations(self) -> int:
  501. """Claim + redeliver in one call. Stable shape for tests/external callers; the startup path
  502. calls the halves separately so the DB half runs inline before the abandonable send task."""
  503. return await self._redeliver_claimed_obligations(await self._claim_pending_obligations())
  504. async def _redeliver_failed_obligations_for_platform(
  505. self, platform: Platform, *, profile: Optional[str] = None,
  506. ) -> int:
  507. """Replay one adapter identity's transient failures after reconnect: the startup sweep cannot
  508. claim live-owner rows, so ``send_path_degraded`` responses would otherwise stay failed until
  509. the next restart. Best-effort; reuses the startup redelivery contract."""
  510. try:
  511. from gateway.delivery_ledger import ledger_enabled, sweep_failed_for_runtime
  512. if not await asyncio.to_thread(ledger_enabled):
  513. return 0
  514. claimed = await asyncio.to_thread(sweep_failed_for_runtime, platform.value, profile=profile)
  515. except Exception:
  516. logger.debug(
  517. "runtime delivery ledger sweep failed after %s reconnect", platform.value, exc_info=True,
  518. )
  519. return 0
  520. if not claimed:
  521. return 0
  522. # Clear before any send so the reconnect path cannot both redeliver AND resume the same turn.
  523. sendable = await self._clear_resume_pending_for_claimed_obligations(claimed, require_success=True)
  524. sendable_ids = {row["obligation_id"] for row in sendable}
  525. for row in claimed:
  526. if row["obligation_id"] not in sendable_ids:
  527. await self._release_runtime_claim_quiet(
  528. row["obligation_id"], "failed to release runtime delivery claim %s",
  529. error=row.get("last_error") or "send_path_degraded",
  530. )
  531. return await self._redeliver_claimed_obligations(sendable)
  532. def _resume_pending_candidates(self, platform=None) -> Optional[list]:
  533. """Snapshot resume-pending entries (optionally scoped to ``platform``); None when
  534. enumeration failed or the restart-loop breaker tripped for this boot."""
  535. try:
  536. with self.session_store._lock:
  537. self.session_store._ensure_loaded_locked()
  538. candidates = [
  539. entry for entry in self.session_store._entries.values()
  540. if entry.resume_pending
  541. and not entry.suspended
  542. and entry.origin is not None
  543. and entry.resume_reason in self._AUTO_RESUME_REASONS
  544. and (platform is None or entry.origin.platform == platform)
  545. ]
  546. except Exception as exc:
  547. logger.warning("Failed to enumerate resume-pending sessions: %s", exc)
  548. return None
  549. # Restart-loop breaker: only boots WITH restart-interrupted sessions count; when tripped, skip
  550. # auto-resume for THIS boot only (inbound still served; sessions stay resume_pending).
  551. if candidates:
  552. try:
  553. from gateway import restart_loop_guard as _rlg
  554. _max_restarts, _window, _max_gap = self._restart_loop_guard_config()
  555. if _rlg.check_and_record(_max_restarts, _window, max_gap_seconds=_max_gap):
  556. return None
  557. except Exception as exc:
  558. logger.debug("Restart-loop guard check skipped: %s", exc)
  559. return candidates
  560. def _resume_owner_authorized(self, session_key: str, source) -> bool:
  561. """Validate the session owner against the CURRENT allowlist: a session created before the
  562. allowlist existed (or whose owner was since removed) must not silently receive a full agent
  563. response just because it carries a resume marker."""
  564. try:
  565. if self._is_user_authorized_for_source(source):
  566. return True
  567. logger.warning(
  568. "Skipping auto-resume for %s: session owner is no "
  569. "longer authorized under the current allowlist", session_key,
  570. )
  571. except Exception as exc:
  572. logger.warning("Skipping auto-resume for %s: authorization check failed: %s", session_key, exc)
  573. return False
  574. def _schedule_resume_pending_sessions(self, platform=None) -> int:
  575. """Auto-continue fresh restart-interrupted sessions: synthesize an empty-text turn (the
  576. ``_is_resume_pending`` injection path owns the wording). Sessions whose adapter is offline stay
  577. ``resume_pending`` for the reconnect watcher, which re-calls this scoped to that ``platform``;
  578. sessions with a running agent are skipped so none is resumed twice."""
  579. from gateway.run import (
  580. _AGENT_PENDING_SENTINEL, _auto_continue_freshness_window, _is_fresh_gateway_interruption,
  581. )
  582. window = _auto_continue_freshness_window()
  583. candidates = self._resume_pending_candidates(platform)
  584. if candidates is None:
  585. return 0
  586. scheduled = 0
  587. for entry in candidates:
  588. # Canonical admissions own restart decisions, including unknown pauses.
  589. # Legacy synthetic resume must not race their restored FIFO.
  590. from gateway.session_authorities import all_authorities, holds_admission
  591. if any(holds_admission(authority, entry.session_id) for authority in all_authorities(self)):
  592. continue
  593. # Epoch math: the marker was stamped naive-local by the previous process, possibly
  594. # on the other side of a DST change; wall-clock subtraction is off by the shift.
  595. marker = entry.last_resume_marked_at or entry.updated_at
  596. if not _is_fresh_gateway_interruption(marker, window_secs=window):
  597. continue
  598. # Already being resumed (e.g. scheduled at startup, still in-flight) — no second turn.
  599. if self._is_session_running(entry.session_key):
  600. continue
  601. source = self._restored_source(entry)
  602. adapter = self._delivery_adapter_for(source)
  603. if adapter is None:
  604. logger.debug(
  605. "Skipping auto-resume for %s: adapter not ready for %s", entry.session_key,
  606. getattr(source.platform, "value", source.platform),
  607. )
  608. continue
  609. if not self._resume_owner_authorized(entry.session_key, source):
  610. continue
  611. # Claim the slot *before* spawning so an inbound message arriving before the task's first
  612. # await queues instead of building a duplicate AIAgent.
  613. _resume_state = self._session_state(entry.session_key)
  614. _resume_state.turn.agent = _AGENT_PENDING_SENTINEL
  615. _resume_state.turn.started_ts = time.time()
  616. note_running_chat_owner(self, entry.session_key, source) # this loop runs in the launch scope
  617. self._persist_active_agents()
  618. # Empty-text internal event: the _is_resume_pending branch prepends the reason-aware note.
  619. event = MessageEvent(text="", message_type=MessageType.TEXT, source=source, internal=True)
  620. task = self._retain_background_task(
  621. asyncio.create_task(self._run_startup_resume_event(adapter, event, entry.session_key))
  622. )
  623. if getattr(self, "_startup_restore_in_progress", False):
  624. tasks = getattr(self, "_startup_restore_tasks", None)
  625. if tasks is None:
  626. tasks = self._startup_restore_tasks = []
  627. tasks.append(task)
  628. scheduled += 1
  629. if scheduled:
  630. logger.info("Scheduled auto-resume for %d restart-interrupted session(s)", scheduled)
  631. return scheduled
  632. def _startup_should_abort(self) -> bool:
  633. return self._restart_requested or self._draining or self._shutdown_event.is_set()
  634. async def _startup_teardown_adapter(self, adapter, platform) -> None:
  635. """Cancel an adapter's background tasks (best-effort) then disconnect it."""
  636. with _log_suppressed(logging.DEBUG, "✗ %s background-task cancel error: %s", platform.value):
  637. await adapter.cancel_background_tasks()
  638. await self._safe_adapter_disconnect(adapter, platform)
  639. def _startup_retry_entry(self, platform, adapter, platform_config, *, queued: bool = True) -> dict:
  640. """``_failed_platforms`` entry for a platform that failed at startup (first retry in 30s)."""
  641. return self._reconnect_queue_entry(platform, adapter, platform_config, attempts=1, delay=30, queued=queued)
  642. async def _abort_startup_if_shutdown_requested(
  643. self, adapter: Optional[BasePlatformAdapter] = None, platform: Optional[Platform] = None
  644. ) -> bool:
  645. """Clean up and exit startup when restart/shutdown begins mid-startup."""
  646. if not self._startup_should_abort():
  647. return False
  648. if adapter is not None and platform is not None:
  649. await self._startup_teardown_adapter(adapter, platform)
  650. stop_task = self._stop_task
  651. current_task = asyncio.current_task()
  652. if stop_task is not None and stop_task is not current_task:
  653. await stop_task
  654. elif not self._shutdown_event.is_set():
  655. await self.stop(
  656. restart=self._restart_requested, detached_restart=self._restart_detached,
  657. service_restart=self._restart_via_service,
  658. )
  659. return True
  660. def _start_loop_liveness_guards(self, loop: asyncio.AbstractEventLoop) -> None:
  661. """Arm the selector floor and out-of-loop watchdog before adapters. Disabled entirely with
  662. ``gateway.loop_watchdog: false`` in config.yaml (config-only knob).
  663. See #69089.
  664. """
  665. from gateway.shutdown_watchdog import _arm_loop_floor_timer, start_loop_liveness_watchdog
  666. config = getattr(self, "config", None)
  667. if config is not None and not getattr(config, "loop_watchdog", True):
  668. return
  669. if getattr(self, "_loop_floor_timer_handle", None) is None:
  670. with _log_suppressed(logging.DEBUG, "Failed to arm gateway loop floor timer", exc_info=True):
  671. self._loop_floor_timer_handle = _arm_loop_floor_timer(loop)
  672. watchdog = getattr(self, "_loop_liveness_watchdog", None)
  673. if watchdog is None or not watchdog.is_alive():
  674. try:
  675. # getattr defaults cover config=None test paths; loaded values are already clamped.
  676. self._loop_liveness_watchdog = start_loop_liveness_watchdog(
  677. loop,
  678. probe_interval=float(getattr(
  679. config, "loop_watchdog_probe_interval_s", DEFAULT_LOOP_WATCHDOG_INTERVAL_S
  680. )),
  681. probe_timeout=float(getattr(
  682. config, "loop_watchdog_probe_timeout_s", DEFAULT_LOOP_WATCHDOG_TIMEOUT_S
  683. )),
  684. max_strikes=int(getattr(
  685. config, "loop_watchdog_max_strikes", DEFAULT_LOOP_WATCHDOG_MAX_STRIKES
  686. )),
  687. )
  688. except Exception:
  689. logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True)
  690. def _stop_loop_liveness_guards(self) -> None:
  691. """Disarm lifetime liveness guards before shutdown can load the loop — including the
  692. heartbeat writer, which would otherwise make a draining gateway look healthy to probes."""
  693. for attr, method, what in (
  694. ("_loop_liveness_watchdog", "stop", "stop gateway loop liveness watchdog"),
  695. ("_loop_floor_timer_handle", "cancel", "cancel gateway loop floor timer"),
  696. ("_loop_heartbeat_task", "cancel", "cancel gateway loop heartbeat task"),
  697. ):
  698. guard = getattr(self, attr, None)
  699. setattr(self, attr, None)
  700. if guard is not None:
  701. with _log_suppressed(logging.DEBUG, "Failed to %s", what, exc_info=True):
  702. getattr(guard, method)()
  703. async def _consume_clean_shutdown_marker(self, marker_path) -> int:
  704. """Discard orphan turn markers before consuming a clean-exit receipt. Raises (fail closed):
  705. continuing with the old receipt would let a later unclean exit masquerade as clean."""
  706. discarded = await self.async_session_store.discard_active_turn_markers()
  707. marker_path.unlink()
  708. return discarded
  709. async def _recover_unclean_sessions(self) -> tuple[int, int]:
  710. """Recover only the turns the dead process left marked: one whose reply is already in the
  711. transcript is owed delivery, not a new answer; any other resumes once. An unmarked session
  712. finished its turn (the marker is held until the reply is ledgered), so nothing re-runs it —
  713. the old 120 s recency sweep re-answered every recently active chat. Returns (resumed,
  714. ledgered)."""
  715. from gateway.run import _float_env
  716. resumed = ledgered = 0
  717. max_age = max(60 * 60, int(max(1.0, _float_env("HERMES_AGENT_TIMEOUT", 1800)) * 2))
  718. with _log_suppressed(logging.WARNING, "Crash-left reply recovery on startup failed: %s"):
  719. ledgered = await self._ledger_crash_left_replies(max_age)
  720. with _log_suppressed(logging.WARNING, "Exact active-turn recovery on startup failed: %s"):
  721. resumed = await self.async_session_store.recover_interrupted_turns(max_age_seconds=max_age)
  722. return resumed, ledgered
  723. async def _ledger_crash_left_replies(self, max_age_seconds: int) -> int:
  724. """Settle every marked turn whose final reply was persisted and clear its marker, so
  725. auto-resume does not regenerate it: a reply live delivery would have suppressed is owed
  726. nothing, any other goes to the delivery ledger for the boot sweep. Without the ledger a
  727. presentable reply stays marked and resumes."""
  728. from gateway.delivery_ledger import compute_obligation_id, ledger_enabled, record_crash_left_reply
  729. ledger_on = await asyncio.to_thread(ledger_enabled)
  730. cutoff = time.time() - max_age_seconds # older markers are cleared, never acted on
  731. with self.session_store._lock:
  732. self.session_store._ensure_loaded_locked()
  733. marked = [
  734. (e.session_key, e.session_id, e.active_turn_token, e.active_turn_started_at, e.origin,
  735. e.transport_profile)
  736. for e in self.session_store._entries.values()
  737. if e.active_turn_token and e.active_turn_started_at and e.origin and not e.suspended
  738. ]
  739. ledgered = 0
  740. for key, session_id, token, started_at, origin, profile in marked:
  741. started = started_at.timestamp() # aware UTC marker; a pre-upgrade naive one reads as local
  742. if started < cutoff:
  743. continue
  744. decision = await self.async_session_store.get_session_metadata(key, "room_silence_decision", {})
  745. text = self._crash_left_reply(await self.async_session_store.load_transcript(session_id),
  746. started, origin, silence_decision=decision, active_token=token)
  747. if text is None or (text and not ledger_on):
  748. continue # no final reply to deliver: the turn resumes
  749. if text:
  750. await asyncio.to_thread(
  751. record_crash_left_reply,
  752. obligation_id=compute_obligation_id(key, f"crash:{token}", text), session_key=key,
  753. platform=str(getattr(origin.platform, "value", origin.platform)), chat_id=origin.chat_id,
  754. thread_id=origin.thread_id, content=text, since=started, adapter_profile=profile)
  755. if await self.async_session_store.clear_turn_active(key, token) and text:
  756. ledgered += 1
  757. return ledgered
  758. def _crash_left_reply(
  759. self, history: list, started: float, origin, *, silence_decision=None, active_token=None,
  760. ) -> Optional[str]:
  761. """What a crash-left turn owes, judged as live delivery would have: ``None`` when it never
  762. persisted a final reply after *started*; ``""`` when nothing would have been presented (a
  763. silence marker on a machinery turn or on a turn the adapter reported as not addressed to the
  764. bot, a muted diagnostic wake); else the text to send, with any other bare silence marker
  765. replaced by the same notice the live path sends."""
  766. from gateway.platforms.base import _strip_media_directives
  767. from gateway.response_filters import (
  768. is_intentional_silence_response, is_machinery_display_kind, silence_allowed,
  769. strip_trailing_silence_marker,
  770. )
  771. from gateway.run import _sanitize_gateway_final_response
  772. from gateway.run_turn import _unexpected_silence_reply
  773. from gateway.warning_notifications import diagnostic_turn_muted
  774. from hermes_cli.timefmt import coerce_epoch
  775. visible = [m for m in history if m.get("role") not in ("session_meta", "system")]
  776. last = visible[-1] if visible else {}
  777. if (last.get("role") != "assistant" or last.get("tool_calls") or not isinstance(last.get("content"), str)
  778. or (coerce_epoch(last.get("timestamp")) or 0) < started):
  779. return None
  780. prompt = next((m for m in reversed(visible) if m.get("role") == "user"), {})
  781. if is_machinery_display_kind(prompt.get("display_kind")):
  782. try: # the owning profile's display policy, as the adapter reads it at delivery
  783. scope = self._media_delivery_scope_for_source(origin)
  784. except Exception:
  785. logger.debug("Crash-left reply: no routed scope for %s", origin.chat_id, exc_info=True)
  786. scope = nullcontext()
  787. with scope:
  788. if diagnostic_turn_muted(prompt.get("display_metadata"), origin.platform):
  789. return ""
  790. if is_intentional_silence_response(last["content"]):
  791. decision = silence_decision or {}
  792. proven_silent = (active_token is not None and decision.get("token") == active_token
  793. and decision.get("timestamp") == last.get("timestamp")
  794. and decision.get("silent") is True)
  795. silent_ok = silence_allowed(
  796. prompt.get("display_kind"), (prompt.get("display_metadata") or {}).get("reply_expected"),
  797. allow_silence=proven_silent, agent_result={})
  798. return "" if silent_ok else _unexpected_silence_reply()
  799. reply = strip_trailing_silence_marker(last["content"])
  800. return _strip_media_directives(_sanitize_gateway_final_response(origin.platform, reply)).strip() or None
  801. @staticmethod
  802. def _start_hosted_room_worker_sync():
  803. """Start the local Group Chat worker without importing the dashboard."""
  804. import tui_gateway.server
  805. from tui_gateway import methods_groups
  806. service = methods_groups.get_hosted_room_service()
  807. if service is None:
  808. service = methods_groups.start_hosted_room_service()
  809. if service is None:
  810. raise RuntimeError("Group Chat worker has no bound session backend")
  811. status = service.runtime.status()
  812. if not status.get("running") or status.get("stopping"):
  813. raise RuntimeError("Group Chat worker did not start")
  814. return service
  815. async def _ensure_hosted_room_worker(self):
  816. if getattr(self, 'session_authority', None) is not None:
  817. from gateway.session_hosted_service import ensure_hosted_service
  818. return await ensure_hosted_service(self)
  819. return await asyncio.to_thread(self._start_hosted_room_worker_sync)
  820. async def _hosted_room_worker_watcher(self, interval: float = 1.0) -> None:
  821. """Keep the room worker alive for the messaging gateway lifetime."""
  822. while self._running:
  823. await self._ensure_hosted_room_worker()
  824. await asyncio.sleep(interval)
  825. async def _stop_hosted_room_worker(self, timeout: float = 5.0) -> bool:
  826. """Pause room execution durably without interrupting accepted turns."""
  827. if getattr(self, 'session_authority', None) is not None:
  828. from gateway.session_hosted_service import stop_hosted_service
  829. return await stop_hosted_service(self, timeout=timeout)
  830. from tui_gateway import methods_groups
  831. return await asyncio.to_thread(methods_groups.stop_hosted_room_service, timeout=timeout)
  832. def _start_loop_heartbeat_task(self) -> None:
  833. """Start the loop-liveness heartbeat task (idempotent, best-effort). An asyncio task so a
  834. frozen loop stops refreshing ``state/gateway.heartbeat``; cancelled with the others in stop().
  835. See #66892.
  836. """
  837. with _log_suppressed(logging.DEBUG, "Failed to start gateway loop heartbeat", exc_info=True):
  838. _existing_hb = getattr(self, "_loop_heartbeat_task", None)
  839. if _existing_hb is not None and not _existing_hb.done():
  840. return
  841. task = self._loop_heartbeat_task = asyncio.create_task(
  842. loop_heartbeat_forever(
  843. interval_s=DEFAULT_HEARTBEAT_INTERVAL_S,
  844. start_time=getattr(self, "_gateway_started_at", 0.0),
  845. )
  846. )
  847. # PERMANENT watcher tag so the scale-to-zero idle check doesn't count it as busy forever.
  848. task._hermes_supervised_watcher = True # type: ignore[attr-defined]
  849. _bg = getattr(self, "_background_tasks", None)
  850. if _bg is not None:
  851. self._track_task_in(_bg, task)
  852. def _open_faulthandler_log(self):
  853. """Open (append) ``<log_dir>/gateway_faulthandler.log``, creating the directory."""
  854. from gateway.run import get_hermes_home
  855. log_dir = getattr(self.config, "log_dir", None) or os.path.join(str(get_hermes_home()), "logs")
  856. os.makedirs(log_dir, exist_ok=True)
  857. return open(os.path.join(log_dir, "gateway_faulthandler.log"), "a", encoding="utf-8") # windows-footgun: ok (append log writer, not a read)
  858. def _start_install_faulthandler(self) -> None:
  859. """Enable faulthandler (stderr or a log file) plus the SIGUSR2 stack-dump hook."""
  860. # sys.stderr may be None (Windows VBS / pythonw / detached service): fall back to a log file.
  861. try:
  862. # Enable faulthandler for stack dumps on freezes/crashes (#70344). Falls back to a log file when
  863. # sys.stderr is None (Windows VBS / pythonw / detached service) — otherwise the gateway would
  864. # die here and take every adapter offline. See #71671.
  865. faulthandler.enable()
  866. except (RuntimeError, ValueError, OSError):
  867. with _log_suppressed(logging.DEBUG, "faulthandler.enable() unavailable", exc_info=True):
  868. faulthandler.enable(file=self._open_faulthandler_log(), all_threads=True)
  869. # SIGUSR2 stack dump to file for service managers that drop stderr; POSIX-only.
  870. # chain=False: SIGUSR2's default disposition is "terminate", so chaining to it
  871. # dumps the stacks and then kills the gateway the operator was trying to inspect.
  872. _sigusr2 = getattr(signal, "SIGUSR2", None)
  873. if _sigusr2 is not None and hasattr(faulthandler, "register"):
  874. with _log_suppressed(logging.DEBUG, "Could not set up faulthandler file logging", exc_info=True):
  875. faulthandler.register(
  876. _sigusr2, file=self._open_faulthandler_log(), all_threads=True, chain=False,
  877. )
  878. async def _start_log_startup_environment(self) -> None:
  879. """Bind the gateway loop, disarm the startup watchdog, and log the startup environment."""
  880. try:
  881. self._gateway_loop = asyncio.get_running_loop()
  882. except RuntimeError:
  883. self._gateway_loop = None
  884. if self._gateway_loop is not None:
  885. self._start_loop_liveness_guards(self._gateway_loop)
  886. # Loop live: the loop-liveness watchdog takes over from the startup watchdog. Disarm even
  887. # when loop guards are config-disabled; only inside this branch (no live loop = stay armed).
  888. with _log_suppressed(logging.DEBUG, "Startup watchdog disarm failed", exc_info=True):
  889. from hermes_startup_watchdog import disarm_startup_watchdog
  890. disarm_startup_watchdog()
  891. logger.info("Session storage: %s", self.config.sessions_dir)
  892. self._start_log_systemd_timing_alignment()
  893. self._start_log_retired_session_reset()
  894. self._log_agent_budget()
  895. # Warn prominently when redaction is opted out; the redactor snapshots its state at import time,
  896. # so this line is the source of truth for the process lifetime.
  897. with suppress(Exception):
  898. # Redaction status: ON by default (#17691).
  899. _redact_raw = os.getenv("HERMES_REDACT_SECRETS", "true")
  900. if _redact_raw.lower() in {"1", "true", "yes", "on"}:
  901. logger.info(
  902. "Secret redaction: ENABLED (tool output, logs, and chat "
  903. "responses are scrubbed before delivery)"
  904. )
  905. else:
  906. logger.warning(
  907. "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). API keys and tokens may appear "
  908. "verbatim in chat output, session JSONs, and logs. Set security.redact_secrets: true "
  909. "in config.yaml to re-enable.", _redact_raw,
  910. )
  911. with suppress(Exception):
  912. from hermes_cli.profiles import get_active_profile_name
  913. _profile = get_active_profile_name() # launch profile, pre-identity (boot log)
  914. if _profile and _profile != "default":
  915. logger.info("Active profile: %s", _profile)
  916. try:
  917. from gateway.status import write_runtime_status
  918. persisted = await asyncio.to_thread(
  919. write_runtime_status,
  920. gateway_state="starting",
  921. exit_reason=None,
  922. clear_profile_platforms=True,
  923. reload_existing=True,
  924. wait_timeout=2.0,
  925. )
  926. if not persisted:
  927. logger.warning("Timed out persisting initial gateway runtime status")
  928. except Exception:
  929. logger.debug("Initial gateway runtime-status write failed", exc_info=True)
  930. with _log_suppressed(logging.DEBUG, "gateway health OTLP export startup failed", exc_info=True):
  931. from hermes_cli.config import load_config
  932. from agent.monitoring.gateway_health_export import start_gateway_health_export
  933. self._gateway_health_export_runtime = start_gateway_health_export(load_config())
  934. if getattr(self._gateway_health_export_runtime, "enabled", False):
  935. logger.info("Gateway health OTLP export: enabled")
  936. # Supply-chain advisories: log only (never block startup or surface to users; only the operator can act).
  937. with _log_suppressed(logging.DEBUG, "security advisory check failed at gateway startup", exc_info=True):
  938. from hermes_cli.security_advisories import detect_compromised, gateway_log_message
  939. _adv_msg = gateway_log_message(detect_compromised())
  940. if _adv_msg:
  941. logger.warning("%s", _adv_msg)
  942. logger.warning("Run `hermes doctor` on the gateway host for full remediation steps.")
  943. def _start_log_retired_session_reset(self) -> None:
  944. """Warn per served profile whose config still declares an idle/daily ``session_reset``,
  945. unless the plugin that honours it is enabled. Never raises."""
  946. with _log_suppressed(logging.DEBUG, "retired session_reset check failed", exc_info=True):
  947. from gateway.config_loader import read_yaml_layers
  948. from hermes_cli.profiles import profiles_to_serve
  949. from hermes_cli.session_reset_retirement import format_notice, reset_plugin_enabled, retired_reset_policy
  950. hits = [(name, found) for name, home in profiles_to_serve(bool(self.config.multiplex_profiles))
  951. if (found := retired_reset_policy(read_yaml_layers(home)))]
  952. if hits and not reset_plugin_enabled():
  953. for name, (path, mode) in hits:
  954. logger.warning("Profile %s: %s", name, format_notice(path, mode))
  955. def _start_log_systemd_timing_alignment(self) -> None:
  956. """Warn when systemd's TimeoutStopSec does not cover the drain window (a unit file from before
  957. an upgrade may encode the old default, so SIGKILL hits mid-drain). Never raises."""
  958. with _log_suppressed(logging.DEBUG, "check_systemd_timing_alignment failed: %s"):
  959. from gateway.shutdown_forensics import check_systemd_timing_alignment
  960. _alignment = check_systemd_timing_alignment(
  961. self._restart_drain_timeout,
  962. getattr(self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT),
  963. )
  964. if _alignment is not None and _alignment.get("mismatch"):
  965. logger.warning(
  966. "Stale systemd unit detected: %s has TimeoutStopSec=%.0fs but drain_timeout=%.0fs "
  967. "cron_drain_timeout=%.0fs (expected >=%.0fs). systemd may SIGKILL the gateway "
  968. "mid-drain. Run `hermes gateway install --force` to regenerate the unit, or shorten "
  969. "agent.restart_drain_timeout / agent.cron_drain_timeout.",
  970. _alignment.get("unit", "(unknown)"), _alignment["timeout_stop_sec"],
  971. _alignment["drain_timeout"],
  972. _alignment.get("cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT),
  973. _alignment["expected_min"],
  974. )
  975. # Builtin platforms whose ``<P>_ALLOWED_USERS`` / ``<P>_ALLOW_ALL_USERS`` env vars count as an
  976. # allowlist / open-access opt-in; plugin platforms are appended at check time.
  977. _ALLOWLIST_ENV_PLATFORMS = (
  978. "TELEGRAM", "DISCORD", "WHATSAPP", "WHATSAPP_CLOUD", "SLACK", "SIGNAL", "EMAIL", "SMS",
  979. "MATTERMOST", "MATRIX", "DINGTALK", "FEISHU", "WECOM", "WECOM_CALLBACK", "WEIXIN",
  980. "BLUEBUBBLES", "QQ", "YUANBAO",
  981. )
  982. _BUILTIN_ALLOWED_USERS_VARS = tuple(f"{p}_ALLOWED_USERS" for p in _ALLOWLIST_ENV_PLATFORMS) + (
  983. "SIGNAL_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS",
  984. "TELEGRAM_GROUP_ALLOWED_CHATS", "GATEWAY_ALLOWED_USERS",
  985. )
  986. _BUILTIN_ALLOW_ALL_VARS = tuple(f"{p}_ALLOW_ALL_USERS" for p in _ALLOWLIST_ENV_PLATFORMS)
  987. def _start_check_access_policy(self) -> bool:
  988. """Warn about missing allowlists; return True when startup must be refused."""
  989. from gateway.run import (
  990. _own_policy_open_startup_violation, _write_runtime_status_quiet
  991. )
  992. # Plugin platforms declare their own allowed_users_env / allow_all_env.
  993. allowed_vars = list(self._BUILTIN_ALLOWED_USERS_VARS)
  994. allow_all_vars = ["GATEWAY_ALLOW_ALL_USERS", *self._BUILTIN_ALLOW_ALL_VARS]
  995. with suppress(Exception):
  996. from gateway.platform_registry import platform_registry
  997. # Enabled plugin platforms were materialized by the config passes; an unconfigured
  998. # adapter's allowlist var cannot gate anything, so never import one just to read it.
  999. entries = [e for e in (*platform_registry.loaded_entries(), *filter(None, (
  1000. platform_registry.get(p.value) for p, cfg in self.config.platforms.items() if cfg.enabled)))
  1001. if e.source == "plugin"]
  1002. allowed_vars += [e.allowed_users_env for e in entries if e.allowed_users_env]
  1003. allow_all_vars += [e.allow_all_env for e in entries if e.allow_all_env]
  1004. # An API-server/webhook/local-only gateway has no messaging sender to gate, so the
  1005. # reminder would be unactionable monitoring noise (#115439).
  1006. non_messaging = {Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK}
  1007. has_messaging = any(
  1008. cfg.enabled and platform not in non_messaging for platform, cfg in self.config.platforms.items()
  1009. )
  1010. if has_messaging and not any(os.getenv(v) for v in allowed_vars) and not any(
  1011. os.getenv(v, "").lower() in {"true", "1", "yes"} for v in allow_all_vars
  1012. ):
  1013. logger.warning(
  1014. "No env user allowlists configured. Messaging platforms default to pairing/allowlist "
  1015. "policies and will deny unknown senders unless you configure platform allowlists (e.g., "
  1016. "TELEGRAM_ALLOWED_USERS=your_id) or explicitly opt in with GATEWAY_ALLOW_ALL_USERS=true "
  1017. "plus dm_policy/group_policy: open on the platform."
  1018. )
  1019. reason = _own_policy_open_startup_violation(self.config)
  1020. if reason:
  1021. platform_value = reason.split(":", 1)[0]
  1022. allow_all_env = (_own_policy_envs(Platform(platform_value)) or (None, None, None))[2]
  1023. logger.error(
  1024. "Refusing to start: %s has dm_policy/group_policy set to 'open' "
  1025. "but neither GATEWAY_ALLOW_ALL_USERS nor %s is enabled.", platform_value,
  1026. allow_all_env or "a platform allow-all flag",
  1027. )
  1028. _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason)
  1029. self._request_clean_exit(reason)
  1030. return True
  1031. return False
  1032. @staticmethod
  1033. def _start_register_plugins_relay_hooks() -> None:
  1034. """Plugin discovery, relay registration and shell-hook/webhook registration. Never raises."""
  1035. # Discover plugins before shell hooks (plugin block decisions win ties). Explicit: the gateway
  1036. # lazily imports run_agent, so model_tools' discover_plugins() side-effect may not have run.
  1037. with _log_suppressed(logging.WARNING, "plugin discovery failed at gateway startup", exc_info=True):
  1038. from hermes_cli.plugins import discover_plugins
  1039. from hermes_cli.plugins_cmd_remove import sweep_pending_plugin_deletes
  1040. sweep_pending_plugin_deletes() # trees a previous life still held when they were uninstalled
  1041. discover_plugins()
  1042. # Relay entrypoints share the effective profile opt-out, including when a
  1043. # deployment injects a URL. No URL or explicitly disabled -> no side effects.
  1044. try:
  1045. from gateway.relay import (
  1046. register_relay_adapter, relay_url, self_provision_relay, send_relay_policy
  1047. )
  1048. # Relay self-provision sets GATEWAY_RELAY_* in os.environ BEFORE registration reads them.
  1049. self_provision_relay()
  1050. if register_relay_adapter():
  1051. logger.info("relay adapter registered (connector at %s)", relay_url())
  1052. # Declare the relevance policy to the connector so the SAME behavior governs relay delivery.
  1053. send_relay_policy()
  1054. except Exception:
  1055. logger.warning("relay adapter registration failed at gateway startup", exc_info=True)
  1056. GatewayStartupMixin._register_launch_profile_config_hooks()
  1057. def _start_relay_provision_retry(self) -> None:
  1058. """Keep provisioning in the background when the boot attempt left the relay without
  1059. credentials; cancelled with ``_background_tasks`` at shutdown. Never raises."""
  1060. try:
  1061. from gateway.relay import relay_provision_pending, retry_self_provision
  1062. if relay_provision_pending():
  1063. self._retain_background_task(asyncio.create_task(
  1064. retry_self_provision(self._on_relay_provisioned), name="relay-provision-retry"))
  1065. except Exception:
  1066. logger.warning("relay self-provision retry could not start", exc_info=True)
  1067. def _on_relay_provisioned(self) -> None:
  1068. """Credentials arrived after boot: the reconnect watcher builds every relay adapter from
  1069. the live env, so pull its queued retry forward instead of waiting out the backoff.
  1070. ``retry_now`` keeps it forward if a pass that began without them is still dialing."""
  1071. info = getattr(self, "_failed_platforms", {}).get(Platform.RELAY)
  1072. if info is not None:
  1073. info["next_retry"] = 0.0
  1074. info["retry_now"] = True
  1075. @staticmethod
  1076. def _register_launch_profile_config_hooks() -> None:
  1077. """The launch profile's ``hooks:`` block, registered under ITS runtime scope when multiplexing.
  1078. Startup runs before any turn scope exists and ``get_secret`` fails closed outside a scope
  1079. while multiplexing is on, so the launch profile needs the scope secondaries already get
  1080. (``_start_secondary_profile_adapters``) or its ``secret_env`` targets cannot resolve.
  1081. """
  1082. from agent.secret_scope import is_multiplex_active
  1083. if not is_multiplex_active():
  1084. GatewayStartupMixin._register_config_hooks(
  1085. "shell-hook/webhook registration failed at gateway startup", level=logging.WARNING)
  1086. return
  1087. from gateway.run import _profile_runtime_scope
  1088. from hermes_constants import get_process_hermes_home
  1089. with _profile_runtime_scope(get_process_hermes_home()):
  1090. GatewayStartupMixin._register_config_hooks(
  1091. "shell-hook/webhook registration failed at gateway startup", level=logging.WARNING)
  1092. @staticmethod
  1093. def _register_config_hooks(fail_fmt: str, *fail_args, level: int = logging.DEBUG) -> None:
  1094. """Register declarative shell hooks + outbound webhooks from the CURRENT scope's config.
  1095. The gateway never prompts (register_from_config defaults to ``interactive=False``), even
  1096. when a hidden console makes stdin look like a TTY, so consent must come from the allowlist,
  1097. --accept-hooks, HERMES_ACCEPT_HOOKS, or hooks_auto_accept: true; ``accept_hooks=False`` lets
  1098. register_from_config resolve env + config.
  1099. Never raises (logged at ``level``).
  1100. """
  1101. try:
  1102. from hermes_cli.config import load_config
  1103. from agent.shell_hooks import register_from_config
  1104. from agent.outbound_webhooks import register_from_config as register_outbound_webhooks
  1105. _hooks_cfg = load_config()
  1106. register_from_config(_hooks_cfg, accept_hooks=False)
  1107. register_outbound_webhooks(_hooks_cfg)
  1108. except Exception:
  1109. logger.log(level, fail_fmt, *fail_args, exc_info=True)
  1110. def _recover_secondary_process_checkpoints(self, process_registry) -> int:
  1111. """Replay every SERVED secondary profile's ``processes.json`` under its own scope.
  1112. The launch profile's file was already read by ``recover_from_checkpoint`` above."""
  1113. if not getattr(self.config, "multiplex_profiles", False):
  1114. return 0
  1115. from gateway.run import _multiplex_profile_homes, _profile_runtime_scope
  1116. from hermes_constants import get_hermes_home
  1117. launch_home = get_hermes_home().resolve()
  1118. recovered = 0
  1119. for profile_name, profile_home in _multiplex_profile_homes(self.config):
  1120. if Path(profile_home).resolve() == launch_home:
  1121. continue
  1122. try:
  1123. with _profile_runtime_scope(Path(profile_home), {}):
  1124. recovered += process_registry.recover_from_checkpoint()
  1125. except Exception:
  1126. logger.warning("Process checkpoint recovery for profile %r failed", profile_name, exc_info=True)
  1127. return recovered
  1128. async def _start_recover_previous_run(self) -> None:
  1129. """Plugins, relay, hooks, then crash/clean-exit recovery of processes and sessions."""
  1130. from gateway.run import _hermes_home
  1131. self._start_register_plugins_relay_hooks()
  1132. self._start_relay_provision_retry()
  1133. # Plugins that load later (force re-discovery, install/enable nudge) re-wire live adapters (#87770).
  1134. with _log_suppressed(logging.WARNING, "plugin re-wire subscription failed", exc_info=True):
  1135. from hermes_cli.plugins import get_plugin_manager
  1136. self._subscribe_plugin_rewire(get_plugin_manager())
  1137. self.hooks.discover_and_load()
  1138. # Recover background processes from checkpoint (crash recovery). ``_checkpoint_path`` is
  1139. # scope-relative, so a served secondary's turn wrote ITS home's processes.json; recover each
  1140. # served profile's file under its scope or those processes are never re-adopted.
  1141. with _log_suppressed(logging.WARNING, "Process checkpoint recovery: %s"):
  1142. from tools.process_registry import process_registry
  1143. recovered = process_registry.recover_from_checkpoint()
  1144. recovered += self._recover_secondary_process_checkpoints(process_registry)
  1145. if recovered:
  1146. logger.info("Recovered %s background process(es) from previous run", recovered)
  1147. # The gateway owns delegation state: replay durable completions the previous
  1148. # process never delivered. Explicit here, never at tools import (clients).
  1149. with _log_suppressed(logging.WARNING, "Could not restore async delegation completions: %s"):
  1150. from tools.async_delegation import restore_undelivered_completions
  1151. restore_undelivered_completions(process_registry.completion_queue)
  1152. # Recover the turns the last process left marked (in flight, or reply not yet ledgered).
  1153. # SKIP after a clean exit — the previous process already drained.
  1154. _clean_marker = _hermes_home / ".clean_shutdown"
  1155. if _clean_marker.exists():
  1156. logger.info("Previous gateway exited cleanly — skipping session suspension")
  1157. try:
  1158. discarded = await self._consume_clean_shutdown_marker(_clean_marker)
  1159. except Exception as exc:
  1160. logger.error(
  1161. "Clean-start marker cleanup failed; refusing startup so the "
  1162. "clean-exit receipt cannot mask a later unclean exit: %s", exc,
  1163. )
  1164. raise RuntimeError("clean-start recovery cleanup failed") from exc
  1165. if discarded:
  1166. logger.info("Discarded %d orphan active-turn marker(s) after clean shutdown", discarded)
  1167. else:
  1168. resumed, ledgered = await self._recover_unclean_sessions()
  1169. if resumed + ledgered:
  1170. logger.info(
  1171. "Recovered %d interrupted turn(s) from previous run (%d to resume, %d reply(ies) "
  1172. "owed delivery)", resumed + ledgered, resumed, ledgered,
  1173. )
  1174. # Stuck-loop detection: a session active across 3+ consecutive restarts is auto-suspended.
  1175. with _log_suppressed(logging.DEBUG, "Stuck-loop detection failed: %s"):
  1176. # Auto-suspend it so the user gets a clean slate on the next message. See #7536.
  1177. stuck = self._suspend_stuck_loop_sessions()
  1178. if stuck:
  1179. logger.warning("Auto-suspended %d stuck-loop session(s)", stuck)
  1180. async def _start_prefilter_platforms(self) -> tuple[bool, int, list, list]:
  1181. """Create + wire an adapter per enabled platform (no connects). Returns
  1182. (aborted, enabled_platform_count, multiplex_skipped_platforms, pending_connects)."""
  1183. from gateway.run import _platform_has_bot_credential
  1184. enabled_platform_count = 0
  1185. _multiplex_on = self._multiplex_on()
  1186. _multiplex_skipped_platforms: list[Platform] = []
  1187. _pending_connects = [] # (platform, platform_config, adapter); connected concurrently later
  1188. for platform, platform_config in self.config.platforms.items():
  1189. if await self._abort_startup_if_shutdown_requested():
  1190. return True, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects
  1191. if not platform_config.enabled:
  1192. continue
  1193. # Multiplex: a platform enabled in the shared config.yaml may hold its token only in a
  1194. # secondary profile's .env; an empty primary would queue a reconnect loop that never heals.
  1195. # Starting that primary adapter with an empty token fails immediately and queues an infinite
  1196. # reconnect loop that can never heal (#64674). Secondary profiles still start their own adapters
  1197. # under _profile_runtime_scope with the real token -- skip the empty primary instead of failing
  1198. # loudly.
  1199. if _multiplex_on and not _platform_has_bot_credential(platform, platform_config):
  1200. logger.info(
  1201. "Skipping %s on default profile: no bot credential in this "
  1202. "profile's secrets. Secondary multiplexed profiles that "
  1203. "provide the token will still connect.", platform.value,
  1204. )
  1205. _multiplex_skipped_platforms.append(platform)
  1206. continue
  1207. enabled_platform_count += 1
  1208. adapter = self._create_adapter(platform, platform_config)
  1209. if not adapter:
  1210. # Distinguish between missing builtin deps and missing plugin
  1211. if platform.value in {m.value for m in Platform.__members__.values()}:
  1212. logger.warning("No adapter available for %s", platform.value)
  1213. else:
  1214. logger.warning(
  1215. "No adapter for '%s' -- is the plugin installed? "
  1216. "(platform is enabled in config.yaml but no plugin registered it)", platform.value,
  1217. )
  1218. # Only a platform that can heal on its own is queued for the reconnect watcher; either way
  1219. # flag it so the unserved enabled platform is visible.
  1220. heals = self._adapter_may_heal(platform, platform_config)
  1221. self._mark_adapter_unavailable(platform.value, platform, retrying=heals, needs_attention=True)
  1222. if heals:
  1223. self._failed_platforms[platform] = self._startup_retry_entry(platform, None, platform_config)
  1224. continue
  1225. # Under multiplexing the default profile needs the same whole-handler runtime scope as a
  1226. # secondary (authorization and prompt rendering run before the agent-turn scope).
  1227. self._wire_adapter_handlers(adapter)
  1228. _pending_connects.append((platform, platform_config, adapter))
  1229. return False, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects
  1230. async def _start_connect_pending(self, _pending_connects: list) -> Optional[list]:
  1231. """Connect the pre-filtered adapters concurrently. Returns the raw per-platform results, or
  1232. None when a restart/shutdown aborted startup mid-connect (adapters already torn down)."""
  1233. async def _connect_one_startup(p, p_cfg, adp):
  1234. """Connect a single platform; never let one block the others (#83791)."""
  1235. if await self._abort_startup_if_shutdown_requested(adp, p):
  1236. return (p, adp, p_cfg, "aborted", None)
  1237. logger.info("Connecting to %s...", p.value)
  1238. self._update_platform_runtime_status(
  1239. p.value, platform_state="connecting", error_code=None, error_message=None,
  1240. )
  1241. try:
  1242. ok = await self._connect_initial_adapter_with_timeout(adp, p)
  1243. except Exception as _exc:
  1244. return (p, adp, p_cfg, "exception", _exc)
  1245. return (p, adp, p_cfg, "ok" if ok else "failed", None)
  1246. if not _pending_connects:
  1247. return []
  1248. # Abort-aware concurrent wait: a restart/shutdown mid-connect cancels pending connects, tears down
  1249. # completed ones, and aborts startup.
  1250. _task_map = {
  1251. asyncio.ensure_future(_connect_one_startup(p, c, a)): (p, c, a) for (p, c, a) in _pending_connects
  1252. }
  1253. _pending_tasks = set(_task_map)
  1254. while _pending_tasks:
  1255. _done, _pending_tasks = await asyncio.wait(_pending_tasks, timeout=0.05)
  1256. if _pending_tasks and self._startup_should_abort():
  1257. break
  1258. else:
  1259. return [_t.exception() or _t.result() for _t in _task_map]
  1260. # Settle in-flight connects FIRST so a completed adapter's disconnect cannot unblock a sibling.
  1261. for _t in _pending_tasks:
  1262. _t.cancel()
  1263. await asyncio.gather(*_pending_tasks, return_exceptions=True)
  1264. # Then tear down adapters whose connect succeeded — never registered, so stop() won't reach them.
  1265. _connected_ok = [
  1266. _t for _t in _task_map
  1267. if _t not in _pending_tasks and not _t.cancelled() and _t.exception() is None and _t.result()[3] == "ok"
  1268. ]
  1269. for _t in [*_pending_tasks, *_connected_ok]:
  1270. await self._startup_teardown_adapter(_task_map[_t][2], _task_map[_t][0])
  1271. await self._abort_startup_if_shutdown_requested()
  1272. return None
  1273. def _startup_queue_transient_failure(
  1274. self, platform, adapter, platform_config, message: str, startup_retryable_errors: list
  1275. ) -> None:
  1276. """Mark a platform ``retrying`` with ``message`` and queue it in ``_failed_platforms``."""
  1277. self._update_platform_runtime_status(
  1278. platform.value, platform_state="retrying", error_code=None, error_message=message,
  1279. )
  1280. startup_retryable_errors.append(f"{platform.value}: {message}")
  1281. self._failed_platforms[platform] = self._startup_retry_entry(platform, adapter, platform_config)
  1282. async def _start_aggregate_connect_results(
  1283. self, _raw: list, startup_retryable_errors: list, startup_nonretryable_errors: list
  1284. ) -> int:
  1285. """Apply connect outcomes to shared state single-threaded (exactly as the original serial
  1286. loop did); returns the connected adapter count."""
  1287. connected_count = 0
  1288. for _item in _raw:
  1289. if isinstance(_item, Exception):
  1290. # Unexpected escape from _connect_one_startup (shouldn't happen); log and skip.
  1291. logger.error("Unexpected startup connect error: %s", _item)
  1292. continue
  1293. platform, adapter, platform_config, outcome, exc = _item
  1294. if outcome == "aborted":
  1295. continue
  1296. if outcome == "exception":
  1297. logger.error("\u2717 %s error: %s", platform.value, exc)
  1298. # An adapter that raised mid-connect may still hold a ClientSession/subprocess; treat
  1299. # unexpected exceptions as transient and queue for retry.
  1300. await self._safe_adapter_disconnect(adapter, platform)
  1301. self._startup_queue_transient_failure(
  1302. platform, adapter, platform_config, str(exc), startup_retryable_errors
  1303. )
  1304. continue
  1305. if outcome == "ok":
  1306. self._publish_primary_adapter(platform, adapter)
  1307. connected_count += 1
  1308. # connect() may return True on a degraded (unconfirmed) receive path; don't stamp "connected".
  1309. _degraded = adapter.send_path_degraded
  1310. self._update_platform_runtime_status(
  1311. platform.value, platform_state="retrying" if _degraded else "connected", error_code=None,
  1312. error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None,
  1313. )
  1314. logger.info("\u2713 %s connected%s", platform.value, " (degraded)" if _degraded else "")
  1315. continue
  1316. # outcome == "failed"
  1317. logger.warning("\u2717 %s failed to connect", platform.value)
  1318. # A failed connect() may have allocated ClientSessions / poll tasks / subprocesses.
  1319. await self._safe_adapter_disconnect(adapter, platform)
  1320. if not adapter.has_fatal_error:
  1321. # No fatal error info means likely a transient issue -- queue for retry
  1322. self._startup_queue_transient_failure(
  1323. platform, adapter, platform_config, "failed to connect", startup_retryable_errors
  1324. )
  1325. continue
  1326. # A live foreign token holder is an ownership conflict, not a blip: retryable only for
  1327. # MID-RUN reconnects; at startup route it non-retryable so the gateway exits 78, not deaf.
  1328. _conflict = is_global_startup_conflict(adapter.fatal_error_code)
  1329. if _conflict:
  1330. self._startup_ownership_conflict = True
  1331. _retryable = adapter.fatal_error_retryable and not _conflict
  1332. self._update_platform_runtime_status(
  1333. platform.value, platform_state="retrying" if _retryable else "fatal",
  1334. error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message,
  1335. )
  1336. target = startup_retryable_errors if _retryable else startup_nonretryable_errors
  1337. target.append(f"{platform.value}: {adapter.fatal_error_message}")
  1338. if _retryable:
  1339. self._failed_platforms[platform] = self._startup_retry_entry(
  1340. platform, adapter, platform_config, queued=False
  1341. )
  1342. return connected_count
  1343. def _startup_fail_fatal_config(self, reason: str) -> None:
  1344. """Record a fatal-config startup failure (exit 78) and request a clean exit."""
  1345. from gateway.run import _write_runtime_status_quiet
  1346. _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason)
  1347. self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE
  1348. self._request_clean_exit(reason)
  1349. self._startup_restore_in_progress = False
  1350. async def _start_secondary_profiles(
  1351. self, connected_count: int, _multiplex_skipped_platforms: list
  1352. ) -> tuple[bool, int]:
  1353. """Bring up multiplexed secondary-profile adapters. Returns (aborted, connected_count)."""
  1354. from gateway.run import MultiplexConfigError
  1355. from tools.process_registry import process_registry as _pr
  1356. # The launch profile's durable completions replay here, not at import (#123265); the
  1357. # secondaries' ledgers are replayed by _restore_secondary_completion_ledgers below.
  1358. _pr.restore_completions()
  1359. # Secondary-profile adapters connect under their own home + credential scope.
  1360. try:
  1361. connected_count += await self._start_secondary_profile_adapters()
  1362. except MultiplexConfigError as e:
  1363. # Invalid multiplexer config — abort cleanly rather than run a half-wired gateway.
  1364. logger.error("Gateway multiplexer config error: %s", str(e))
  1365. self._startup_fail_fatal_config(str(e))
  1366. return True, connected_count
  1367. except Exception as e:
  1368. logger.error("Secondary-profile adapter startup failed: %s", e, exc_info=True)
  1369. finally:
  1370. # Startup authority is one phase: from here on every adapter retry is non-evicting.
  1371. self._platform_lock_takeover_on_start = False
  1372. # A platform skipped on the primary should have been picked up by a secondary owning the token;
  1373. # if none did, the platform is enabled in config.yaml yet silently unserved — surface it loudly so
  1374. # the operator sees a config problem instead of a quiet dead channel (#64674 follow-up).
  1375. for _skipped in _multiplex_skipped_platforms:
  1376. if not any(_skipped in _profile_map for _profile_map in self._profile_adapters.values()):
  1377. logger.warning(
  1378. "%s is enabled but no profile (default or secondary) provided a bot credential for it — "
  1379. "the platform is not being served. Add its token to the profile that should "
  1380. "own it, or disable the platform.", _skipped.value,
  1381. )
  1382. # The mirror image: a SECONDARY enabled shared ingress (WhatsApp/Relay) that only the default can run.
  1383. for _line in self._unserved_shared_ingress_warnings():
  1384. logger.warning(_line)
  1385. return False, connected_count
  1386. def _start_handle_no_connections(
  1387. self, connected_count: int, enabled_platform_count: int, startup_retryable_errors: list,
  1388. startup_nonretryable_errors: list,
  1389. ) -> bool:
  1390. """Log/degrade when a configured platform failed to start; return True when startup must exit.
  1391. The SHARED gate for every platform, not the api_server path: a configured platform that never
  1392. came up must leave a loud, observable trace (ERROR + a ``degraded`` runtime state) even while
  1393. sibling platforms serve normally. One WARNING next to "Gateway running with N platform(s)" let
  1394. a gateway with no API server at all look healthy for half an hour (api_server lost the bind
  1395. race against the gateway it was replacing, and nothing retried).
  1396. """
  1397. from gateway.run import _write_runtime_status_quiet
  1398. if connected_count != 0:
  1399. if startup_nonretryable_errors:
  1400. # Parked fatal failures never heal on their own, so the platforms still serving must
  1401. # not be reported as a healthy run. Retryable peers are deliberately left alone: the
  1402. # reconnect watcher recovers them and their platform entry already says "retrying".
  1403. self._startup_parked_platforms = True
  1404. logger.error(
  1405. "%d configured platform(s) failed to start and are parked (fix the reported error, "
  1406. "then `hermes gateway restart`): %s. The gateway is DEGRADED — it serves the "
  1407. "remaining platform(s) with those unserved.",
  1408. len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors),
  1409. )
  1410. return False
  1411. if startup_nonretryable_errors and not startup_retryable_errors:
  1412. if self._startup_ownership_conflict:
  1413. # Another gateway owns a token/profile: staying up would be a second, deaf owner.
  1414. reason = "; ".join(startup_nonretryable_errors)
  1415. logger.error("Gateway hit a non-retryable startup conflict: %s", reason)
  1416. self._startup_fail_fatal_config(reason)
  1417. return True
  1418. # Every platform is parked, but local clients and cron still need this gateway: exiting 78
  1419. # would take them all down over one unusable bot. Stay up degraded, like the mixed case.
  1420. self._startup_parked_platforms = True
  1421. logger.error(
  1422. "%d configured platform(s) failed to start and are parked (fix the reported error, then "
  1423. "`hermes gateway restart`): %s. The gateway is DEGRADED — it serves local clients and "
  1424. "cron with no messaging platform.",
  1425. len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors),
  1426. )
  1427. return False
  1428. if startup_nonretryable_errors:
  1429. # Mixed (some fatal, some transient): exiting 78 would take the gateway PERMANENTLY down
  1430. # over a blip. Log the fatal side loudly and fall through to the degraded/retry path.
  1431. self._startup_parked_platforms = True
  1432. logger.error(
  1433. # WhatsApp enabled but never paired) while others hit merely transient errors (e.g. Telegram
  1434. # TimedOut during polling startup). Exiting with GATEWAY_FATAL_CONFIG_EXIT_CODE here is
  1435. # wrong in both supervision worlds: under supervisors that honor the exit-78 contract
  1436. # (systemd RestartPreventExitStatus, s6 finish→125 since #51228) the gateway goes
  1437. # PERMANENTLY down over a network blip; under anything else it crash-loops. Either way the
  1438. # retryable platforms never get their retry. Log the fatal side loudly, then fall through to
  1439. # the degraded/retry path below: the reconnect watcher recovers the retryable platforms; the
  1440. # non-retryable ones remain fatal-parked and visible in runtime status.
  1441. "%d platform(s) fatally misconfigured and parked: %s. "
  1442. "Staying alive so retryable platforms can recover.",
  1443. len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors),
  1444. )
  1445. if enabled_platform_count <= 0:
  1446. logger.warning("No messaging platforms enabled.")
  1447. logger.info("Gateway will continue running for cron job execution.")
  1448. return False
  1449. if startup_retryable_errors:
  1450. # All retryable: stay alive (cron runs, watcher recovers) rather than systemd restart-loop.
  1451. logger.warning(
  1452. "Gateway started with no connected platforms — %d platform(s) queued for retry: %s",
  1453. len(startup_retryable_errors), "; ".join(startup_retryable_errors),
  1454. )
  1455. _write_runtime_status_quiet(gateway_state="degraded", exit_reason=None)
  1456. # No adapter for any enabled platform: fleet nodes share one config.yaml but hold a subset of
  1457. # credentials, so degrade gracefully.
  1458. logger.warning(
  1459. # Fall through to the normal "running" state — reconnect watcher takes it from here. In fleet
  1460. # deployments the same config.yaml is shared across nodes that may only have credentials for a
  1461. # subset of platforms. Rather than failing hard, degrade gracefully and allow cron jobs to run
  1462. # (#5196).
  1463. "No adapter could be created for any of the %d configured platform(s). "
  1464. "Check that required dependencies are installed and credentials are set. "
  1465. "Gateway will continue for cron job execution; platforms whose plugin may still register "
  1466. "are queued for background retry.", enabled_platform_count,
  1467. )
  1468. return False
  1469. async def _start_post_connect_services(self, connected_count: int) -> None:
  1470. """Room worker, heartbeat, gateway:startup hook, channel directory, /update notice."""
  1471. from gateway.run import _hermes_home
  1472. try:
  1473. await self._ensure_hosted_room_worker()
  1474. except Exception:
  1475. logger.error(
  1476. "Group Chat worker failed to start; mutating Group Chat commands "
  1477. "will fail closed until supervision recovers it", exc_info=True,
  1478. )
  1479. self._spawn_supervised(self._hosted_room_worker_watcher, "hosted_room_worker")
  1480. self._start_loop_heartbeat_task()
  1481. from gateway.run_heartbeat_restore import restore_heartbeat_watches
  1482. self._start_heartbeat_poller() # Keep retrying even when the first scan is empty.
  1483. await restore_heartbeat_watches(self)
  1484. hook_count = len(self.hooks.loaded_hooks)
  1485. if hook_count:
  1486. logger.info("%s hook(s) loaded", hook_count)
  1487. await self.hooks.emit("gateway:startup", {"platforms": [p.value for p in self.adapters]})
  1488. if connected_count > 0:
  1489. logger.info("Gateway running with %s platform(s)", connected_count)
  1490. # Initial channel directory for send_message name resolution
  1491. with _log_suppressed(logging.WARNING, "Channel directory build failed: %s"):
  1492. from gateway.channel_directory import build_channel_directory
  1493. directory = await build_channel_directory(self.adapters)
  1494. ch_count = sum(len(chs) for chs in directory.get("platforms", {}).values())
  1495. logger.info("Channel directory built: %d target(s)", ch_count)
  1496. # Restarting after a /update still in progress: keep watching so we notify when it finishes.
  1497. notified = await self._send_update_notification()
  1498. if not notified and any(
  1499. (_hermes_home / name).exists()
  1500. for name in (".update_pending.json", ".update_pending.claimed.json")
  1501. ):
  1502. self._schedule_update_notification_watch()
  1503. async def _start_finish_wiring(self, connected_count: int) -> None:
  1504. """Post-connect wiring: services, boot notifications, startup restore, recovered watchers."""
  1505. from gateway.run import _planned_restart_notification_pending, _restart_notification_pending
  1506. from gateway.shutdown_flush import recover_gateway_pending
  1507. await self._start_post_connect_services(connected_count)
  1508. # Let fresh adapters settle before lifecycle sends (helps Discord thread deliveries).
  1509. if connected_count > 0:
  1510. await asyncio.sleep(1.0)
  1511. # Before _send_restart_notification() unlinks the marker: did we boot from a chat /restart?
  1512. # One-shot signal for _is_stale_restart_redelivery.
  1513. if _restart_notification_pending():
  1514. self._booted_from_restart = True
  1515. # Boot-path adapter.send() calls must not pin the inbound restore gate (a Telegram flood-
  1516. # control sleep here once froze every platform).
  1517. # Restart notification, home-channel startup notice, and obligation redelivery all call
  1518. # adapter.send(). Bound them the same way _finish_startup_restore bounds resume turns. See #91969.
  1519. await self._await_startup_boot_sends(
  1520. planned_restart_notification_pending=_planned_restart_notification_pending(),
  1521. )
  1522. # Replay the previous run's pending spool before resume turns and queued inbound write live
  1523. # rows: only this pass tells the store which sessions have spooled rows, and a live row
  1524. # written first would get a lower row id than them for good.
  1525. with _log_suppressed(logging.WARNING, "Pending-message recovery failed: %s"):
  1526. recover_gateway_pending(self)
  1527. # Auto-resume restart-interrupted sessions (ledger-answered ones were cleared above); a failed
  1528. # auto-resume stays visible on the next user message.
  1529. self._schedule_resume_pending_sessions()
  1530. await self._finish_startup_restore()
  1531. from gateway.run_runtime import recover_gateway_native_sessions
  1532. await recover_gateway_native_sessions(self)
  1533. # Surface state.db init failures to messaging platforms before the user loses data.
  1534. # See #88235.
  1535. await self._send_session_db_warning_notifications()
  1536. from gateway.cron_store_notices import install_cron_store_notices
  1537. install_cron_store_notices(self, asyncio.get_running_loop())
  1538. # Resume recovered process watchers. Detach the batch atomically (fresh list, not clear(): a
  1539. # concurrent append during the yield must not be lost); yield every 100 to keep the loop live.
  1540. with _log_suppressed(logging.ERROR, "Recovered watcher setup error: %s"):
  1541. from tools.process_registry import process_registry
  1542. watchers = process_registry.pending_watchers
  1543. process_registry.pending_watchers = []
  1544. for i, watcher in enumerate(watchers):
  1545. self._spawn_supervised(
  1546. lambda w=watcher: self._run_process_watcher(w),
  1547. f"process_watcher:{watcher.get('session_id')}", restart=False,
  1548. )
  1549. logger.info("Resumed watcher for recovered process %s", watcher.get("session_id"))
  1550. if i % 100 == 99:
  1551. await asyncio.sleep(0)
  1552. # Long-lived supervised watchers spawned at the end of start(), in order; supervised name = method
  1553. # name minus the leading underscore.
  1554. _PRE_RECONNECT_WATCHERS = (
  1555. "_session_housekeeping_watcher", "_model_catalog_refresh_watcher", "_session_stall_watcher",
  1556. "_kanban_notifier_watcher", "_kanban_dispatcher_watcher",
  1557. )
  1558. _POST_RECONNECT_WATCHERS = (
  1559. "_handoff_watcher", "_async_delegation_watcher", "_loop_wakeup_watcher", "_profile_reconcile_watcher",
  1560. )
  1561. def _start_spawn_background_watchers(self) -> None:
  1562. """Spawn the long-lived supervised background watchers."""
  1563. for method in self._PRE_RECONNECT_WATCHERS:
  1564. self._spawn_supervised(getattr(self, method), method[1:])
  1565. if self._failed_platforms:
  1566. logger.info(
  1567. "Starting reconnection watcher for %d failed platform(s): %s",
  1568. len(self._failed_platforms), ", ".join(p.value for p in self._failed_platforms),
  1569. )
  1570. # Supervised: an escaping exception is restarted with backoff instead of stranding queued
  1571. # platforms (the ensure hook only runs on a NEW fatal). ``on_spawn`` keeps the handle current.
  1572. self._spawn_reconnect_watcher()
  1573. for method in self._POST_RECONNECT_WATCHERS:
  1574. self._spawn_supervised(getattr(self, method), method[1:])
  1575. from gateway.runtime_downgrade_exit import runtime_downgrade_watcher
  1576. self._spawn_supervised(lambda: runtime_downgrade_watcher(self), "runtime_downgrade_watcher")
  1577. # Scale-to-zero watcher ONLY when opted in, messaging is relay-only/absent, and a wakeUrl exists.
  1578. try:
  1579. if self._scale_to_zero_should_arm():
  1580. logger.info(
  1581. "scale-to-zero: armed (idle timeout %.0fs) — watching for idle",
  1582. self._scale_to_zero_idle_timeout_seconds(),
  1583. )
  1584. self._spawn_supervised(self._scale_to_zero_watcher, "scale_to_zero_watcher")
  1585. else:
  1586. # Say WHY an OPTED-IN instance didn't arm (non-opted stays silent).
  1587. self._log_scale_to_zero_not_armed_reason()
  1588. except Exception:
  1589. logger.debug("scale-to-zero: arm check failed at startup", exc_info=True)
  1590. # Drain-control watcher: reconciles new-turn acceptance with the dashboard's ``.drain_request.json``
  1591. # marker (prior-instantiation markers are ignored via epoch).
  1592. self._spawn_supervised(self._drain_control_watcher, "drain_control_watcher")
  1593. @staticmethod
  1594. async def _start_flush_runtime_status() -> None:
  1595. """Bound startup on the latest diagnostic snapshot, never on filesystem health."""
  1596. try:
  1597. from gateway.status import flush_runtime_status_async
  1598. if not await flush_runtime_status_async(timeout=2.0):
  1599. logger.warning("Timed out flushing final startup runtime status")
  1600. except Exception:
  1601. logger.debug("Final startup runtime-status flush failed", exc_info=True)
  1602. async def start(self) -> bool:
  1603. """Start the gateway and all configured platform adapters."""
  1604. try:
  1605. return await self._start_impl()
  1606. finally:
  1607. # Every startup path (early aborts included) ends here: bound startup on the latest
  1608. # diagnostic snapshot once, instead of flushing at each return.
  1609. await self._start_flush_runtime_status()
  1610. async def _start_impl(self) -> bool:
  1611. logger.info("Starting Hermes Gateway...")
  1612. self._start_install_faulthandler()
  1613. await self._start_log_startup_environment()
  1614. if await self._abort_startup_if_shutdown_requested():
  1615. return True
  1616. if self._start_check_access_policy():
  1617. return True
  1618. await self._start_recover_previous_run()
  1619. # Serialize startup restore against inbound: adapters receive as soon as they connect, so inbound
  1620. # queues until every synthetic resume turn has finished.
  1621. self._startup_restore_in_progress = True
  1622. self._startup_restore_queue = []
  1623. self._startup_restore_tasks = []
  1624. # Fresh boot: the gate opens while the turn machinery is still cold (skeleton prompts). Warm NOW
  1625. # to overlap the connects; _finish_startup_restore awaits it (bounded).
  1626. self._start_startup_warmup()
  1627. startup_nonretryable_errors: list[str] = []
  1628. startup_retryable_errors: list[str] = []
  1629. self._startup_parked_platforms = False # fresh boot: no platform has failed yet
  1630. self._startup_ownership_conflict = False
  1631. (
  1632. _aborted, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects
  1633. ) = await self._start_prefilter_platforms()
  1634. if _aborted:
  1635. return True
  1636. if await self._abort_startup_if_shutdown_requested():
  1637. return True
  1638. _raw = await self._start_connect_pending(_pending_connects)
  1639. if _raw is None:
  1640. return True
  1641. connected_count = await self._start_aggregate_connect_results(
  1642. _raw, startup_retryable_errors, startup_nonretryable_errors
  1643. )
  1644. if await self._abort_startup_if_shutdown_requested():
  1645. return True
  1646. _aborted, connected_count = await self._start_secondary_profiles(
  1647. connected_count, _multiplex_skipped_platforms
  1648. )
  1649. if _aborted:
  1650. return True
  1651. if self._start_handle_no_connections(
  1652. connected_count, enabled_platform_count, startup_retryable_errors, startup_nonretryable_errors
  1653. ):
  1654. return True
  1655. if await self._abort_startup_if_shutdown_requested():
  1656. return True
  1657. self.delivery_router.adapters = self.adapters
  1658. self._wire_teams_pipeline_runtime()
  1659. self._running = True
  1660. self._install_plugin_message_injector()
  1661. # A boot that could not start every configured platform is not a normal run: stamp ``degraded``
  1662. # so ``gateway status`` / /api/status / the health snapshot surface it, instead of only a log
  1663. # line next to "Gateway running with N platform(s)".
  1664. self._update_runtime_status(self._serving_state())
  1665. await self._start_finish_wiring(connected_count)
  1666. self._start_spawn_background_watchers()
  1667. from hermes_cli.observability.shared_metrics_startup import record_process_ready
  1668. record_process_ready("gateway_boot", background=True)
  1669. logger.info("Press Ctrl+C to stop")
  1670. return True
  1671. @dataclasses.dataclass
  1672. class _HandoffDestination:
  1673. """Resolved destination for one handoff row."""
  1674. platform: Platform
  1675. platform_name: str
  1676. transport: Any
  1677. home: Any
  1678. home_chat_id: str
  1679. effective_thread_id: Optional[str]
  1680. source: SessionSource
  1681. handoff_config: Any
  1682. def _handoff_resolve_scope(self, profile_name: Optional[str]):
  1683. """Return (config, adapters) for the profile that queued the handoff. For a secondary
  1684. profile the watcher already entered _profile_runtime_scope, so a fresh load resolves THAT
  1685. profile's config; fail closed — self.config would deliver to the WRONG chat."""
  1686. from gateway.run import load_gateway_config
  1687. if not profile_name or profile_name == "default":
  1688. return self.config, self.adapters
  1689. secondary = (self._profile_adapters or {}).get(profile_name)
  1690. if not secondary:
  1691. raise RuntimeError(f"profile '{profile_name}' has no live adapters in this gateway")
  1692. try:
  1693. return load_gateway_config(), secondary
  1694. except Exception as exc:
  1695. logger.error(
  1696. "Handoff: could not load config for profile %s; failing the handoff instead of "
  1697. "delivering via the primary's config", profile_name, exc_info=True,
  1698. )
  1699. raise RuntimeError(f"could not load config for profile '{profile_name}': {exc}") from exc
  1700. async def _handoff_resolve_destination(
  1701. self, row: dict[str, Any], profile_name: Optional[str]
  1702. ) -> GatewayStartupMixin._HandoffDestination:
  1703. """Resolve platform, transport, home channel, thread and destination source for a row."""
  1704. from gateway.delivery import resolve_delivery_transport
  1705. cli_session_id = row["id"]
  1706. platform_name = (row.get("handoff_platform") or "").strip().lower()
  1707. if not platform_name:
  1708. raise RuntimeError("handoff_platform is empty")
  1709. try:
  1710. platform = Platform(platform_name)
  1711. except (ValueError, KeyError):
  1712. raise RuntimeError(f"unknown platform '{platform_name}'")
  1713. handoff_config, handoff_adapters = self._handoff_resolve_scope(profile_name)
  1714. # Alias-aware transport: a relay-fronted gateway registers ONE Platform.RELAY adapter fronting
  1715. # N logical platforms, so a literal adapters.get() would miss a deliverable one.
  1716. transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters)
  1717. if not transport:
  1718. raise RuntimeError(f"platform '{platform_name}' is not active in this gateway")
  1719. home = handoff_config.get_home_channel(platform)
  1720. if not home or not home.chat_id:
  1721. raise RuntimeError(
  1722. f"no home channel configured for {platform_name}; run /sethome on the desired chat first"
  1723. )
  1724. home_chat_id = str(home.chat_id)
  1725. # Fresh thread for the handoff's own scrollback; None when unsupported or creation failed.
  1726. cli_title = row.get("title") or cli_session_id[:8]
  1727. try:
  1728. new_thread_id = await transport.adapter.create_handoff_thread(
  1729. home_chat_id, t("gateway.startup.handoff_thread_title", title=cli_title),
  1730. )
  1731. except Exception as exc:
  1732. logger.debug("Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True)
  1733. new_thread_id = None
  1734. effective_thread_id = new_thread_id or (str(home.thread_id) if home.thread_id else None)
  1735. # Telegram private-chat DM topics use the DM-topic source shape (user_id == chat_id) so the
  1736. # synthetic turn binds the same key later inbound turns arrive on (`dm`, not `thread`).
  1737. is_telegram_private_chat = (
  1738. platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(home_chat_id)
  1739. )
  1740. is_thread = bool(new_thread_id) and not is_telegram_private_chat
  1741. chat_type = "thread" if is_thread else "dm"
  1742. scope_id = None
  1743. if platform == Platform.TELEGRAM and not is_telegram_private_chat:
  1744. # The Telegram adapter keys forum-topic replies ``group:<chat>:<topic>`` (never
  1745. # ``thread``); bind the handoff on the same slot.
  1746. chat_type = "group"
  1747. if platform == Platform.SLACK:
  1748. # Slack keys a thread reply on the parent channel's type ("dm" for a D… channel, else
  1749. # "group") plus the workspace id — never on a "thread" slot. Mirror the adapter's inbound
  1750. # source shape or the first reply after a restart lands on a different key (#111896).
  1751. chat_type = "dm" if home_chat_id.startswith("D") else "group"
  1752. scope_for_chat = getattr(transport.adapter, "scope_id_for_chat", None)
  1753. scope_id = home.scope_id or (scope_for_chat(home_chat_id) if callable(scope_for_chat) else None)
  1754. if platform == Platform.MATRIX:
  1755. # Matrix likewise keys an in-thread reply on the ROOM's type (#112918); the adapter's
  1756. # get_chat_info reports "dm"/"group" for the home room.
  1757. chat_type = "dm" if await self._handoff_home_is_dm(transport.adapter, home_chat_id) else "group"
  1758. # Discord builds in-thread messages with ``chat_id == thread id``: key on the thread's OWN id.
  1759. dest_source = SessionSource(
  1760. platform=platform,
  1761. chat_id=str(effective_thread_id) if (
  1762. is_thread and platform == Platform.DISCORD and effective_thread_id
  1763. ) else home_chat_id,
  1764. chat_name=home.name,
  1765. chat_type=chat_type,
  1766. user_id=home_chat_id if is_telegram_private_chat else "system:handoff",
  1767. user_name="Handoff", thread_id=effective_thread_id, profile=profile_name,
  1768. scope_id=scope_id,
  1769. )
  1770. # The handoff turn answers through the queuing profile's own bot; naming it keeps the
  1771. # turn's authorization home and the saved receiving bot on that profile, not a primary guess.
  1772. self._canonicalize(dest_source, transport_profile=profile_name)
  1773. return self._HandoffDestination(
  1774. platform=platform, platform_name=platform_name, transport=transport, home=home,
  1775. home_chat_id=home_chat_id, effective_thread_id=effective_thread_id, source=dest_source,
  1776. handoff_config=handoff_config,
  1777. )
  1778. @staticmethod
  1779. async def _handoff_home_is_dm(adapter, home_chat_id: str) -> bool:
  1780. """Whether the home chat is a DM by the adapter's own ``get_chat_info`` (``type: "dm"``);
  1781. unknown/failed → not a DM, the common handoff target."""
  1782. try:
  1783. info = await adapter.get_chat_info(home_chat_id)
  1784. except Exception:
  1785. logger.debug("Handoff: get_chat_info failed for %s", home_chat_id, exc_info=True)
  1786. return False
  1787. return isinstance(info, dict) and info.get("type") == "dm"
  1788. def _handoff_session_key(self, dest, profile_name: Optional[str]) -> str:
  1789. """Destination session_key by the adapters' own rules. Thread keys omit user_id so the next
  1790. message shares it. Namespaced to the queuing profile (else a multiplexed gateway builds
  1791. ``agent:main:...`` while the profile's adapter routes on ``agent:<profile>:...``); the store
  1792. resolver is only the root fallback. The isinstance check is load-bearing: a Mock store returns
  1793. a truthy MagicMock."""
  1794. platform_cfg = dest.handoff_config.platforms.get(dest.platform)
  1795. extra = platform_cfg.extra if platform_cfg else {}
  1796. handoff_profile = profile_name if (profile_name and profile_name != "default") else None
  1797. if handoff_profile is None:
  1798. try:
  1799. store = getattr(self.async_session_store, "_store", self.async_session_store)
  1800. resolver = getattr(store, "_resolve_profile_for_key", None)
  1801. # Resolve the bound text channel's channel_prompt so voice input gets the same per-channel
  1802. # context as typed messages (#50149).
  1803. if callable(resolver):
  1804. resolved = resolver(dest.source)
  1805. if isinstance(resolved, str) and resolved.strip():
  1806. handoff_profile = resolved
  1807. except Exception:
  1808. logger.debug("Handoff: could not resolve profile namespace", exc_info=True)
  1809. return build_session_key(
  1810. dest.source, group_sessions_per_user=extra.get("group_sessions_per_user", True),
  1811. thread_sessions_per_user=extra.get("thread_sessions_per_user", False), profile=handoff_profile,
  1812. )
  1813. async def _handoff_repoint(self, dest, session_key: str, cli_session_id: str) -> None:
  1814. """Point the destination route at the handed-off conversation; ``switch_session`` ends the
  1815. route's prior session and reopens this one. A handoff carries the conversation the user is
  1816. looking at: a local chat's latest ``/new`` segment, then that segment's compression tip
  1817. (``handoff_head``), so its whole chain moves onto the lane together with the local receipt
  1818. that guards it, and a viewer still holding it open (the Desktop that ran /handoff) follows
  1819. it (#143286)."""
  1820. from gateway.session_route_recovery import conversation_head, follow_hand_off
  1821. authority, local_owner, target = await conversation_head(self, cli_session_id)
  1822. # Ensure a session_store entry exists for this key; switch_session then re-points it.
  1823. await self.async_session_store.get_or_create_session(dest.source)
  1824. switched = await self.async_session_store.switch_session(session_key, target, preserve_prompt_pin=False)
  1825. if switched is None:
  1826. raise RuntimeError(f"could not switch session key {session_key} → {target}")
  1827. if authority is not None:
  1828. follow_hand_off(authority, dest.source, session_key, authority.logical_owner(target), local_owner)
  1829. async def _process_handoff(self, row: dict[str, Any], profile_name: Optional[str] = None) -> None:
  1830. """Execute one handoff row; raises on failure (caller marks failed). ``profile_name`` (None =
  1831. root) is the profile whose store queued it — load-bearing under multiplex: secondaries live in
  1832. ``_profile_adapters`` and the key must be namespaced ``agent:<profile>:...`` or nobody reads it."""
  1833. cli_session_id = row["id"]
  1834. dest = await self._handoff_resolve_destination(row, profile_name)
  1835. session_key = self._handoff_session_key(dest, profile_name)
  1836. await self._handoff_repoint(dest, session_key, cli_session_id)
  1837. # Evict the cached AIAgent (rebuild against the CLI session_id, like /resume) and clear stale
  1838. # running-agent state so the synthetic turn isn't queued behind it.
  1839. self._evict_cached_agent(session_key)
  1840. self._release_running_agent_state(session_key)
  1841. cli_title = row.get("title") or cli_session_id[:8]
  1842. synthetic_event = MessageEvent(
  1843. text=(
  1844. f"[Session was just handed off from CLI (\"{cli_title}\") to this "
  1845. f"channel. The full prior conversation history is loaded above. "
  1846. f"Briefly confirm you're working here and summarize what we were "
  1847. f"working on, so the user can continue from this device.]"
  1848. ),
  1849. source=dest.source,
  1850. internal=True,
  1851. )
  1852. logger.info(
  1853. "Handoff: dispatching synthetic turn for CLI session %s → %s "
  1854. "(home=%s, thread=%s, session_key=%s)",
  1855. cli_session_id, dest.platform_name, dest.home.chat_id, dest.effective_thread_id, session_key,
  1856. )
  1857. # Inline _handle_message keeps success/failure observable (handle_message would detach it).
  1858. response_text = await self._handle_message(synthetic_event)
  1859. if not response_text:
  1860. # Streaming may have delivered inline; the agent ran without raising — success.
  1861. return
  1862. # Reply into the new thread (else the home channel) via the resolved transport, so a relay-fronted
  1863. # logical platform is stamped on the outbound frame.
  1864. send_metadata = {"thread_id": dest.effective_thread_id} if dest.effective_thread_id else None
  1865. try:
  1866. result = await dest.transport.send(
  1867. dest.platform, str(dest.home.chat_id), response_text, send_metadata,
  1868. )
  1869. except Exception as exc:
  1870. raise RuntimeError(f"adapter.send failed: {exc}") from exc
  1871. if not getattr(result, "success", True):
  1872. raise RuntimeError(f"adapter.send failed: {_send_error(result)}")