run_adapters.py 104 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806
  1. """Adapter connect/disconnect, fatal-error recovery, reconnect watcher and multiplex profile
  2. adapters for GatewayRunner (mixin bound via the MRO).
  3. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), so
  4. ``patch("gateway.run.X")`` keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import logging
  8. from typing import TYPE_CHECKING
  9. import asyncio
  10. import contextlib
  11. from contextlib import suppress
  12. import functools
  13. import os
  14. import time
  15. import weakref as _weakref
  16. from agent.async_utils import consume_detached_task_result
  17. from contextvars import Context
  18. from datetime import datetime, timedelta, timezone, UTC
  19. from gateway.config import (
  20. ON_ALL_ADAPTERS_DOWN_POLICIES,
  21. SHARED_LISTENER_MIRROR_PLATFORMS,
  22. Platform,
  23. platform_binds_port as _platform_binds_port,
  24. )
  25. from gateway.platforms.base import BasePlatformAdapter
  26. from gateway.platforms.helpers import carry_inbound_dedup, hand_over_held_inbound
  27. from gateway.run_adapters_recovery import (
  28. GatewaySecondaryReconnectMixin,
  29. _adapter_unavailable_message,
  30. )
  31. from gateway.run_shutdown import _log_suppressed
  32. from gateway.session import SessionSource
  33. from hermes_cli.observability.shared_metrics_gateway import record_platform_connect, record_platform_disconnect
  34. from pathlib import Path
  35. from typing import Any, Awaitable, Callable, Dict, Optional
  36. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  37. from gateway.run import GatewayRunner
  38. from gateway.run_turn_runner import TurnRunner
  39. # Log-record parity with the origin module.
  40. logger = logging.getLogger("gateway.run")
  41. _UNSET = object() # "no per-profile human_delay snapshot": fall back to the primary's value
  42. class _UnresolvedProfileHome:
  43. """A NAMED routed profile whose home does not resolve — never the same thing as ``None``
  44. ("this body is the launch profile's own work"). Overloading ``None`` for both let an inbound
  45. message on a secondary's bot run with the LAUNCH profile's ``.env`` and frozen env."""
  46. __slots__ = ()
  47. def __repr__(self) -> str: # log/diagnostic readability
  48. return "<unresolved profile home>"
  49. UNRESOLVED_PROFILE_HOME = _UnresolvedProfileHome()
  50. class GatewayAdapterLifecycleMixin(GatewaySecondaryReconnectMixin):
  51. """Adapter lifecycle: connect/teardown, fatal recovery, reconnect watcher, multiplex profiles."""
  52. @staticmethod
  53. async def _wait_or_detach(task: asyncio.Future, timeout: float) -> bool:
  54. """Wait up to ``timeout`` for ``task``; on deadline (or our own cancellation) detach it. Not
  55. ``asyncio.wait_for``: that WAITS for the cancelled child, so a connect()/close() swallowing
  56. ``CancelledError`` blocks recovery forever. True if it finished in time."""
  57. done: set = set()
  58. try:
  59. done, _pending = await asyncio.wait({task}, timeout=timeout)
  60. finally:
  61. if task not in done: # timed out, or our own cancellation
  62. task.cancel()
  63. task.add_done_callback(consume_detached_task_result)
  64. return task in done
  65. async def _await_adapter_cleanup_with_timeout(self, awaitable: Awaitable[Any], timeout: float) -> bool:
  66. """Await adapter cleanup with a detach-on-deadline bound; True when it completed."""
  67. if timeout <= 0:
  68. await awaitable
  69. return True
  70. task = asyncio.ensure_future(awaitable)
  71. if not await self._wait_or_detach(task, timeout):
  72. return False
  73. await task
  74. return True
  75. async def _safe_adapter_disconnect(self, adapter, platform) -> None:
  76. """Call adapter.disconnect() defensively (bounded, never raises, tolerates partial-init state):
  77. after a failed connect() partial resources (ClientSession, poll tasks, subprocesses) leak."""
  78. timeout = self._adapter_disconnect_timeout_secs()
  79. label = platform.value if platform is not None else "adapter"
  80. with _log_suppressed(logging.DEBUG, "Defensive %s disconnect after failed connect raised: %s", label):
  81. if not await self._await_adapter_cleanup_with_timeout(adapter.disconnect(), timeout):
  82. logger.warning(
  83. "Timed out after %.1fs while disconnecting %s adapter; continuing shutdown",
  84. timeout, label,
  85. )
  86. async def _bounded_adapter_teardown(self, adapter, platform, *, profile: Optional[str] = None) -> None:
  87. """Tear down one adapter on the shutdown path with bounded awaits (never raises). Unbounded,
  88. a half-dead transport stalls past systemd's ``TimeoutStopSec``; the SIGKILL skips ``atexit``
  89. PID-file cleanup and the next start dies with "PID file race lost".
  90. Both ``cancel_background_tasks()`` and ``disconnect()`` can block indefinitely when a platform's
  91. network state is half-dead (e.g. a wedged Feishu/Lark WebSocket thread waiting on I/O). See #14128.
  92. """
  93. timeout = self._adapter_disconnect_timeout_secs()
  94. suffix = f" (profile: {profile})" if profile else ""
  95. started_at = time.monotonic()
  96. try:
  97. if not await self._await_adapter_cleanup_with_timeout(adapter.cancel_background_tasks(), timeout):
  98. logger.warning(
  99. "✗ %s background-task cancel timed out after %.1fs - forcing continue%s",
  100. platform.value, timeout, suffix,
  101. )
  102. except Exception as e:
  103. logger.debug("✗ %s background-task cancel error%s: %s", platform.value, suffix, e)
  104. with _log_suppressed(
  105. logging.ERROR, "✗ %s disconnect error after %.2fs%s: %s",
  106. platform.value, time.monotonic() - started_at, suffix,
  107. ):
  108. if await self._await_adapter_cleanup_with_timeout(adapter.disconnect(), timeout):
  109. logger.info(
  110. "✓ %s disconnected (%.2fs)%s", platform.value, time.monotonic() - started_at, suffix,
  111. )
  112. else:
  113. logger.warning(
  114. "✗ %s disconnect timed out after %.1fs - forcing continue%s",
  115. platform.value, timeout, suffix,
  116. )
  117. @staticmethod
  118. def _env_timeout_override(name: str) -> Optional[float]:
  119. """Non-negative float from env var ``name``; None when unset or unparseable (warned)."""
  120. raw = os.getenv(name, "").strip()
  121. if not raw:
  122. return None
  123. try:
  124. return max(0.0, float(raw))
  125. except ValueError:
  126. logger.warning("Ignoring invalid %s=%r", name, raw)
  127. return None
  128. def _adapter_disconnect_timeout_secs(self) -> float:
  129. """Return the per-adapter disconnect timeout used during shutdown."""
  130. from gateway.run import _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT
  131. override = self._env_timeout_override("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT")
  132. return _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT if override is None else override
  133. def _platform_connect_timeout_secs(self, platform=None, *, initial: bool = False) -> float:
  134. """Per-platform connect timeout. Telegram's full 180s is NOT spent at cold start (it would
  135. hold the gateway out of ``running``); the watcher retries with the full budget.
  136. ``initial=True`` marks the cold-start connect awaited before the gateway reaches ``running``. The
  137. cold-start wait is capped and the platform is handed to the reconnect watcher, which retries with
  138. the full budget (and ``is_reconnect=True``, preserving the offline update queue — #46621).
  139. """
  140. from gateway.run import (
  141. _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT, _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT,
  142. _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT,
  143. )
  144. override = self._env_timeout_override("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT")
  145. if override is not None:
  146. return override
  147. if platform != Platform.TELEGRAM:
  148. return _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT
  149. return _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT if initial else _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT
  150. async def _connect_adapter_with_timeout(
  151. self, adapter, platform, *, is_reconnect: bool = False, initial: bool = False
  152. ) -> bool:
  153. """Connect with a bound so one platform can't block others. ``is_reconnect``: cold boot
  154. drops the stale server-side queue, a reconnect keeps it. ``initial``: capped budget.
  155. ``is_reconnect`` is forwarded to ``adapter.connect()`` so platform adapters can distinguish a cold
  156. first boot (drop any stale server-side queue) from a watcher reconnect after a prolonged outage
  157. (preserve the queue so messages sent during the outage are delivered rather than silently dropped —
  158. #46621).
  159. ``initial`` selects the capped cold-start budget for platforms whose full connect budget is too long
  160. to spend before the gateway reaches ``running`` (#85993 — Telegram's 180s).
  161. """
  162. try:
  163. ok = await self._connect_adapter_bounded(adapter, platform, is_reconnect=is_reconnect, initial=initial)
  164. except Exception as exc:
  165. record_platform_connect(adapter, platform, is_reconnect=is_reconnect, ok=False, exc=exc)
  166. raise
  167. record_platform_connect(adapter, platform, is_reconnect=is_reconnect, ok=bool(ok))
  168. return ok
  169. async def _connect_adapter_bounded(self, adapter, platform, *, is_reconnect: bool, initial: bool) -> bool:
  170. timeout = self._platform_connect_timeout_secs(platform, initial=initial)
  171. if timeout <= 0:
  172. return await adapter.connect(is_reconnect=is_reconnect)
  173. task = asyncio.ensure_future(adapter.connect(is_reconnect=is_reconnect))
  174. if await self._wait_or_detach(task, timeout):
  175. return bool(await task)
  176. raise TimeoutError(f"{platform.value} connect timed out after {timeout:g}s")
  177. async def _connect_initial_adapter_with_timeout(self, adapter, platform) -> bool:
  178. """Cold-start connect with replace intent visible ONLY during this await, so a later
  179. network recovery can never evict a healthy token holder."""
  180. adapter._platform_lock_takeover_allowed = bool(self._platform_lock_takeover_on_start)
  181. try:
  182. return await self._connect_adapter_with_timeout(adapter, platform, initial=True)
  183. finally:
  184. adapter._platform_lock_takeover_allowed = False
  185. async def _handle_reaction_event(self, ctx: dict[str, Any]) -> None:
  186. """Fan a normalised reaction event out to the HookRegistry; errors never block the adapter."""
  187. event_name = str(ctx.get("event_name") or "reaction:added")
  188. with _log_suppressed(logging.DEBUG, "[Gateway] reaction hook emit failed", exc_info=True):
  189. await self.hooks.emit(event_name, ctx)
  190. async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None:
  191. """React to an adapter failure after startup (retryable → background reconnect queue). Runs
  192. detached: the notification arrives on the failing adapter's own polling task, which the
  193. handler's disconnect can cancel mid-flight, stranding the platform half-handled."""
  194. tasks = getattr(self, "_fatal_handler_tasks", None)
  195. if tasks is None:
  196. tasks = self._fatal_handler_tasks = set()
  197. # shield(): a plain `await task` would tunnel the caller's cancellation into the detached
  198. # task; with shield the caller sees CancelledError and the handler runs to completion.
  199. await asyncio.shield(
  200. self._track_task_in(tasks, asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter)))
  201. )
  202. def _reconnect_queue_entry(
  203. self, platform, adapter, platform_config, *, attempts: int, delay: float, queued: bool = True
  204. ) -> dict:
  205. """Build a ``_failed_platforms`` entry (startup failures and runtime fatals share the shape)."""
  206. now = time.monotonic()
  207. return {
  208. "config": platform_config, "attempts": attempts, "next_retry": now + delay,
  209. **({"queued_at": now} if queued else {}),
  210. "credential_claim": self._adapter_credential_claim(platform, adapter),
  211. "listener_claim": self._adapter_listener_claim(platform, adapter),
  212. # The replaced instance: its dedup caches seed each candidate, its held inbound moves on publish.
  213. "predecessor": adapter,
  214. }
  215. def _queue_retryable_fatal_platform(self, adapter: BasePlatformAdapter) -> bool:
  216. """Queue a retryable fatal adapter for background reconnection (True when newly queued).
  217. Must not await: callers run this BEFORE any disconnect so a wedged close can't strand it.
  218. Idempotent if already queued. See #80598.
  219. """
  220. if not adapter.fatal_error_retryable:
  221. return False
  222. platform_config = self.config.platforms.get(adapter.platform)
  223. if not platform_config:
  224. return False
  225. if adapter.platform in self._failed_platforms:
  226. # Already queued is exactly when the watcher may have died (supervision gave up); without
  227. # this backstop nothing retries and the stranded check treats "queued" as safe.
  228. # Nothing to enqueue -- but "already queued" is precisely the state in which the watcher has had
  229. # time to die, and the enqueue branch below holds the ONLY call to
  230. # _ensure_reconnect_watcher_running(). _spawn_supervised auto-restarts the watcher after a crash
  231. # (#71758), but only _MAX_SUPERVISED_RESTARTS times in rapid succession; past that it logs
  232. # "giving up restarts" and the watcher stays dead forever. _ensure_reconnect_watcher_running is
  233. # the documented backstop for exactly that budget exhaustion (#70344) -- and it was unreachable
  234. # for a platform already in the queue, which is the only kind of platform the watcher can have
  235. # been retrying long enough to exhaust it on. The result is a silent permanent outage: nothing
  236. # retries, and the stranded check in _handle_adapter_fatal_error_detached deliberately treats a
  237. # queued platform as safe, so the process never restarts either (#90386).
  238. self._ensure_reconnect_watcher_running()
  239. return False
  240. self._failed_platforms[adapter.platform] = self._reconnect_queue_entry(
  241. adapter.platform, adapter, platform_config, attempts=0, delay=0.0,
  242. )
  243. logger.info("%s queued for background reconnection", adapter.platform.value)
  244. # Ensure the reconnect watcher is alive — if it died (e.g. from exhausting its restart budget),
  245. # respawn it so queued platforms are not permanently stranded (#70344).
  246. self._ensure_reconnect_watcher_running()
  247. return True
  248. def _on_all_adapters_down(self) -> str:
  249. """Normalized ``GatewayConfig.on_all_adapters_down``: ``"exit"`` (default — a supervising
  250. service manager restarts the process) or ``"stay_alive"`` (launchers with no supervisor,
  251. e.g. the desktop app's direct ``hermes serve`` child, where a failure exit only severs the
  252. UI's websockets and drops in-flight assistant messages; #118080)."""
  253. value = getattr(getattr(self, "config", None), "on_all_adapters_down", None)
  254. return value if value in ON_ALL_ADAPTERS_DOWN_POLICIES else "exit"
  255. async def _handle_adapter_fatal_error_detached(self, adapter: BasePlatformAdapter) -> None:
  256. """Run the fatal handler; a platform left stranded (not reconnected, not queued, not
  257. intentionally disabled) exits the gateway with failure so the service manager restarts it."""
  258. try:
  259. # Outer hard deadline: the stranded check in ``finally`` only runs when we return.
  260. timeout = self._adapter_disconnect_timeout_secs()
  261. if timeout <= 0:
  262. # Outer hard deadline (#80598): even with queue-before-disconnect, a hang anywhere in the
  263. # impl (status write side effects, detach races, etc.) must not leave this task wedged
  264. # forever — the stranded check in ``finally`` only runs when we return.
  265. await self._handle_adapter_fatal_error_impl(adapter)
  266. else:
  267. # Disconnect budget + proportional bookkeeping overhead (tests shrink the timeout).
  268. outer = timeout + min(2.0, max(0.05, timeout))
  269. if not await self._await_adapter_cleanup_with_timeout(
  270. self._handle_adapter_fatal_error_impl(adapter), outer
  271. ):
  272. logger.error(
  273. "Fatal-error handling for %s timed out after %.1fs; "
  274. "ensuring reconnect queue is populated", adapter.platform.value, outer,
  275. )
  276. # Best-effort queue before re-raising: a cancelled fatal handler must not strand a
  277. # retryable platform (#80598).
  278. # Best-effort queue so an unexpected raise mid-handler cannot leave a retryable platform
  279. # permanently deaf (#80598).
  280. self._queue_retryable_fatal_platform(adapter)
  281. except asyncio.CancelledError:
  282. # A cancelled or raising fatal handler must not strand a retryable platform.
  283. self._queue_retryable_best_effort(adapter, "cancellation")
  284. raise
  285. except Exception:
  286. logger.exception("Fatal-error handling for %s raised unexpectedly", adapter.platform.value)
  287. self._queue_retryable_best_effort(adapter, "exception")
  288. finally:
  289. platform = adapter.platform
  290. shutdown_event = getattr(self, "_shutdown_event", None)
  291. if (
  292. adapter.fatal_error_retryable
  293. and platform not in self.adapters
  294. and platform not in getattr(self, "_failed_platforms", {})
  295. and not (shutdown_event is not None and shutdown_event.is_set())
  296. ):
  297. if self._on_all_adapters_down() == "stay_alive":
  298. # No supervisor will revive this process, so exiting only severes the UI's
  299. # connections and drops in-flight assistant messages (#118080). Stay alive and
  300. # hand recovery to the reconnect watcher; the messaging platform stays down
  301. # either way, but cron / api_server / dashboard keep serving.
  302. logger.warning(
  303. "%s adapter was lost without entering the reconnection queue; "
  304. "on_all_adapters_down=stay_alive — gateway staying alive, reconnect "
  305. "watcher owns recovery.", platform.value,
  306. )
  307. self._ensure_reconnect_watcher_running()
  308. else:
  309. logger.error(
  310. "%s adapter was lost without entering the reconnection "
  311. "queue; exiting gateway so the service manager restarts it.", platform.value,
  312. )
  313. self._exit_reason = f"{platform.value} adapter lost without reconnection queue"
  314. self._exit_with_failure = True
  315. await self.stop()
  316. def _queue_retryable_best_effort(self, adapter: BasePlatformAdapter, why: str) -> None:
  317. with _log_suppressed(
  318. logging.DEBUG, "Failed to queue %s after fatal-handler %s",
  319. adapter.platform.value, why, exc_info=True,
  320. ):
  321. self._queue_retryable_fatal_platform(adapter)
  322. async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None:
  323. # Snapshot the slot owner first: a stale notification must not touch a healthy platform.
  324. existing = self.adapters.get(adapter.platform)
  325. if existing is not None and existing is not adapter:
  326. logger.debug(
  327. "Ignoring stale fatal error from a superseded %s adapter instance: %s",
  328. adapter.platform.value, adapter.fatal_error_code or "unknown",
  329. )
  330. return
  331. logger.error(
  332. "Fatal %s adapter error (%s): %s", adapter.platform.value,
  333. adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error",
  334. )
  335. # relay_disabled (credential revoked by opt-out) renders "disabled", not red fatal/retrying.
  336. self._update_platform_runtime_status(
  337. adapter.platform.value,
  338. platform_state=(
  339. "disabled" if adapter.fatal_error_code == "relay_disabled"
  340. else "retrying" if adapter.fatal_error_retryable else "fatal"
  341. ),
  342. error_code=adapter.fatal_error_code,
  343. error_message=adapter.fatal_error_message,
  344. )
  345. if existing is adapter:
  346. record_platform_disconnect(adapter)
  347. # Claim for teardown BEFORE awaiting disconnect(), else a second fatal disconnects it twice.
  348. self.adapters.pop(adapter.platform, None)
  349. self.delivery_router.adapters = self.adapters
  350. # Queue BEFORE any disconnect await: a wedged close() once left platforms permanently deaf.
  351. self._queue_retryable_fatal_platform(adapter)
  352. if existing is adapter:
  353. # Bounded by the shutdown-path timeout so this always returns to the stranded check.
  354. # Queue retryable failures BEFORE any disconnect await (#80598). A half-dead transport can wedge
  355. # native close() (or swallow CancelledError inside it) so the previous "disconnect then queue"
  356. # order left platforms permanently deaf inside a live process even after the network recovered.
  357. # Populate the queue first so the reconnect watcher always has work; teardown is best-effort
  358. # after.
  359. await self._safe_adapter_disconnect(adapter, adapter.platform)
  360. if not self.adapters and not self._failed_platforms:
  361. if adapter.fatal_error_retryable and self._on_all_adapters_down() == "stay_alive":
  362. # No supervising service manager to revive the process (#118080): stay alive and
  363. # keep serving cron / api_server / dashboard while the reconnect watcher owns
  364. # recovery of the lost platform.
  365. logger.warning(
  366. "No connected messaging platforms remain; on_all_adapters_down=stay_alive — "
  367. "gateway staying alive, reconnect watcher owns recovery of %s.",
  368. adapter.platform.value,
  369. )
  370. self._ensure_reconnect_watcher_running()
  371. return
  372. if self._park_last_fatal_platform(adapter):
  373. return
  374. self._exit_reason = adapter.fatal_error_message or "All messaging adapters disconnected"
  375. if adapter.fatal_error_retryable:
  376. self._exit_with_failure = True
  377. logger.error("No connected messaging platforms remain. Shutting down gateway for service restart.")
  378. else:
  379. logger.error("No connected messaging platforms remain. Shutting down gateway cleanly.")
  380. await self.stop()
  381. elif not self.adapters and self._failed_platforms:
  382. # All down but queued: stay alive (cron runs, watcher recovers) rather than restart-loop.
  383. logger.warning(
  384. "No connected messaging platforms remain, but %d platform(s) "
  385. "queued for reconnection — gateway staying alive, watcher will "
  386. "retry in background.", len(self._failed_platforms),
  387. )
  388. def _retain_background_task(self, task: asyncio.Task) -> asyncio.Task:
  389. """Register ``task`` in ``_background_tasks`` (created lazily for bare test runners)."""
  390. tasks = getattr(self, "_background_tasks", None)
  391. if not isinstance(tasks, set):
  392. tasks = self._background_tasks = set()
  393. tasks.add(task)
  394. task.add_done_callback(tasks.discard)
  395. return task
  396. @staticmethod
  397. def _track_task_in(tasks: set, task: asyncio.Task) -> asyncio.Task:
  398. """Register ``task`` in an arbitrary lifecycle set with self-removal on completion."""
  399. tasks.add(task)
  400. task.add_done_callback(tasks.discard)
  401. return task
  402. def _request_clean_exit(self, reason: str) -> None:
  403. self._exit_cleanly = True
  404. self._exit_reason = reason
  405. self._shutdown_event.set()
  406. @staticmethod
  407. def _supervised_backoff(attempt: int) -> float:
  408. """Capped exponential respawn delay (a method so tests can collapse the schedule)."""
  409. return min(60, 2 ** min(attempt, 6))
  410. def _spawn_supervised(
  411. self, coro_factory, name, *, restart=True, _attempt=0, on_spawn=None, on_give_up=None
  412. ):
  413. """Launch a long-lived supervised background task: exceptions a bare ``create_task`` drops
  414. are logged, and it respawns with capped backoff up to ``_MAX_SUPERVISED_RESTARTS`` rapid
  415. failures (counter resets after ``_SUPERVISED_HEALTHY_SECS`` healthy). Fresh ``Context`` per
  416. spawn (an inherited delegated-child marker would make the Kanban dispatcher reject its own
  417. writes). ``on_spawn`` fires on EVERY spawn incl. respawns — handle trackers MUST pass it or a
  418. respawn leaves a stale handle and a SECOND watcher; ``on_give_up(name)`` fires at budget end.
  419. ``on_give_up`` (optional) is invoked with ``name`` when supervision is abandoned — the restart
  420. budget is spent and this task will never be respawned by the supervisor again. Supervision being
  421. finite is correct; having no owner of the invariant afterwards is not. A task that still has queued
  422. work depending on it needs somewhere to hand that fact to, and before this hook existed the only
  423. thing standing between budget exhaustion and a permanent silent outage was a *later, unrelated
  424. event* happening to call ``_ensure_...`` (#90386). This is the supervisor telling its caller "I am
  425. done; the invariant is yours now", which is a thing only the supervisor knows.
  426. """
  427. # Spawn timestamp lets ``_done`` tell a rapid crash-loop from a healthy-run-then-crash.
  428. _started = time.monotonic()
  429. # No create_task kwargs (test doubles mock a narrow signature); Context().run isolates instead.
  430. task = Context().run(lambda: asyncio.create_task(coro_factory()))
  431. # PERMANENT watcher: the scale-to-zero idle check ignores it (else busy forever).
  432. task._hermes_supervised_watcher = True # type: ignore[attr-defined]
  433. self._retain_background_task(task)
  434. if on_spawn is not None:
  435. # Record the live handle NOW so external trackers don't point at a dead prior task.
  436. try:
  437. on_spawn(task)
  438. except Exception: # pragma: no cover - defensive; a tracker must never kill the spawn
  439. logger.debug("on_spawn callback for %s raised", name, exc_info=True)
  440. def _done(t):
  441. self._background_tasks.discard(t)
  442. if t.cancelled():
  443. return
  444. exc = t.exception()
  445. if exc is None:
  446. # Clean return = deliberate shutdown or a self-disabling watcher; NEVER respawn.
  447. return
  448. logger.error("Supervised task %s died: %r", name, exc, exc_info=exc)
  449. if not (restart and self._running):
  450. return
  451. # A healthy run before the crash is a FRESH failure, not a crash-loop: reset the counter.
  452. healthy = time.monotonic() - _started >= self._SUPERVISED_HEALTHY_SECS
  453. effective_attempt = 0 if healthy else _attempt
  454. if effective_attempt >= self._MAX_SUPERVISED_RESTARTS:
  455. logger.error(
  456. "Supervised task %s died %d times in rapid succession "
  457. "(each within %ds of restart) — giving up restarts", name,
  458. effective_attempt, self._SUPERVISED_HEALTHY_SECS,
  459. )
  460. if on_give_up is not None:
  461. try:
  462. on_give_up(name)
  463. except Exception: # pragma: no cover - defensive
  464. logger.debug("on_give_up callback for %s raised", name, exc_info=True)
  465. return
  466. backoff = self._supervised_backoff(effective_attempt)
  467. async def _respawn():
  468. await asyncio.sleep(backoff)
  469. if self._running:
  470. self._spawn_supervised(
  471. coro_factory, name, restart=restart, _attempt=effective_attempt + 1,
  472. on_spawn=on_spawn, on_give_up=on_give_up, # only the LAST give-up matters
  473. )
  474. # The done callback runs in its registration context; isolate the backoff task too.
  475. self._retain_background_task(Context().run(lambda: asyncio.create_task(_respawn())))
  476. task.add_done_callback(_done)
  477. return task
  478. async def _handoff_watcher(self, interval: float = 2.0, drain_timeout: float = 30.0) -> None:
  479. """Process pending CLI→gateway session handoffs from ``state.db``: claim atomically (pending
  480. → running), re-bind the home channel to the CLI session_id, dispatch a synthetic event, mark
  481. ``completed``/``failed``."""
  482. from gateway.run import _async_profile_runtime_scope, _reclaim_stale, _resolve_handoff_watch_scopes
  483. from gateway.run_idle_gates import off_loop_gate, profile_has_pending_handoff
  484. await asyncio.sleep(5) # let platforms connect before dispatching through them
  485. # Does _process_handoff accept the profile argument? Test stand-ins bind a one-arg callable.
  486. try:
  487. import inspect as _inspect
  488. _process_takes_profile = len(_inspect.signature(self._process_handoff).parameters) >= 2
  489. except Exception:
  490. _process_takes_profile = False
  491. # In-flight dispatches by session id: a handoff is a FULL agent turn, so never process inline.
  492. inflight: dict[str, asyncio.Task] = {}
  493. async def _dispatch(row, session_id, session_db, profile_name) -> None:
  494. """Run one claimed handoff to a terminal state, off the poll path."""
  495. try:
  496. await self._process_handoff(*((row, profile_name) if _process_takes_profile else (row,)))
  497. await session_db.complete_handoff(session_id)
  498. except asyncio.CancelledError:
  499. # Leave the row 'running' so the next start's reclaim marks it failed with a clear reason.
  500. raise
  501. except Exception as exc:
  502. logger.warning("Handoff for session %s failed: %s", session_id, exc, exc_info=True)
  503. with _log_suppressed(logging.DEBUG, "Could not record handoff failure", exc_info=True):
  504. await session_db.fail_handoff(session_id, str(exc))
  505. finally:
  506. inflight.pop(session_id, None)
  507. async def _tick(profile_name: Optional[str] = None) -> None:
  508. """One poll of the CURRENTLY-SCOPED store; ``profile_name`` (None = root) routes delivery
  509. to that profile's OWN adapter. A closure, not a method: tests bind ``_handoff_watcher`` onto
  510. a ``SimpleNamespace`` with only ``_session_db``/``_running``/``_process_handoff``."""
  511. session_db = getattr(self, "_session_db", None)
  512. if session_db is None:
  513. return
  514. pending = await session_db.list_pending_handoffs()
  515. for row in pending:
  516. session_id = row.get("id")
  517. if not session_id or session_id in inflight:
  518. continue
  519. if not await session_db.claim_handoff(session_id):
  520. # Another tick or another gateway already claimed it.
  521. continue
  522. # INVARIANT (do not weaken): created inside _profile_runtime_scope but RUNS after it
  523. # exits; it sees the profile scope only because ensure_future copies the Context.
  524. # Positional, not keyword: the watcher's existing unit tests bind a stand-in
  525. # ``_process_handoff(row)`` with no second parameter, and a keyword call would TypeError
  526. # into the failure branch — turning a passing suite into a silent no-op watcher. Arity is
  527. # probed above. It still sees the profile's home and secret scope only because
  528. # ``set_hermes_home_override`` and ``set_secret_scope`` are ContextVar-based — ensure_future
  529. # copies the current Context into the Task. If either seam is ever migrated to a
  530. # thread-local or module global, secondary- profile handoffs silently regress to
  531. # primary-config delivery (the exact bug fixed in #91217) while still recording
  532. # handoff_state='completed'.
  533. inflight[session_id] = asyncio.ensure_future(
  534. _dispatch(row, session_id, session_db, profile_name)
  535. )
  536. # A row still 'running' at startup died mid-dispatch and blocks request_handoff until reclaimed.
  537. def _scope(profile_home): # local: tests bind this watcher onto bare SimpleNamespace runners
  538. return GatewayAdapterLifecycleMixin._async_scope_or_null(_async_profile_runtime_scope, profile_home)
  539. # Multiplex scope resolution walks the filesystem (profiles_to_serve) off the loop, so a
  540. # stalled walk cannot trip the liveness probe — startup reclaim and every tick alike.
  541. for _pname, _phome in await _resolve_handoff_watch_scopes(self):
  542. with _log_suppressed(logging.DEBUG, "Stale-handoff reclaim failed", exc_info=True):
  543. async with _scope(_phome):
  544. await _reclaim_stale(self)
  545. try:
  546. while self._running:
  547. try:
  548. for profile_name, profile_home in await _resolve_handoff_watch_scopes(self):
  549. # Idle gate (run_idle_gates): skip the scope entry when the profile's store
  550. # holds no pending handoff. The root poll (None) is unscoped and stays cheap.
  551. if profile_home is not None and not await off_loop_gate(
  552. self, lambda home=profile_home: profile_has_pending_handoff(home)):
  553. continue
  554. async with _scope(profile_home):
  555. await _tick(profile_name)
  556. except asyncio.CancelledError:
  557. raise
  558. except Exception as exc:
  559. logger.debug("Handoff watcher tick error: %s", exc, exc_info=True)
  560. await asyncio.sleep(interval)
  561. finally:
  562. # Bounded drain: cancelling would strand in-flight rows in 'running'.
  563. pending_tasks = [t for t in inflight.values() if not t.done()]
  564. if pending_tasks:
  565. with _log_suppressed(logging.DEBUG, "Handoff drain raised", exc_info=True):
  566. await asyncio.wait(pending_tasks, timeout=drain_timeout)
  567. for task in pending_tasks:
  568. if not task.done():
  569. task.cancel()
  570. def _on_reconnect_watcher_gave_up(self, name: str = "") -> None:
  571. """Own the reconnect invariant once supervision gives up: while running with queued
  572. platforms, a watcher is live or a bounded respawn is scheduled (no later event can notice
  573. a dead watcher). Slow-tier exhaustion logs loudly; deliberately NOT a process restart.
  574. Before this, the only thing that noticed a dead watcher was a *later fatal error from some other
  575. platform* reaching ``_queue_retryable_fatal_platform``. That is event-coupled recovery: it needs an
  576. event that, by construction, may never come. #81036 moved queue publication ahead of disconnect and
  577. drops the failed adapter from the live map, so once the watcher's budget is spent there may be no
  578. adapter left that can emit the event recovery was waiting on. The platform stays queued, nothing
  579. retries it, and the stranded check in ``_handle_adapter_fatal_error_detached`` treats a queued
  580. platform as safe — so the process is never restarted either.
  581. """
  582. if not getattr(self, "_running", False):
  583. return
  584. if getattr(self, "_failed_platforms", None):
  585. self._schedule_slow_reconnect_watcher_respawn(attempt=0)
  586. else:
  587. # Nothing depends on the watcher; the enqueue path spawns a fresh one when needed.
  588. logger.warning(
  589. "Reconnect watcher supervision exhausted with an empty retry "
  590. "queue — leaving it down until a platform is queued."
  591. )
  592. def _schedule_slow_reconnect_watcher_respawn(self, *, attempt: int) -> None:
  593. """Bounded slow-tier respawn of the reconnect watcher."""
  594. if attempt >= self._MAX_SLOW_WATCHER_RESPAWNS:
  595. logger.error(
  596. "Reconnect watcher could not be kept alive after %d slow respawns; %d platform(s) remain "
  597. "queued and unattended: %s. Manual intervention or a gateway restart is required.",
  598. attempt, len(self._failed_platforms), ", ".join(str(p) for p in self._failed_platforms),
  599. )
  600. return
  601. async def _slow_respawn() -> None:
  602. await asyncio.sleep(self._RECONNECT_WATCHER_SLOW_RETRY_SECS)
  603. task = getattr(self, "_reconnect_watcher_task", None)
  604. if (
  605. not getattr(self, "_running", False)
  606. or not getattr(self, "_failed_platforms", None) # queue drained while waiting
  607. or (task is not None and not task.done()) # a watcher came back; stand down
  608. ):
  609. return
  610. logger.warning(
  611. "Reconnect watcher still down with %d platform(s) queued — slow respawn %d/%d",
  612. len(self._failed_platforms), attempt + 1, self._MAX_SLOW_WATCHER_RESPAWNS,
  613. )
  614. self._spawn_reconnect_watcher(
  615. on_give_up=lambda _name: self._schedule_slow_reconnect_watcher_respawn(attempt=attempt + 1)
  616. )
  617. self._retain_background_task(asyncio.create_task(_slow_respawn()))
  618. def _spawn_reconnect_watcher(self, *, on_give_up=None):
  619. """Launch the reconnect watcher. ``on_spawn`` is load-bearing: without it a supervised
  620. respawn leaves ``_reconnect_watcher_task`` dead and ``_ensure_...`` spawns a second one."""
  621. self._reconnect_watcher_task = self._spawn_supervised(
  622. self._platform_reconnect_watcher, "platform_reconnect_watcher",
  623. on_spawn=lambda t: setattr(self, "_reconnect_watcher_task", t),
  624. on_give_up=on_give_up or self._on_reconnect_watcher_gave_up,
  625. )
  626. return self._reconnect_watcher_task
  627. def _ensure_reconnect_watcher_running(self) -> None:
  628. """Respawn a dead reconnect watcher (called on BOTH _queue_retryable_fatal_platform paths:
  629. the re-fatal of an already-queued platform is the only case that exhausts the budget).
  630. If the tracked reconnect watcher task has died (e.g. from exhausting its restart budget, or a
  631. terminal exception that _spawn_supervised could not recover), respawns it so platforms queued for
  632. reconnection are not permanently stranded. Called from _queue_retryable_fatal_platform on BOTH paths
  633. (#70344, #90386): after a new enqueue, and after a re-fatal for a platform that is already queued --
  634. the latter being the only case in which the watcher can have been retrying long enough to exhaust
  635. its supervised restart budget.
  636. """
  637. task = getattr(self, "_reconnect_watcher_task", None)
  638. if not getattr(self, "_running", False) or (task is not None and not task.done()):
  639. return # not running, or already alive
  640. logger.warning(
  641. "Reconnect watcher task is dead (done=%s) — respawning",
  642. task.done() if task is not None else "N/A",
  643. )
  644. self._spawn_reconnect_watcher()
  645. async def _platform_reconnect_watcher(self) -> None:
  646. """Periodically retry failed platforms: backoff 30s → 300s cap, retryable failures retry
  647. forever (self-heal), non-retryable drop out. Pausing is manual only (``/platform pause``)."""
  648. async def _idle(seconds: int, until_queued: bool = False) -> bool:
  649. """Sleep in 1s steps; False once the runner stops (or, if asked, once work is queued)."""
  650. for _ in range(seconds):
  651. if not self._running:
  652. return False
  653. if until_queued and self._failed_platforms:
  654. break
  655. await asyncio.sleep(1)
  656. return True
  657. await asyncio.sleep(10) # initial delay — let startup finish
  658. while self._running:
  659. if not self._failed_platforms:
  660. if not await _idle(30, until_queued=True):
  661. return
  662. continue
  663. now = time.monotonic()
  664. for platform in list(self._failed_platforms.keys()):
  665. if not self._running:
  666. return
  667. await self._reconnect_failed_platform(platform, now)
  668. if not await _idle(10): # re-check every 10 seconds
  669. return
  670. def _flag_reconnect_needs_attention(
  671. self, platform, info: dict, now: float, *, status_key: Optional[str] = None
  672. ) -> bool:
  673. """Flag NEEDS_ATTENTION (once) past the threshold — a signal, NOT a circuit breaker. The threshold
  674. is the bound profile's ``agent.reconnect_attention_after``: secondaries call this inside their
  675. ``_profile_runtime_scope`` with their ``<profile>:<platform>`` status key. True = flagged now."""
  676. from gateway.run import _reconnect_needs_attention
  677. if info.get("attention_flagged") or not _reconnect_needs_attention(info, now):
  678. return False
  679. info["attention_flagged"] = True
  680. queued_for = now - info.get("queued_at", now)
  681. logger.warning(
  682. "%s has been failing/reconnecting continuously for %.1f hours (%d attempts) — flagging "
  683. "NEEDS_ATTENTION. Retries continue, but this usually means a permanent problem (revoked "
  684. "credentials, missing intents, broken sidecar). Check `hermes status` / `/platform list`.",
  685. status_key or platform.value, queued_for / 3600.0, info.get("attempts", 0),
  686. )
  687. self._update_platform_runtime_status(
  688. status_key or platform.value, platform_state="retrying", needs_attention=True,
  689. retrying_since=(datetime.now(UTC) - timedelta(seconds=queued_for)).isoformat(),
  690. )
  691. return True
  692. def _mark_platform_fatal(self, status_key: str, adapter) -> None:
  693. """Record an adapter's fatal error code/message as ``fatal`` runtime status."""
  694. self._update_platform_runtime_status(
  695. status_key, platform_state="fatal", error_code=adapter.fatal_error_code,
  696. error_message=adapter.fatal_error_message,
  697. )
  698. def _bump_reconnect_backoff(
  699. self, platform, info: dict, attempt: int, error_code, error_message: str
  700. ) -> int:
  701. """Mark the platform retrying and record the failed attempt; returns the backoff applied.
  702. ``retry_now`` still set here was raised DURING this attempt (the pass clears it when it starts):
  703. what the attempt lacked arrived after it began (relay credentials), so the retry stays immediate.
  704. """
  705. from gateway.run import _reconnect_backoff
  706. self._update_platform_runtime_status(
  707. platform.value, platform_state="retrying", error_code=error_code, error_message=error_message,
  708. )
  709. backoff = 0 if info.pop("retry_now", False) else _reconnect_backoff(attempt)
  710. info["attempts"] = attempt
  711. info["next_retry"] = time.monotonic() + backoff
  712. return backoff
  713. def _adapter_may_heal(self, platform, platform_config) -> bool:
  714. """Whether a platform whose ``_create_adapter`` returned None can heal without a config change.
  715. Only an unregistered plugin can (a plugin load can fail transiently). A builtin whose probe fails
  716. (missing deps/creds), a registered plugin returning None, or an empty bot credential needs a config
  717. change: retrying it re-warns forever at the backoff cap (#5196 fleet nodes). Shared by startup and
  718. the reconnect watcher so both classify the same platform the same way."""
  719. from gateway.platform_registry import platform_registry
  720. from gateway.run import _BUILTIN_ADAPTERS, _platform_has_bot_credential
  721. return (
  722. platform not in _BUILTIN_ADAPTERS
  723. and not platform_registry.is_registered(platform.value)
  724. and _platform_has_bot_credential(platform, platform_config)
  725. )
  726. async def _reconnect_failed_platform(self, platform, now: float) -> None:
  727. """One watcher pass for a queued platform: gate, attempt, and record the outcome."""
  728. from gateway.run import _dispose_unused_adapter, _platform_has_bot_credential
  729. info = self._failed_platforms.get(platform)
  730. # None: removed concurrently since the caller's snapshot. Paused needs /platform resume.
  731. if info is None or info.get("paused"):
  732. return
  733. self._flag_reconnect_needs_attention(platform, info, now)
  734. if now < info["next_retry"]:
  735. return # not time yet
  736. platform_config = info["config"]
  737. attempt = info["attempts"] + 1
  738. info.pop("retry_now", None) # this attempt already sees whatever the request was for
  739. # Empty-token primary configs can never reconnect; drop them so multiplex setups
  740. # where a secondary profile owns the bot do not spin forever.
  741. # See #64674.
  742. if not _platform_has_bot_credential(platform, platform_config):
  743. self._drop_from_reconnect_queue(platform, "no bot credential on queued config")
  744. return
  745. logger.info("Reconnecting %s (attempt %d)...", platform.value, attempt)
  746. adapter = None
  747. try:
  748. from gateway.platform_registry import platform_registry
  749. # A re-armed plugin import + register() joins its load deadline: run it off the event loop.
  750. await self._run_in_executor_with_context(platform_registry.get, platform.value)
  751. adapter = self._create_adapter(platform, platform_config)
  752. if not adapter:
  753. if not self._adapter_may_heal(platform, platform_config):
  754. # Became builtin/registered-but-None: a config change is needed, so stop retrying.
  755. self._mark_adapter_unavailable(platform.value, platform, retrying=False, needs_attention=True)
  756. self._drop_from_reconnect_queue(platform, "adapter creation returned None")
  757. return
  758. # Unregistered plugin: keep it queued and re-arm a failed load (capped) for the next tick.
  759. await self._rearm_failed_plugin_load(platform, info)
  760. backoff = self._bump_reconnect_backoff(
  761. platform, info, attempt, "adapter_unavailable", _adapter_unavailable_message(platform),
  762. )
  763. logger.info("Reconnect %s: no adapter yet, next retry in %ds", platform.value, backoff)
  764. return
  765. carry_inbound_dedup(info.get("predecessor"), adapter)
  766. self._wire_adapter_handlers(adapter)
  767. # is_reconnect keeps the server-side update queue so offline-period messages are delivered.
  768. success = await self._connect_adapter_with_timeout(adapter, platform, is_reconnect=True)
  769. if success:
  770. await self._install_reconnected_adapter(platform, adapter)
  771. elif adapter.has_fatal_error and not adapter.fatal_error_retryable:
  772. self._mark_platform_fatal(platform.value, adapter)
  773. logger.warning(
  774. "Reconnect %s: non-retryable error (%s), removing from retry queue",
  775. platform.value, adapter.fatal_error_message,
  776. )
  777. # Never installed on self.adapters: dispose here or its __init__ resources leak ~2 fds each.
  778. # The adapter is about to be dropped from the queue without ever being installed on
  779. # self.adapters, so nothing else will call disconnect() on it. We must dispose it here,
  780. # otherwise the resource owners it constructed in __init__ (ResponseStore for
  781. # APIServerAdapter, etc.) leak 2 fds each. The gateway hits the 2560-fd limit after ~12h of
  782. # failed reconnects at the 300s backoff cap (#37011).
  783. await _dispose_unused_adapter(adapter)
  784. del self._failed_platforms[platform]
  785. else:
  786. # Retryable failures retry at the cap forever (never auto-pause). Same fd-leak dispose.
  787. backoff = self._bump_reconnect_backoff(
  788. platform, info, attempt, adapter.fatal_error_code,
  789. adapter.fatal_error_message or "failed to reconnect",
  790. )
  791. logger.info("Reconnect %s failed, next retry in %ds", platform.value, backoff)
  792. # Same fd-leak concern as the non-retryable branch above: the adapter failed to connect and
  793. # is being thrown away. Without an explicit dispose call, the resources it opened in
  794. # __init__ stay open until the next GC pass — and aiohttp/SQLite handles don't get GC'd
  795. # promptly, so 2 fds/retry leak at 300s backoff cap = ~12 fds/hour (#37011).
  796. await _dispose_unused_adapter(adapter)
  797. # Inbound it held during connect or salvaged in disconnect goes back to the predecessor.
  798. hand_over_held_inbound(adapter, info.get("predecessor"))
  799. except Exception as e:
  800. if adapter is not None:
  801. # An exception escaping connect leaves the adapter in the same unowned state.
  802. await _dispose_unused_adapter(adapter)
  803. if self.adapters.get(platform) is not adapter: # an installed adapter owns its queue
  804. hand_over_held_inbound(adapter, info.get("predecessor"))
  805. # A reconnect exception is transient; keep retrying at the cap rather than auto-pausing.
  806. backoff = self._bump_reconnect_backoff(platform, info, attempt, None, str(e))
  807. logger.warning("Reconnect %s error: %s, next retry in %ds", platform.value, e, backoff)
  808. def _drop_from_reconnect_queue(self, platform, reason: str) -> None:
  809. logger.warning("Reconnect %s: %s, removing from retry queue", platform.value, reason)
  810. del self._failed_platforms[platform]
  811. def _publish_primary_adapter(self, platform, adapter) -> None:
  812. """Register a connected primary adapter and wire voice mode/input (transcription without /voice join)."""
  813. self.adapters[platform] = adapter
  814. if platform == Platform.API_SERVER:
  815. from gateway.session_api_turn import recover_api_turns
  816. recover_api_turns(adapter)
  817. self._sync_voice_mode_state_to_adapter(adapter)
  818. self._bind_voice_input_callback(adapter)
  819. def _schedule_planned_restart_replay(self) -> None:
  820. """Replay the owed planned-restart notice after a reconnect, in the background: notification delivery
  821. must not hold up adapter recovery or other platforms' reconnects."""
  822. from gateway.run import _planned_restart_notification_pending
  823. if _planned_restart_notification_pending():
  824. task = self._retain_background_task(asyncio.create_task(
  825. self._replay_pending_planned_restart_notification(),
  826. ))
  827. task.add_done_callback(self._late_failure_callback("planned-restart notification replay failed"))
  828. async def _install_reconnected_adapter(self, platform, adapter) -> None:
  829. """Publish a freshly reconnected primary adapter and replay what it missed while down."""
  830. self._publish_primary_adapter(platform, adapter)
  831. self.delivery_router.adapters = self.adapters
  832. hand_over_held_inbound(self._failed_platforms.pop(platform).get("predecessor"), adapter)
  833. # connect() returning True does not mean the receive path is confirmed -- Telegram's degraded
  834. # reconnect returns True so the gateway stays up while its own ladder retries. Stamping "connected"
  835. # here would undo the adapter's accurate status.
  836. _degraded = adapter.send_path_degraded
  837. self._update_platform_runtime_status(
  838. platform.value, platform_state="retrying" if _degraded else "connected", error_code=None,
  839. error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None,
  840. needs_attention=False, retrying_since=None,
  841. )
  842. if _degraded:
  843. logger.info("⚠ %s reconnected in degraded mode (receive path not yet confirmed)", platform.value)
  844. else:
  845. logger.info("✓ %s reconnected successfully", platform.value)
  846. self._schedule_planned_restart_replay()
  847. # Responses rejected while down are owned by this live process (startup recovery cannot claim them).
  848. with _log_suppressed(
  849. logging.DEBUG, "failed-obligation redelivery after %s reconnect failed",
  850. platform.value, exc_info=True,
  851. ):
  852. await self._redeliver_failed_obligations_for_platform(platform)
  853. # Rebuild channel directory with the new adapter
  854. with suppress(Exception):
  855. from gateway.channel_directory import build_channel_directory
  856. await build_channel_directory(self.adapters)
  857. # A platform offline at startup skipped its restart-interrupted sessions; resume them now.
  858. try:
  859. self._schedule_resume_pending_sessions(platform=platform)
  860. except Exception:
  861. logger.debug("resume-pending reschedule after %s reconnect failed", platform.value, exc_info=True)
  862. async def _cancel_secondary_profile_reconnect_tasks(self) -> None:
  863. """Cancel profile-scoped reconnects before tearing down their registry, so a reconnect
  864. mid-setup cannot republish an adapter after the registry drains (bounded wait)."""
  865. pending = self._profile_failed_platforms
  866. if not isinstance(pending, dict):
  867. return
  868. current = asyncio.current_task()
  869. tasks = [
  870. task
  871. for profile_pending in pending.values()
  872. if isinstance(profile_pending, dict)
  873. for task in profile_pending.values()
  874. if isinstance(task, asyncio.Task) and task is not current and not task.done()
  875. ]
  876. for task in tasks:
  877. task.cancel()
  878. timeout = self._adapter_disconnect_timeout_secs()
  879. if tasks and timeout > 0:
  880. _done, unfinished = await asyncio.wait(tasks, timeout=timeout)
  881. if unfinished:
  882. logger.warning(
  883. "Timed out waiting for %d secondary profile reconnect task(s) during shutdown", len(unfinished),
  884. )
  885. pending.clear()
  886. async def _start_secondary_profile_adapters(self) -> int:
  887. """Bring up adapters for every non-active profile (multiplex only); returns connected count.
  888. Each profile connects under its own HERMES_HOME + secret scope; credential/listener collisions
  889. are refused here — the only point seeing every profile's credentials together."""
  890. from gateway.run import MultiplexConfigError, _multiplex_profile_homes
  891. from gateway.run_profile_reconcile import profile_serve_signature
  892. if not self._multiplex_on():
  893. # Runtime-status publication re-stamps the previous writer's record in place, so a multiplexer's
  894. # ``served_profiles`` would outlive it into this single-profile run and `hermes -p X ...`
  895. # would keep refusing (exit 78) / reporting "served" for profiles nobody serves.
  896. with _log_suppressed(logging.DEBUG, "could not clear served_profiles", exc_info=True):
  897. from gateway.status import publish_runtime_status
  898. publish_runtime_status(served_profiles=[])
  899. return 0
  900. try:
  901. from hermes_cli.profiles import get_active_profile_name, profiles_to_serve, profile_is_parked
  902. except Exception:
  903. return 0
  904. if self._multiplex_on():
  905. # Primary adapters belong to default. A named launcher is not skipped here —
  906. # it still needs its own secondary adapter and credential ownership entry.
  907. active = getattr(self, "_primary_profile_name", None) or "default"
  908. else:
  909. active = get_active_profile_name() or "default" # launch profile, pre-identity (adapter boot)
  910. for name, home in profiles_to_serve(True, include_parked=True):
  911. if name != "default" and profile_is_parked(home):
  912. logger.info("profile '%s' is parked (gateway.parked); not served by this gateway", name)
  913. connected = 0
  914. claimed = self._primary_resource_claims(active)
  915. profile_homes = _multiplex_profile_homes(self.config)
  916. self._served_profile_signatures = {}
  917. transient_failed = set()
  918. for profile_name, profile_home in profile_homes:
  919. if profile_name == active:
  920. continue # handled by the primary startup loop
  921. # Preserve changes made while the initial connection is awaiting I/O.
  922. scan_signature = profile_serve_signature(profile_home)
  923. try:
  924. connected += await self._start_one_profile_adapters(profile_name, profile_home, claimed)
  925. except MultiplexConfigError:
  926. raise
  927. except Exception as e:
  928. logger.error("Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True)
  929. # Not acknowledged: the reconcile watcher retries a transiently-failed profile.
  930. transient_failed.add(profile_name)
  931. else:
  932. self._served_profile_signatures[profile_name] = scan_signature
  933. self._record_served_profiles(active, profile_homes)
  934. # ``_note_served_profiles`` fills a missing signature with the current one; that refill
  935. # would park a transiently-failed profile before the first watcher tick can retry it.
  936. for profile_name in transient_failed:
  937. self._served_profile_signatures.pop(profile_name, None)
  938. # Cached configs follow the served set: a profile that failed to start (or stopped being
  939. # served) keeps no home channel in the host-wide notice fan-out, where it would be owed a
  940. # notice no transport can deliver and ``.restart_pending.json`` would never be unlinked.
  941. configs = getattr(self, "_profile_configs", None)
  942. if configs is not None:
  943. for profile_name in [p for p in configs if p not in self._served_profile_signatures]:
  944. configs.pop(profile_name, None)
  945. self._restore_secondary_completion_ledgers(profile_homes)
  946. return connected
  947. def _primary_resource_claims(self, active: str) -> dict[tuple, str]:
  948. """Resource claim -> owning profile for every live or queued primary adapter (credential:
  949. one account polled once; listener: one bind+port). A queued retryable primary owns both."""
  950. claimed: dict[tuple, str] = {}
  951. for _plat, _ad in self.adapters.items():
  952. fp = self._adapter_credential_fingerprint(_ad)
  953. for claim in ((_plat, fp) if fp is not None else None, self._adapter_listener_claim(_plat, _ad)):
  954. if claim is not None:
  955. claimed[claim] = active
  956. for retry_info in getattr(self, "_failed_platforms", {}).values():
  957. for claim_name in ("credential_claim", "listener_claim"):
  958. retry_claim = retry_info.get(claim_name)
  959. if isinstance(retry_claim, tuple):
  960. claimed[retry_claim] = active
  961. return claimed
  962. def _record_served_profiles(self, active: str, profile_homes) -> None:
  963. """Record the served set (eligible for routing/HTTP prefixes/cron/runtime scope — broader
  964. than "has a connected adapter") for `hermes status`; seed per-profile PairingStores."""
  965. with _log_suppressed(logging.DEBUG, "could not record served_profiles", exc_info=True):
  966. from gateway.status import publish_runtime_status
  967. from gateway.pairing import PairingStore
  968. served = [active] + sorted(name for name, _home in profile_homes if name != active)
  969. self._note_served_profiles(profile_homes)
  970. for name in served:
  971. if name and name not in self.pairing_stores:
  972. self.pairing_stores[name] = (
  973. self.pairing_store if name == active else PairingStore(profile=name)
  974. )
  975. publish_runtime_status(served_profiles=served)
  976. # The host record is what a second `gateway run` reads to decide attach-vs-start; keep
  977. # its served set in step with the live one (it is republished, never re-claimed).
  978. from gateway.host_rendezvous import ROLE_GATEWAY, owns_host_lock, publish_record
  979. if owns_host_lock(ROLE_GATEWAY):
  980. from hermes_constants import get_hermes_home
  981. publish_record(ROLE_GATEWAY, profiles=tuple(served), home=str(get_hermes_home()))
  982. async def _load_secondary_profile_config(self, profile_name: str, profile_home: Path):
  983. """Hydrate + enter ``profile_home``'s scope once; return its gateway config. Raises
  984. ``MultiplexConfigError`` (open dm/group policy). Port-binding platforms are NOT refused: the
  985. default profile owns the single shared listener and a secondary's port-binders are built in
  986. shared-listener mode (``/p/<profile>/...``) by ``_start_one_profile_adapters``."""
  987. from gateway.run import (
  988. MultiplexConfigError, _load_gateway_config,
  989. _own_policy_open_startup_violation, _profile_runtime_scope,
  990. )
  991. from gateway.config import load_gateway_config
  992. from hermes_cli.env_loader import hydrate_profile_secret_sources
  993. # Hydrate external secret sources off-loop ONCE: sync hydration would stall every heartbeat.
  994. await asyncio.to_thread(hydrate_profile_secret_sources, profile_home)
  995. # A platform that left core for a catalog plugin is installed before discovery below.
  996. from gateway.run_startup import recover_left_core_in
  997. await asyncio.to_thread(recover_left_core_in, profile_home, hydrate_secrets=False)
  998. with _profile_runtime_scope(profile_home, hydrate_secrets=False):
  999. profile_runtime_cfg = _load_gateway_config()
  1000. from hermes_cli.plugins import discover_plugins, get_plugin_manager
  1001. discover_plugins()
  1002. self._subscribe_plugin_rewire(get_plugin_manager(), profile_name, profile_home)
  1003. # This profile's `hooks:` block: start() registered before any profile scope existed.
  1004. self._register_config_hooks(
  1005. "shell-hook/webhook registration failed for profile '%s'", profile_name, level=logging.WARNING,
  1006. )
  1007. profile_cfg = load_gateway_config()
  1008. violation = _own_policy_open_startup_violation(profile_cfg)
  1009. self._snapshot_profile_busy_modes(profile_name, profile_runtime_cfg)
  1010. if violation:
  1011. raise MultiplexConfigError(
  1012. f"Profile '{profile_name}' enables {violation}. "
  1013. "Enable GATEWAY_ALLOW_ALL_USERS or the platform allow-all flag "
  1014. "for that profile, or change dm_policy/group_policy away from 'open'."
  1015. )
  1016. return profile_cfg
  1017. @staticmethod
  1018. def _credential_claim_origin(profile_name: str, profile_home, platform: Platform, token: str) -> Optional[str]:
  1019. """Where *token* was configured for *profile_name*: that profile's ``.env``, or ambient env.
  1020. ``None`` when the value is not in either place (yaml-only, or unknown). An env label
  1021. is only returned when the profile's own ``.env`` does not contain the value — a shell
  1022. export that merely echoes the file is the file.
  1023. """
  1024. from gateway.config import PLATFORM_TOKEN_ENV_NAMES
  1025. env_name = PLATFORM_TOKEN_ENV_NAMES.get(platform)
  1026. if not env_name or not isinstance(token, str) or not token.strip():
  1027. return None
  1028. token = token.strip()
  1029. in_dotenv = False
  1030. if profile_home:
  1031. try:
  1032. from agent.secret_scope import load_env_file
  1033. in_dotenv = (load_env_file(Path(profile_home) / ".env").get(env_name) or "").strip() == token
  1034. except Exception:
  1035. in_dotenv = False
  1036. if in_dotenv:
  1037. if profile_name and profile_name != "default":
  1038. return f"profiles/{profile_name}/.env"
  1039. return ".env"
  1040. if (os.environ.get(env_name) or "").strip() == token:
  1041. return f"env {env_name}"
  1042. return None
  1043. def _duplicate_credential_origins(
  1044. self, owner: str, profile_name: str, profile_home, platform: Platform, adapter: Any,
  1045. ) -> tuple:
  1046. """``(owner_origin, incoming_origin)`` for a same-credential refusal; either may be None."""
  1047. token = None
  1048. for obj, attr in (
  1049. (adapter, "token"), (adapter, "bot_token"),
  1050. (getattr(adapter, "config", None), "token"),
  1051. ):
  1052. val = getattr(obj, attr, None) if obj is not None else None
  1053. if isinstance(val, str) and val.strip():
  1054. token = val.strip()
  1055. break
  1056. if not token:
  1057. return None, None
  1058. incoming = self._credential_claim_origin(profile_name, profile_home, platform, token)
  1059. owner_home = None
  1060. if owner == "default":
  1061. try:
  1062. from hermes_constants import get_default_hermes_root
  1063. owner_home = get_default_hermes_root()
  1064. except Exception:
  1065. owner_home = None
  1066. else:
  1067. try:
  1068. from hermes_cli.profiles import get_profile_dir
  1069. owner_home = get_profile_dir(owner)
  1070. except Exception:
  1071. owner_home = None
  1072. owner_origin = (
  1073. self._credential_claim_origin(owner, owner_home, platform, token) if owner_home else None
  1074. )
  1075. return owner_origin, incoming
  1076. def _refuse_duplicate_claim(
  1077. self, claim, claimed: dict[tuple, str], profile_name: str, platform: Platform, kind: str,
  1078. *, owner_origin: Optional[str] = None, incoming_origin: Optional[str] = None,
  1079. ) -> bool:
  1080. """Log + park a secondary adapter whose credential/listener another profile owns (True when
  1081. refused). NOT disconnected: it never connected, and for a same-credential Photon adapter
  1082. disconnect() would shut down the primary profile's live sidecar."""
  1083. owner = claimed.get(claim) if claim is not None else None
  1084. if owner is None:
  1085. return False
  1086. pv = platform.value
  1087. env_derived = any(
  1088. isinstance(origin, str) and origin.startswith("env ")
  1089. for origin in (owner_origin, incoming_origin)
  1090. )
  1091. def _who(name: str, origin: Optional[str]) -> str:
  1092. return f"{name} ({origin})" if env_derived and origin else name
  1093. head = f"Profile '{_who(owner, owner_origin)}' and '{_who(profile_name, incoming_origin)}' both configure {pv} "
  1094. if kind == "credential":
  1095. message = head + f"with the same credential. Give each profile its own {pv} credential."
  1096. logger.error(
  1097. "Profile '%s' and '%s' both configure %s with the same credential — refusing to start the "
  1098. "duplicate (one credential cannot be consumed twice). Give each profile its own %s credential.",
  1099. _who(owner, owner_origin), _who(profile_name, incoming_origin), pv, pv,
  1100. )
  1101. else:
  1102. bind, port = claim[-2:]
  1103. message = head + f"sidecars on the same listener. Configure a distinct listener for profile '{profile_name}'."
  1104. logger.error(
  1105. "Profile '%s' and '%s' both configure %s sidecars on %s:%s — refusing to start the duplicate "
  1106. "listener. Set platforms.%s.extra.sidecar_port to a distinct port for profile '%s'.",
  1107. owner, profile_name, pv, bind, port, pv, profile_name,
  1108. )
  1109. self._update_platform_runtime_status(
  1110. f"{profile_name}:{platform.value}", platform_state="fatal",
  1111. error_code=f"duplicate_{kind}", error_message=message,
  1112. )
  1113. return True
  1114. def _note_unserved_secondary_platform(self, profile_name: str, platform: Platform) -> None:
  1115. """Report unpaired WhatsApp or secondary-only Relay. Log the reason and remedy and stamp a
  1116. ``<profile>:<platform>`` status entry so ``hermes gateway status --profile X`` and the
  1117. dashboard show *why* the channel is dead instead of nothing at all."""
  1118. noted = getattr(self, "_unserved_secondary_platforms", None)
  1119. if noted is None:
  1120. noted = self._unserved_secondary_platforms = set()
  1121. if (profile_name, platform) in noted:
  1122. return
  1123. noted.add((profile_name, platform))
  1124. pv = platform.value
  1125. if platform is Platform.WHATSAPP:
  1126. message = f"WhatsApp is not paired; pair it: hermes -p {profile_name} whatsapp"
  1127. logger.info("[MULTIPLEX] Profile '%s': %s", profile_name, message)
  1128. self._update_platform_runtime_status(
  1129. f"{profile_name}:{pv}", platform_state="disabled",
  1130. error_code="whatsapp_unpaired", error_message=message,
  1131. )
  1132. return
  1133. logger.info(
  1134. "[MULTIPLEX] Profile '%s': %s is enabled but not served — %s is process-level shared ingress "
  1135. "owned by the default profile under multiplex. Enable and configure %s on the default profile "
  1136. "(it serves every profile), or disable it in profile '%s'.",
  1137. profile_name, pv, pv, pv, profile_name,
  1138. )
  1139. self._update_platform_runtime_status(
  1140. f"{profile_name}:{pv}", platform_state="disabled", error_code="multiplex_shared_ingress",
  1141. error_message="not served under multiplex (shared ingress owned by default)",
  1142. )
  1143. def _unserved_shared_ingress_warnings(self) -> list:
  1144. """Loud ``not being served`` lines for shared-ingress platforms secondaries enabled while
  1145. NO profile (default included) actually runs them; empty when the default serves the platform."""
  1146. noted = getattr(self, "_unserved_secondary_platforms", None) or ()
  1147. lines = []
  1148. for platform in sorted({p for _n, p in noted}, key=lambda p: p.value):
  1149. if platform is Platform.WHATSAPP:
  1150. continue # unpaired sessions have their own per-profile remedy
  1151. if platform in self.adapters or platform in (getattr(self, "_failed_platforms", None) or {}):
  1152. continue # the default owns it: secondaries ARE served through the shared adapter
  1153. profiles = sorted(n for n, p in noted if p is platform)
  1154. lines.append(
  1155. f"{platform.value} is enabled in profile(s) {', '.join(profiles)} but not on the default "
  1156. f"profile — the platform is not being served. Under multiplex {platform.value} is shared "
  1157. "ingress: enable and configure it on the default profile, or disable it in those profiles."
  1158. )
  1159. return lines
  1160. async def _start_one_profile_adapters(
  1161. self, profile_name: str, profile_home: Path, claimed: dict[tuple, str]
  1162. ) -> int:
  1163. """Create+connect one profile's adapters under its runtime scope."""
  1164. from gateway.run import _platform_has_bot_credential, _profile_runtime_scope
  1165. profile_cfg = await self._load_secondary_profile_config(profile_name, profile_home)
  1166. # Keep the served profile's config: host-wide passes (planned-restart notices) must reach
  1167. # every served profile's home channels, and this is the only place it is loaded.
  1168. configs = getattr(self, "_profile_configs", None)
  1169. if configs is None:
  1170. configs = self._profile_configs = {}
  1171. configs[profile_name] = profile_cfg
  1172. multiplex = self._multiplex_on()
  1173. profile_map = self._profile_adapters.setdefault(profile_name, {})
  1174. connected = 0
  1175. for platform, platform_config in profile_cfg.platforms.items():
  1176. if not platform_config.enabled:
  1177. continue
  1178. # Runtime re-scan of a served profile (config/.env changed): only platforms that are not
  1179. # already live or queued for reconnect are built — never a second poller on the same bot.
  1180. # Nor held ones (tearing down after a fatal error, or awaiting the boot handoff): their own reconnect
  1181. # rebuilds them, or none does because the error was not retryable.
  1182. if platform in profile_map or platform in (
  1183. (getattr(self, "_profile_failed_platforms", None) or {}).get(profile_name) or {}) or (
  1184. (profile_name, platform) in (getattr(self, "_secondary_held_claims", None) or {})):
  1185. continue
  1186. # No credential in THIS profile's scope: an adapter would fan inbound across every such profile.
  1187. with _profile_runtime_scope(profile_home, hydrate_secrets=False):
  1188. has_credential = _platform_has_bot_credential(platform, platform_config)
  1189. if multiplex and not has_credential:
  1190. if platform is Platform.WHATSAPP:
  1191. self._note_unserved_secondary_platform(profile_name, platform)
  1192. logger.info(
  1193. "[MULTIPLEX] Profile '%s': skipping %s - no bot credential "
  1194. "in this profile's secrets", profile_name, platform.value,
  1195. )
  1196. continue
  1197. # Relay still uses process-level shared ingress; WhatsApp owns a session per profile.
  1198. if multiplex and platform is Platform.RELAY:
  1199. self._note_unserved_secondary_platform(profile_name, platform)
  1200. continue
  1201. # api_server / webhook: the default's listener already mirrors them at /p/<profile>/; a second
  1202. # instance here would fight the default for the port (#100397).
  1203. if multiplex and platform.value in SHARED_LISTENER_MIRROR_PLATFORMS:
  1204. logger.info(
  1205. "[MULTIPLEX] Profile '%s': %s is served by the default profile's listener at /p/%s/ — "
  1206. "not starting a second listener", profile_name, platform.value, profile_name,
  1207. )
  1208. continue
  1209. with _profile_runtime_scope(profile_home, hydrate_secrets=False):
  1210. try:
  1211. adapter = self._create_adapter(platform, platform_config)
  1212. except Exception:
  1213. logger.error("[MULTIPLEX] Profile '%s': _create_adapter('%s') raised", profile_name,
  1214. platform.value, exc_info=True)
  1215. continue
  1216. # Classify in this profile's scope: plugin registration is per profile home.
  1217. heals = False if adapter else self._adapter_may_heal(platform, platform_config)
  1218. if not adapter:
  1219. self._note_unbuilt_secondary_platform(profile_name, platform, heals)
  1220. continue
  1221. # Same-token / same-listener conflict detection — refuse a duplicate poll or bind.
  1222. # Held claims too (a profile earlier in the sweep whose connect failed, or one healing, still owns
  1223. # its bot), and once live, owners as of now: a runtime caller's snapshot predates these awaits.
  1224. owners = self._current_claim_owners() if getattr(self, "_running", False) else self._held_claim_owners()
  1225. claims = self._claim_secondary_platform(
  1226. profile_name, profile_home, platform, adapter, {**claimed, **owners})
  1227. if claims is None:
  1228. continue
  1229. self._configure_profile_adapter(adapter, profile_name, platform)
  1230. # Hold the bot across the connect await; a failed connect's reconnect takes the hold over.
  1231. self._hold_secondary_claims(profile_name, platform, adapter, claims)
  1232. try:
  1233. try:
  1234. with _profile_runtime_scope(profile_home, hydrate_secrets=False):
  1235. success = await self._connect_initial_adapter_with_timeout(adapter, platform)
  1236. if not success:
  1237. logger.warning("✗ %s failed to connect (profile: %s)", platform.value, profile_name)
  1238. except Exception as e:
  1239. logger.error("✗ %s error (profile: %s): %s", platform.value, profile_name, e)
  1240. success = False
  1241. if not success:
  1242. await self._safe_adapter_disconnect(adapter, platform)
  1243. self._schedule_secondary_profile_startup_reconnect(profile_name, platform, adapter)
  1244. continue
  1245. profile_map[platform] = adapter
  1246. finally:
  1247. self._release_secondary_claims(profile_name, platform, adapter)
  1248. # Restore persisted /voice state for this bot (primary startup and reconnects do too).
  1249. # See #84872.
  1250. self._sync_voice_mode_state_to_adapter(adapter)
  1251. for claim in claims:
  1252. if claim is not None:
  1253. claimed[claim] = profile_name
  1254. connected += 1
  1255. logger.info("✓ %s connected (profile: %s)", platform.value, profile_name)
  1256. return connected
  1257. def _wire_adapter_handlers(
  1258. self, adapter: BasePlatformAdapter, *, message_handler=None, fatal_error_handler=None,
  1259. busy_session_handler=None, authorization_check=None, platform_event_handler=None,
  1260. busy_text_mode: Optional[str] = None, busy_text_timing: Optional[tuple[float, float]] = None,
  1261. human_delay: Optional[tuple[int, int]] | object = _UNSET,
  1262. ) -> None:
  1263. """Install the runner callbacks every adapter needs (defaults = primary handlers;
  1264. secondary wiring passes profile-scoped variants). ``set_reaction_handler`` is optional."""
  1265. adapter.set_message_handler(message_handler or self._primary_message_handler())
  1266. adapter.set_fatal_error_handler(fatal_error_handler or self._handle_adapter_fatal_error)
  1267. adapter.set_session_store(self.session_store)
  1268. adapter.set_busy_session_handler(busy_session_handler or self._primary_busy_session_handler())
  1269. _set_reaction = getattr(adapter, "set_reaction_handler", None)
  1270. if callable(_set_reaction):
  1271. _set_reaction(self._handle_reaction_event)
  1272. adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id)
  1273. _set_transfer = getattr(adapter, "set_topic_transfer_fn", None)
  1274. if callable(_set_transfer):
  1275. _set_transfer(self._transfer_telegram_root_dm)
  1276. adapter.set_authorization_check(
  1277. authorization_check or self._make_adapter_auth_check(adapter.platform)
  1278. )
  1279. adapter.set_platform_event_handler(platform_event_handler or self._primary_platform_event_handler())
  1280. adapter._busy_text_mode = (self._busy_text_mode if busy_text_mode is None else busy_text_mode)
  1281. timing = busy_text_timing or getattr(self, "_busy_text_timing", None)
  1282. if timing:
  1283. adapter._busy_text_debounce_seconds, adapter._busy_text_hard_cap_seconds = timing
  1284. adapter._human_delay_range_ms = (
  1285. getattr(self, "_human_delay", None) if human_delay is _UNSET else human_delay)
  1286. def _configure_profile_adapter(
  1287. self, adapter: BasePlatformAdapter, profile_name: str, platform: Platform
  1288. ) -> None:
  1289. """Install the profile-scoped handlers shared by startup and reconnect."""
  1290. # Runtime status is process-scoped: key on profile:platform so health shows WHICH secondary failed.
  1291. adapter._runtime_status_platform_key = f"{profile_name}:{platform.value}"
  1292. # Declare ownership BEFORE any inbound event: adapter-level session keys are derived at ingress,
  1293. # before the handler stamps source.profile (else every secondary keys into `agent:main:`).
  1294. _set_owner = getattr(adapter, "set_owner_profile", None)
  1295. if callable(_set_owner):
  1296. _set_owner(profile_name)
  1297. # Voice transcripts from this bot's channels dispatch through THIS adapter (primary wiring lives at
  1298. # connect time; see #75198).
  1299. text_modes = getattr(self, "_busy_text_modes_by_profile", None)
  1300. timings = getattr(self, "_busy_text_timing_by_profile", None)
  1301. delays = getattr(self, "_human_delay_by_profile", None)
  1302. self._wire_adapter_handlers(
  1303. adapter,
  1304. message_handler=self._make_profile_message_handler(profile_name),
  1305. fatal_error_handler=self._make_profile_fatal_error_handler(profile_name, platform),
  1306. busy_session_handler=self._make_profile_busy_session_handler(profile_name),
  1307. authorization_check=self._make_adapter_auth_check(platform, profile_name=profile_name),
  1308. platform_event_handler=self._make_profile_platform_event_handler(profile_name),
  1309. busy_text_mode=(
  1310. text_modes.get(profile_name, self._busy_text_mode)
  1311. if isinstance(text_modes, dict)
  1312. else self._busy_text_mode
  1313. ),
  1314. busy_text_timing=(timings.get(profile_name) if isinstance(timings, dict) else None),
  1315. human_delay=(delays.get(profile_name, _UNSET) if isinstance(delays, dict) else _UNSET),
  1316. )
  1317. # Voice transcripts from this bot's channels dispatch through THIS adapter.
  1318. self._bind_voice_input_callback(adapter)
  1319. # Secondary adapters carry their profile so prune paths namespace topic bindings correctly.
  1320. # See #76423.
  1321. adapter._hermes_profile_name = profile_name
  1322. # A secondary's port-binding adapter never binds: the default profile owns the one shared
  1323. # listener, which forwards /p/<profile>/<path> to this adapter's app (shared_ingress.py).
  1324. if self._multiplex_on() and platform.value not in SHARED_LISTENER_MIRROR_PLATFORMS \
  1325. and _platform_binds_port(platform.value, getattr(getattr(adapter, "config", None), "extra", None)):
  1326. adapter._shared_listener_profile = profile_name
  1327. def _make_profile_fatal_error_handler(
  1328. self, profile_name: str, platform: Platform
  1329. ) -> Callable[[BasePlatformAdapter], Awaitable[None]]:
  1330. """Route a secondary-profile fatal error to that profile's reconnect slot."""
  1331. return functools.partial(self._handle_profile_adapter_fatal_error, profile_name, platform)
  1332. async def _handle_profile_adapter_fatal_error(
  1333. self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter
  1334. ) -> None:
  1335. """Remove a failed multiplexed adapter (the primary-only fatal handler ignores them)."""
  1336. profile_map = getattr(self, "_profile_adapters", {}).get(profile_name)
  1337. if not isinstance(profile_map, dict) or profile_map.get(platform) is not adapter:
  1338. logger.debug(
  1339. "Ignoring stale fatal error from secondary %s adapter (profile: %s)",
  1340. platform.value, profile_name,
  1341. )
  1342. return
  1343. profile_map.pop(platform, None)
  1344. # Hold its bot across the teardown await so a healing profile cannot take it meanwhile; the reconnect
  1345. # scheduled below takes the hold over, and every other exit releases it.
  1346. self._hold_secondary_claims(profile_name, platform, adapter, self._adapter_claims(platform, adapter))
  1347. try:
  1348. # The notification may arrive outside the profile's scope: the row is that profile's.
  1349. if (profile_home := self._routed_profile_home(profile_name)) is not UNRESOLVED_PROFILE_HOME:
  1350. record_platform_disconnect(adapter, hermes_home=profile_home)
  1351. await self._safe_adapter_disconnect(adapter, platform)
  1352. if not self._running or getattr(self, "_profile_adapters", {}).get(profile_name) is not profile_map:
  1353. return # shutting down, or the profile was unserved while it tore down: nothing to reconnect
  1354. self._schedule_secondary_profile_reconnect(profile_name, platform, adapter)
  1355. finally:
  1356. self._release_secondary_claims(profile_name, platform, adapter)
  1357. logger.error(
  1358. "Fatal %s adapter error for multiplexed profile %s (%s)", platform.value, profile_name,
  1359. adapter.fatal_error_code or "unknown",
  1360. )
  1361. @staticmethod
  1362. def _routed_profile_home(profile_name: str):
  1363. """A named routed profile's home, or :data:`UNRESOLVED_PROFILE_HOME` when it does not
  1364. resolve (deleted or renamed mid-run, invalid name, transient OSError).
  1365. Never ``None``: ``None`` is reserved for "this body is the launch profile's own work", and
  1366. answering it for an unresolvable NAMED profile is what made a secondary's inbound message
  1367. run on the launch profile's credentials.
  1368. """
  1369. from hermes_cli.profiles import get_profile_dir
  1370. try:
  1371. return get_profile_dir(profile_name)
  1372. except Exception:
  1373. logger.warning(
  1374. "Profile home for '%s' does not resolve; its handlers run with no profile scope "
  1375. "(credential reads fail closed instead of borrowing the launch profile's)",
  1376. profile_name, exc_info=True)
  1377. return UNRESOLVED_PROFILE_HOME
  1378. @staticmethod
  1379. def _scope_or_null(scope_factory, profile_home):
  1380. """The runtime scope for one body, by what ``profile_home`` IS:
  1381. * a home — that profile's scope;
  1382. * ``None`` — no routed profile: the LAUNCH profile's own work, so bind the launch profile
  1383. explicitly once this process multiplexes. A bare ``nullcontext()`` made the launch
  1384. profile the one tenant running on ambient ``os.environ`` and the process home, which a
  1385. secondary's context may have poisoned; under the one-process-per-host ruling it is a
  1386. tenant like any other. Single-profile hosts are unchanged (no-op until activation);
  1387. * :data:`UNRESOLVED_PROFILE_HOME` — a NAMED profile whose home is gone: bind nothing, so an
  1388. unscoped ``get_secret`` raises ``UnscopedSecretError`` under multiplexing. A profile never
  1389. borrows another profile's value, and "we cannot tell whose this is" must fail closed.
  1390. """
  1391. if profile_home is UNRESOLVED_PROFILE_HOME:
  1392. return contextlib.nullcontext()
  1393. if profile_home is not None:
  1394. return scope_factory(profile_home)
  1395. from tui_gateway.launch_profile_policy import launch_profile_scope_if_multiplexed
  1396. return launch_profile_scope_if_multiplexed()
  1397. @staticmethod
  1398. def _async_scope_or_null(scope_factory, profile_home):
  1399. """``async with`` twin of :meth:`_scope_or_null` (for ``_async_profile_runtime_scope``)."""
  1400. if profile_home is UNRESOLVED_PROFILE_HOME:
  1401. return contextlib.nullcontext()
  1402. if profile_home is not None:
  1403. return scope_factory(profile_home)
  1404. from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
  1405. return async_launch_profile_scope_if_multiplexed()
  1406. def _canonicalize(self, source, *, transport_profile: Optional[str] = None,
  1407. primary_home: Optional[Path] = None):
  1408. """Runner-side identity seam: the pinned :class:`RoutingIdentity` of *source*, resolving it
  1409. once when absent. ``transport_profile`` names a secondary's own bot (its handlers know it by
  1410. construction); ``None`` = the primary/shared bot. ``None`` result = rejected route under
  1411. multiplexing (the caller drops; the ingress gate warns once) or a source that cannot resolve
  1412. (bare test rigs) — the legacy readers then stay in force."""
  1413. if source is None:
  1414. return None
  1415. from gateway.session_identity import canonical_identity
  1416. try:
  1417. identity = canonical_identity(
  1418. source, runner=self, transport_profile=transport_profile, primary_home=primary_home)
  1419. except Exception:
  1420. logger.debug("identity resolution failed; legacy readers stay in force", exc_info=True)
  1421. return None
  1422. if transport_profile and not getattr(source, "profile", None):
  1423. with suppress(Exception):
  1424. source.profile = transport_profile # a secondary's own event is at least its own
  1425. return identity
  1426. def _make_profile_message_handler(self, profile_name: str):
  1427. """Message handler that canonicalizes the event's identity FIRST, then delegates under the
  1428. profile scope (auth runs BEFORE the agent-turn scope, so the profile's ``.env`` must be
  1429. visible here)."""
  1430. from gateway.run import _async_profile_runtime_scope
  1431. profile_home = self._routed_profile_home(profile_name)
  1432. from gateway.session_ingress_context import native_callback, register_transport_home
  1433. # An unresolvable named profile has no transport home: the native callback then fails
  1434. # closed under session authority (``profile_mismatch``) instead of naming a path.
  1435. transport_home = None if profile_home is UNRESOLVED_PROFILE_HOME else profile_home
  1436. register_transport_home(self, profile_name, transport_home)
  1437. async def _handler(_runner, event):
  1438. identity = self._canonicalize(getattr(event, "source", None), transport_profile=profile_name)
  1439. if transport_home is not None:
  1440. event.source._authorization_profile_home = transport_home
  1441. with native_callback(self, event, transport_home, profile_name):
  1442. async with self._async_scope_or_null(
  1443. _async_profile_runtime_scope, self._routed_runtime_home(identity, profile_home)):
  1444. return await self._handle_message(event)
  1445. return _handler.__get__(self)
  1446. @staticmethod
  1447. def _routed_runtime_home(identity, profile_home):
  1448. """A secondary bot's event runs in its ``bot_profile`` route's runtime scope (as the primary's handler
  1449. does), so that runtime's authority admits the turn and later completion wakes find it on the same
  1450. ledger (#139726). Authorization still reads the transport home (``_under_authorization_profile``)."""
  1451. if identity is None or profile_home is UNRESOLVED_PROFILE_HOME:
  1452. return profile_home
  1453. return identity.runtime_home
  1454. def _make_profile_busy_session_handler(self, profile_name: str):
  1455. """Busy-path twin: canonicalize FIRST, then resolve busy policy under the profile scope
  1456. (auth runs against the profile's own allowlist, same as the cold-path message handler)."""
  1457. from gateway.run import _async_profile_runtime_scope
  1458. profile_home = self._routed_profile_home(profile_name)
  1459. async def _handler(event, _session_key):
  1460. identity = self._canonicalize(event.source, transport_profile=profile_name)
  1461. async with self._async_scope_or_null(
  1462. _async_profile_runtime_scope, self._routed_runtime_home(identity, profile_home)):
  1463. return await self._handle_active_session_busy_message(event, self._session_key_for_source(event.source))
  1464. return _handler
  1465. def _make_default_profile_message_handler(self):
  1466. """Scope primary-adapter messages to their routed multiplex profile. Authorization stays
  1467. with the transport profile (a routed profile may have no credential/allowlist)."""
  1468. from gateway.run import _async_profile_runtime_scope, get_hermes_home
  1469. default_home = Path(get_hermes_home())
  1470. from gateway.session_ingress_context import register_transport_home
  1471. register_transport_home(self, None, default_home)
  1472. async def _handler(_runner, event):
  1473. # A rejected route still enters ``_handle_message``, whose ingress gate drops it fail-closed.
  1474. profile_home = self._admit_primary_source(event.source, default_home) or default_home
  1475. from gateway.session_ingress_context import native_callback
  1476. with native_callback(self, event, default_home):
  1477. async with _async_profile_runtime_scope(profile_home):
  1478. return await self._handle_message(event)
  1479. return _handler.__get__(self)
  1480. def _make_default_profile_busy_session_handler(self):
  1481. """Busy-path twin of ``_make_default_profile_message_handler``: busy callbacks bypass the message
  1482. handler, so the routed scope and transport-home authorization must be re-established here or the
  1483. follow-up is authorized in whatever scope is ambient (#103717)."""
  1484. from gateway.run import _async_profile_runtime_scope, get_hermes_home
  1485. default_home = Path(get_hermes_home())
  1486. async def _handler(event, _session_key):
  1487. source = event.source
  1488. profile_home = self._admit_primary_source(source, default_home)
  1489. if profile_home is None:
  1490. return True # rejected route: swallow, same disposition as the ingress gate
  1491. async with _async_profile_runtime_scope(profile_home):
  1492. return await self._handle_active_session_busy_message(
  1493. event, self._session_key_for_source(source)
  1494. )
  1495. return _handler
  1496. def _admit_primary_source(self, source, default_home: Path) -> Optional[Path]:
  1497. """Canonicalize a primary-adapter source (transport home for authorization, routed profile
  1498. for the runtime) and return the runtime home to scope the turn under; ``None`` when the
  1499. route targets an unserved profile. Route ≠ admitting bot."""
  1500. identity = self._canonicalize(source, primary_home=default_home)
  1501. if identity is not None:
  1502. return identity.runtime_home
  1503. return None if getattr(source, "profile_route_rejected", False) is True else default_home
  1504. def _primary_message_handler(self):
  1505. """Return the correctly scoped handler for a primary adapter."""
  1506. shared = getattr(self, 'session_authority', None) is not None
  1507. if self._multiplex_on() or shared:
  1508. return self._make_default_profile_message_handler()
  1509. return self._standalone_scoped(self._handle_message)
  1510. def _primary_busy_session_handler(self):
  1511. """Return the correctly scoped busy-session handler for a primary adapter."""
  1512. if self._multiplex_on():
  1513. return self._make_default_profile_busy_session_handler()
  1514. return self._standalone_scoped(self._handle_active_session_busy_message)
  1515. def _standalone_scoped(self, handler):
  1516. """Standalone twin of the ``_make_default_profile_*`` wrappers: run ``handler`` under
  1517. ``_standalone_launch_scope`` so slash commands and turns keep resolving the launch profile's
  1518. credentials after a hosted room flipped the process-wide guard (#112878). The credential
  1519. body is marked at entry (``launch_body``): a turn already running when a second profile
  1520. switches this gateway to multiplexing keeps resolving its launch credentials."""
  1521. from tui_gateway.launch_profile_policy import launch_body
  1522. async def _handler(*args):
  1523. with launch_body(), self._standalone_launch_scope():
  1524. return await handler(*args)
  1525. return _handler
  1526. def _multiplex_on(self) -> bool:
  1527. return bool(getattr(self.config, "multiplex_profiles", False))
  1528. async def _handle_gateway_platform_event(self, event: dict, source) -> None:
  1529. """Authorize and publish one normalized adapter event to plugin hooks."""
  1530. # Observer failures must never break the adapter's update loop.
  1531. with _log_suppressed(logging.DEBUG, "gateway_platform_event hook dispatch failed", exc_info=True):
  1532. from hermes_cli.lifecycle import has_hook, invoke_hook
  1533. if has_hook("gateway_platform_event") and self._is_user_authorized_for_source(source):
  1534. invoke_hook("gateway_platform_event", **event)
  1535. def _make_profile_platform_event_handler(self, profile_name: str):
  1536. """Bind platform-event auth and hook dispatch to one multiplex profile."""
  1537. from gateway.run import _profile_runtime_scope
  1538. profile_home = self._routed_profile_home(profile_name)
  1539. async def _handler(event, source):
  1540. self._canonicalize(source, transport_profile=profile_name)
  1541. with self._scope_or_null(_profile_runtime_scope, profile_home):
  1542. return await self._handle_gateway_platform_event(event, source)
  1543. return _handler
  1544. def _make_default_profile_platform_event_handler(self):
  1545. """Scope primary-transport events to their routed multiplex profile."""
  1546. from gateway.run import _profile_runtime_scope, get_hermes_home
  1547. default_home = Path(get_hermes_home())
  1548. async def _handler(event, source):
  1549. profile_home = self._admit_primary_source(source, default_home)
  1550. if profile_home is None:
  1551. return None # rejected route: same disposition as the message ingress gate
  1552. with _profile_runtime_scope(profile_home):
  1553. return await self._handle_gateway_platform_event(event, source)
  1554. return _handler
  1555. def _primary_platform_event_handler(self):
  1556. if self._multiplex_on():
  1557. return self._make_default_profile_platform_event_handler()
  1558. return self._standalone_scoped(self._handle_gateway_platform_event)
  1559. @staticmethod
  1560. def _adapter_credential_claim(platform: Platform, adapter: Any) -> Optional[tuple]:
  1561. """Return the exclusive credential resource claimed by an adapter."""
  1562. from gateway.run import GatewayRunner
  1563. fingerprint = GatewayRunner._adapter_credential_fingerprint(adapter)
  1564. return None if fingerprint is None else (platform, fingerprint)
  1565. @staticmethod
  1566. def _adapter_listener_claim(platform: Platform, adapter: Any) -> Optional[tuple]:
  1567. """Exclusive listener claim (Photon sidecar bind+port): distinct credentials still cannot
  1568. share a port, so the later adapter is rejected before connect() disturbs the first."""
  1569. bind = getattr(adapter, "_sidecar_bind", None)
  1570. if getattr(platform, "value", None) != "photon" or not isinstance(bind, str) or not bind.strip():
  1571. return None
  1572. try:
  1573. port = int(getattr(adapter, "_sidecar_port", None))
  1574. except (TypeError, ValueError, OverflowError):
  1575. return None
  1576. return ("listener", "photon", bind.strip().lower(), port)
  1577. @staticmethod
  1578. def _adapter_credential_fingerprint(adapter: Any) -> Optional[str]:
  1579. """Salted, log-safe hash of an adapter's credential; None when none is discoverable
  1580. (conflict detection is then skipped)."""
  1581. # Many adapters (Discord) keep the token on `config`; without that fallback the check is skipped.
  1582. candidates = [
  1583. (adapter, attr) for attr in (
  1584. "token", "bot_token", "_token", "api_token", "_bot_token",
  1585. "_project_secret", # Photon/Spectrum: project credentials, not a bot token
  1586. "_app_id", # Feishu/Lark app_id — stable, log-safe, already the _app_lock_identity
  1587. "_client_id", "_bot_id", # Teams / WeCom app-style id pairs
  1588. )
  1589. ] + [(getattr(adapter, "config", None), attr) for attr in ("token", "bot_token")]
  1590. for obj, attr in candidates:
  1591. val = getattr(obj, attr, None)
  1592. if isinstance(val, str) and val.strip():
  1593. import hashlib
  1594. # surrogatepass: a token holding a lone surrogate still hashes (valid tokens hash as before), so a
  1595. # claim never raises mid-unserve or mid-teardown.
  1596. return hashlib.sha256(("hermes-mux:" + val.strip()).encode("utf-8", "surrogatepass")).hexdigest()[:16]
  1597. return None
  1598. def _create_adapter(self, platform: Platform, config: Any) -> Optional[BasePlatformAdapter]:
  1599. """Create an adapter bound to this runner (every lifecycle path goes through here so
  1600. adapters can resolve inbound profile routes before handlers or connect())."""
  1601. adapter = self._instantiate_adapter(platform, config)
  1602. if adapter is not None:
  1603. adapter.gateway_runner = self
  1604. return adapter
  1605. def _instantiate_adapter(self, platform: Platform, config: Any) -> Optional[BasePlatformAdapter]:
  1606. """Instantiate the adapter for a platform: plugin registry first, then built-ins."""
  1607. from gateway.run import _instantiate_builtin_adapter
  1608. if hasattr(config, "extra") and isinstance(config.extra, dict):
  1609. config.extra.setdefault("group_sessions_per_user", self.config.group_sessions_per_user)
  1610. config.extra.setdefault(
  1611. "thread_sessions_per_user", getattr(self.config, "thread_sessions_per_user", False)
  1612. )
  1613. with _log_suppressed(logging.DEBUG, "Platform registry lookup for '%s' failed: %s", platform.value):
  1614. from gateway.platform_registry import platform_registry
  1615. if platform_registry.is_registered(platform.value):
  1616. adapter = platform_registry.create_adapter(platform.value, config)
  1617. if adapter is None: # registered but failed — never fall through to built-ins
  1618. logger.error(
  1619. "Platform '%s' is registered but adapter creation failed "
  1620. "(check dependencies and config)", platform.value,
  1621. )
  1622. return adapter
  1623. return _instantiate_builtin_adapter(platform, config)
  1624. def _make_adapter_auth_check(
  1625. self, platform: Platform, profile_name: Optional[str] = None
  1626. ) -> Callable[[str, Optional[str], Optional[str]], bool]:
  1627. """Platform-bound auth callback for adapters (prompt-injection mitigation for fetched
  1628. context); delegates to :meth:`_is_user_authorized`. ``profile_name`` binds a secondary to
  1629. its scope; for the shared primary (None) the routed profile is stamped so its pairing store
  1630. is consulted while allowlist reads stay under the transport home.
  1631. Without this an inline-button caller approved only in the routed profile's pairing store was denied
  1632. (#86296), because the adapter's callback source was never route-stamped. A secondary-owned bot's
  1633. callback fires outside the profile runtime scope (straight off the adapter's event loop), so the
  1634. check re-enters the owning profile's scope per call — the gate's scoped env read otherwise falls
  1635. back to os.environ (the default profile's env) and denies the secondary's own allowlisted
  1636. callers (#120639).
  1637. """
  1638. from gateway.run import get_hermes_home
  1639. transport_home = Path(get_hermes_home()) if self._multiplex_on() and profile_name is None else None
  1640. # Resolved once; the scope is entered per call so an ``.env`` allowlist edit reaches the next
  1641. # tap, matching the message path's per-message re-read.
  1642. profile_home = self._routed_profile_home(profile_name) if profile_name else None
  1643. def check(
  1644. user_id: str, chat_type: Optional[str] = None, chat_id: Optional[str] = None, *,
  1645. is_bot: bool = False, thread_id: Optional[str] = None,
  1646. ) -> bool:
  1647. if not user_id:
  1648. return False
  1649. source = SessionSource(
  1650. platform=platform, chat_id=chat_id or "", chat_type=chat_type or "group",
  1651. user_id=user_id, thread_id=thread_id, is_bot=bool(is_bot), profile=profile_name,
  1652. )
  1653. # Same transport provenance as ``build_source``, so policy reads resolve the receiving adapter.
  1654. registry = (
  1655. (getattr(self, "_profile_adapters", None) or {}).get(profile_name)
  1656. if profile_name else getattr(self, "adapters", None)
  1657. ) or {}
  1658. adapter = registry.get(platform)
  1659. if adapter is not None:
  1660. source._transport_adapter_ref = _weakref.ref(adapter)
  1661. if transport_home is None:
  1662. # Sync, on the adapter's event loop (per tap, per inline-query keystroke): never
  1663. # hydrate external secret sources here — that takes the process-global source lock
  1664. # (#99519). Startup and the message path hydrate off-loop; this reads their cache.
  1665. from gateway.run import _profile_runtime_scope
  1666. with self._scope_or_null(
  1667. functools.partial(_profile_runtime_scope, hydrate_secrets=False), profile_home):
  1668. return self._is_user_authorized(source)
  1669. # Canonicalize FIRST (callback sources never went through ``build_source``): the routed
  1670. # profile's pairing store is consulted, allowlists read under the transport home.
  1671. if self._canonicalize(source, primary_home=transport_home) is None:
  1672. return False # fail-closed, like the ``_handle_message`` ingress gate
  1673. return self._is_user_authorized_for_source(source)
  1674. return check