run_adapters_recovery.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425
  1. """The multiplex secondary-profile reconnect lane for GatewayRunner (mixin bound via the MRO).
  2. A secondary profile's platform lands here when its connect failed, it went fatal mid-run, or its
  3. adapter was never built because no plugin registered it in that profile (its load failed or overran
  4. its deadline, #137268). One task per (profile, platform) rebuilds it under that profile's own runtime
  5. scope until it connects or can no longer heal.
  6. Split out of ``gateway/run_adapters.py`` (its size ratchet is at its target). ``gateway.run``
  7. internals are imported lazily inside method bodies (import cycle), so ``patch("gateway.run.X")``
  8. keeps intercepting them at call time.
  9. """
  10. from __future__ import annotations
  11. import asyncio
  12. import functools
  13. import logging
  14. import time
  15. from typing import Optional
  16. from gateway.config import Platform
  17. from gateway.platforms.base import BasePlatformAdapter
  18. from gateway.platforms.helpers import carry_inbound_dedup, hand_over_held_inbound
  19. from gateway.restart import is_global_startup_conflict
  20. from gateway.run_shutdown import _log_suppressed
  21. logger = logging.getLogger("gateway.run")
  22. _MAX_PLUGIN_LOAD_REARMS = 3 # per queued platform: transient load failures heal, a broken plugin stops re-importing
  23. def _adapter_unavailable_message(platform: Platform, *, retrying: bool = True) -> str:
  24. """Actionable ``adapter_unavailable`` status text, shared by startup and the reconnect watchers so
  25. ``hermes status`` keeps the plugin/deps/credentials hint after the first retry."""
  26. message = (
  27. f"No adapter available for enabled {platform.value}; check the plugin, dependencies, and credentials."
  28. )
  29. return f"{message} Retrying in the background." if retrying else message
  30. class GatewaySecondaryReconnectMixin:
  31. """Queue, rebuild and reconnect a secondary profile's platform, plus the capped plugin-load re-arm
  32. that both reconnect lanes (launch profile and secondaries) share."""
  33. def _mark_adapter_unavailable(
  34. self, status_key: str, platform: Platform, *, retrying: bool, needs_attention: Optional[bool] = None,
  35. ) -> None:
  36. """Stamp ``adapter_unavailable`` runtime status (``retrying`` or ``fatal``) with the actionable hint, for
  37. an enabled platform whose ``_create_adapter`` returned None. Shared by both lanes' startup and retries."""
  38. self._update_platform_runtime_status(
  39. status_key, platform_state="retrying" if retrying else "fatal", error_code="adapter_unavailable",
  40. error_message=_adapter_unavailable_message(platform, retrying=retrying), needs_attention=needs_attention,
  41. )
  42. def _stand_down_unbuilt_secondary(self, profile_name: str, platform: Platform, reason: str) -> None:
  43. """A never-built platform whose retry gave up for a config reason (disabled, credential removed): replace
  44. the ``retrying`` stamp startup gave it, or status keeps promising a retry nothing runs."""
  45. self._update_platform_runtime_status(
  46. f"{profile_name}:{platform.value}", platform_state="disabled", error_code=None, error_message=reason,
  47. needs_attention=False,
  48. )
  49. async def _rearm_failed_plugin_load(self, platform: Platform, info: dict) -> None:
  50. """Re-arm a failed plugin load for a queued platform, capped per queue entry, so the next attempt's
  51. import can succeed. The CALLER's runtime scope picks the plugin manager (one per profile home).
  52. Off the loop: the re-arm takes the manager's discovery lock, which a sweep or deferred load holds."""
  53. from hermes_cli.plugins import get_plugin_manager
  54. rearms = info.get("load_rearms", 0)
  55. if rearms < _MAX_PLUGIN_LOAD_REARMS and await self._run_in_executor_with_context(
  56. get_plugin_manager().rearm_failed_platform, platform.value):
  57. info["load_rearms"] = rearms + 1
  58. def _claim_secondary_platform(
  59. self, profile_name: str, profile_home, platform: Platform, adapter: BasePlatformAdapter,
  60. claimed: dict[tuple, str],
  61. ) -> Optional[tuple]:
  62. """Refuse a secondary adapter whose credential/listener another profile owns (a second poller on
  63. one bot, a second bind on one port); otherwise record the profile as the bot's owner and return
  64. its ``(credential_claim, listener_claim)``. None = refused (parked fatal)."""
  65. credential_claim, listener_claim = self._adapter_claims(platform, adapter)
  66. owner_name = claimed.get(credential_claim) if credential_claim is not None else None
  67. owner_origin, incoming_origin = (None, None)
  68. if owner_name:
  69. owner_origin, incoming_origin = self._duplicate_credential_origins(
  70. owner_name, profile_name, profile_home, platform, adapter,
  71. )
  72. if self._refuse_duplicate_claim(
  73. credential_claim, claimed, profile_name, platform, "credential",
  74. owner_origin=owner_origin, incoming_origin=incoming_origin,
  75. ) or self._refuse_duplicate_claim(listener_claim, claimed, profile_name, platform, "listener"):
  76. return None
  77. # The profile owns this bot from here on, connected or not: cron and the delivery drain read
  78. # ownership here from another thread, never from the live map a fatal error rewrites.
  79. bot_platforms = getattr(self, "_profile_bot_platforms", None)
  80. if bot_platforms is None:
  81. bot_platforms = self._profile_bot_platforms = {}
  82. bot_platforms.setdefault(profile_name, set()).add(platform)
  83. return credential_claim, listener_claim
  84. def _disown_secondary_platform(self, profile_name: str, platform: Platform) -> None:
  85. """Undo ``_claim_secondary_platform``'s ownership record for a never-built platform that gave up for good:
  86. cron and the delivery drain read it to decide that the profile owns the bot."""
  87. (getattr(self, "_profile_bot_platforms", None) or {}).get(profile_name, set()).discard(platform)
  88. def _adapter_claims(self, platform: Platform, adapter: BasePlatformAdapter) -> tuple[Optional[tuple], Optional[tuple]]:
  89. """``(credential_claim, listener_claim)`` of one adapter; either may be None. Never raises: unserve and a
  90. fatal teardown compute it after the adapter left the map."""
  91. return self._adapter_credential_claim(platform, adapter), self._adapter_listener_claim(platform, adapter)
  92. def _hold_secondary_claims(self, profile_name: str, platform: Platform, owner, claims: tuple) -> None:
  93. """Hold a secondary platform's credential/listener claims while it is not in ``_profile_adapters``
  94. (initial connect, failed boot connect awaiting the handoff, fatal teardown, unserve teardown, queued
  95. to reconnect, or mid-heal), so no other profile takes the same bot meanwhile. A task owner's hold is
  96. released when that task finishes, however it finishes (cancelled before it ever ran included); an
  97. adapter owner (being connected or torn down) releases its own, so it never drops a newer hold on the
  98. same platform. Unserve drops all of the profile's holds once its adapters are torn down."""
  99. held = getattr(self, "_secondary_held_claims", None)
  100. if held is None:
  101. held = self._secondary_held_claims = {}
  102. key = (profile_name, platform)
  103. previous = held.get(key)
  104. held[key] = (owner, claims)
  105. if isinstance(owner, asyncio.Future) and (previous is None or previous[0] is not owner):
  106. owner.add_done_callback(lambda done: self._release_secondary_claims(profile_name, platform, done))
  107. def _release_secondary_claims(self, profile_name: str, platform: Platform, owner) -> None:
  108. """Drop the hold on (profile, platform) if ``owner`` still owns it (a newer owner keeps its hold)."""
  109. held = getattr(self, "_secondary_held_claims", None) or {}
  110. entry = held.get((profile_name, platform))
  111. if entry is not None and entry[0] is owner:
  112. del held[(profile_name, platform)]
  113. def _held_claim_owners(self, exclude: Optional[tuple] = None) -> dict[tuple, str]:
  114. """Claim -> profile for every held secondary claim but the ``exclude`` (profile, platform) hold."""
  115. held = getattr(self, "_secondary_held_claims", None) or {}
  116. return {
  117. claim: key[0] for key, (_owner, claims) in held.items() if key != exclude
  118. for claim in claims if claim is not None
  119. }
  120. def _current_claim_owners(self, exclude: Optional[tuple] = None) -> dict[tuple, str]:
  121. """Who owns each bot/listener right now: held claims (but ``exclude``'s) plus everything live. Read at
  122. the moment of a claim check, never from a snapshot taken before an await (an owner may connect or let
  123. go meanwhile)."""
  124. active = getattr(self, "_primary_profile_name", None) or "default"
  125. return {**self._held_claim_owners(exclude), **self._live_resource_claims(active)}
  126. def _note_unbuilt_secondary_platform(self, profile_name: str, platform: Platform, heals: bool) -> None:
  127. """A secondary platform whose ``_create_adapter`` returned None at startup: say why, stamp its
  128. ``<profile>:<platform>`` status, and queue it when it can heal (no plugin registered it in this
  129. profile's scope, so a later import can still succeed). Anything else needs a config change:
  130. retrying it would only re-warn at the backoff cap (#5196)."""
  131. pv = platform.value
  132. if heals:
  133. logger.warning(
  134. "[MULTIPLEX] Profile '%s': no plugin registered platform '%s' - is the plugin installed? (a load "
  135. "that failed or overran its deadline also leaves it unregistered); retrying in the background",
  136. profile_name, pv,
  137. )
  138. else:
  139. logger.warning(
  140. "[MULTIPLEX] Profile '%s': skipping platform '%s' - adapter creation returned None",
  141. profile_name, pv,
  142. )
  143. self._mark_adapter_unavailable(f"{profile_name}:{pv}", platform, retrying=heals, needs_attention=True)
  144. if heals:
  145. self._schedule_secondary_profile_startup_reconnect(profile_name, platform, None)
  146. async def _secondary_reconnect_attempt(self, profile_name: str, platform: Platform, predecessor=None):
  147. """One scoped attempt to rebuild+connect a secondary adapter → ``(adapter, success)``;
  148. ``(None, None)`` = give up for good (disabled, credential removed, refused as a duplicate, or
  149. adapter unavailable for a reason only a config change fixes); ``(None, False)`` = no adapter yet
  150. but the platform can still heal (no plugin registered it, so a later import can succeed). Caller
  151. tears down a RETURNED adapter; one whose configure/connect raised is torn down here."""
  152. from gateway.platform_registry import platform_registry
  153. from gateway.run import _platform_has_bot_credential, _profile_runtime_scope
  154. # Lazy + per-attempt: keeps test monkeypatches on these modules live.
  155. from hermes_cli.profiles import get_profile_dir
  156. from hermes_cli.env_loader import hydrate_profile_secret_sources
  157. from gateway.config import load_gateway_config
  158. profile_home = get_profile_dir(profile_name)
  159. # Hydrate external secret sources off-loop so they cannot starve heartbeats.
  160. await asyncio.to_thread(hydrate_profile_secret_sources, profile_home)
  161. with _profile_runtime_scope(profile_home, hydrate_secrets=False):
  162. profile_config = load_gateway_config().platforms.get(platform)
  163. if profile_config is None or not profile_config.enabled:
  164. if predecessor is None:
  165. self._stand_down_unbuilt_secondary(profile_name, platform, "disabled in config")
  166. return None, None
  167. # Startup credential gate mirror: a removed credential must not rebuild.
  168. # Mirrors the startup credential gate (#84079): a credential removed from this profile's scope
  169. # must not rebuild an adapter that would fan out turns.
  170. if not _platform_has_bot_credential(platform, profile_config):
  171. logger.info(
  172. "Secondary %s reconnect skipped: no bot credential (profile: %s)",
  173. platform.value, profile_name,
  174. )
  175. if predecessor is None:
  176. self._stand_down_unbuilt_secondary(profile_name, platform, "no bot credential configured")
  177. return None, None
  178. # A re-armed plugin import + register() joins its load deadline: run it off the event loop,
  179. # in this profile's scope (the registry keeps deferred loaders per profile home).
  180. await self._run_in_executor_with_context(platform_registry.get, platform.value)
  181. adapter = self._create_adapter(platform, profile_config)
  182. if adapter is None:
  183. if self._adapter_may_heal(platform, profile_config):
  184. logger.info("Secondary %s: no adapter yet (profile: %s)", platform.value, profile_name)
  185. return None, False
  186. logger.warning(
  187. "Secondary %s reconnect skipped: adapter unavailable (profile: %s)",
  188. platform.value, profile_name,
  189. )
  190. self._mark_adapter_unavailable(
  191. f"{profile_name}:{platform.value}", platform, retrying=False, needs_attention=True)
  192. return None, None
  193. if predecessor is None:
  194. # Queued without an adapter, so it never passed the startup ownership check: run it before
  195. # connect() against live and held owners (but this platform's own hold from an earlier tick),
  196. # and hold the claim (no await between the check and the hold) so a second healing profile
  197. # with the same bot is refused meanwhile.
  198. claims = self._claim_secondary_platform(
  199. profile_name, profile_home, platform, adapter,
  200. self._current_claim_owners(exclude=(profile_name, platform)))
  201. if claims is None:
  202. return None, None
  203. self._hold_secondary_claims(profile_name, platform, asyncio.current_task(), claims)
  204. else: # re-hold what the rebuilt adapter claims now (its token may have changed; as on main, a
  205. # reconnect with a predecessor is not re-checked against other owners)
  206. self._hold_secondary_claims(
  207. profile_name, platform, asyncio.current_task(), self._adapter_claims(platform, adapter))
  208. carry_inbound_dedup(predecessor, adapter)
  209. try:
  210. self._configure_profile_adapter(adapter, profile_name, platform)
  211. success = await self._connect_adapter_with_timeout(adapter, platform, is_reconnect=True)
  212. except BaseException:
  213. # Caller never sees this adapter; release its partial resources and return its held inbound.
  214. await self._safe_adapter_disconnect(adapter, platform)
  215. hand_over_held_inbound(adapter, predecessor)
  216. raise
  217. return adapter, success
  218. def _end_secondary_reconnect(self, profile_name: str, platform: Platform, predecessor, adapter, success) -> bool:
  219. """Decide whether a reconnect ends after an attempt that installed nothing, and settle ownership: it
  220. gave up for good (``(None, None)``: needs a config change), its connected adapter was not installed (a
  221. newer task won the slot, or shutdown began), or it hit a non-retryable fatal. A never-built platform
  222. that ends for good (not the connected case) drops its bot ownership, which cron and the delivery drain
  223. read."""
  224. if adapter is None:
  225. final = success is None
  226. elif success:
  227. return True # connected but not installed: a newer task owns the platform, or shutdown began
  228. else:
  229. final = getattr(adapter, "has_fatal_error", False) and not getattr(adapter, "fatal_error_retryable", True)
  230. if final and predecessor is None:
  231. self._disown_secondary_platform(profile_name, platform)
  232. return final
  233. async def _run_secondary_profile_reconnect(
  234. self, profile_name: str, platform: Platform, predecessor=None
  235. ) -> None:
  236. """Reconnect a retryable secondary adapter under its own profile scope."""
  237. from gateway.run import _profile_runtime_scope, _reconnect_backoff
  238. attempts = 0
  239. # Same escalation shape as the primary queue entry; ``queued_at`` is this task's start.
  240. queue_info = {"queued_at": time.monotonic(), "attempts": 0}
  241. current_task = asyncio.current_task()
  242. try:
  243. while self._running:
  244. adapter = None
  245. rearm = False
  246. try:
  247. adapter, success = await self._secondary_reconnect_attempt(
  248. profile_name, platform, predecessor
  249. )
  250. if adapter is None:
  251. if self._end_secondary_reconnect(profile_name, platform, predecessor, None, success):
  252. return
  253. rearm = True # no adapter yet but it can heal: re-arm the failed load, then retry
  254. elif success and self._running:
  255. profile_map = self._profile_adapters.setdefault(profile_name, {})
  256. if platform not in profile_map:
  257. profile_map[platform] = adapter
  258. hand_over_held_inbound(predecessor, adapter)
  259. self._sync_voice_mode_state_to_adapter(adapter)
  260. logger.info("✓ %s reconnected (profile: %s)", platform.value, profile_name)
  261. await self._redeliver_failed_obligations_for_platform(
  262. platform, profile=profile_name
  263. )
  264. # What a primary reconnect replays too: the owed notice spans served profiles' home
  265. # channels, and sessions boot skipped for this offline adapter wait for this call.
  266. self._schedule_planned_restart_replay()
  267. try:
  268. self._schedule_resume_pending_sessions(platform=platform)
  269. except Exception:
  270. logger.debug("resume-pending reschedule after %s reconnect failed (profile: %s)",
  271. platform.value, profile_name, exc_info=True)
  272. return
  273. if adapter is not None:
  274. # Not installed (newer reconnect won the slot, shutdown began, or connect failed):
  275. # release partial resources and return its held inbound; stop only for a non-retryable fatal.
  276. await self._safe_adapter_disconnect(adapter, platform)
  277. hand_over_held_inbound(adapter, predecessor)
  278. if self._end_secondary_reconnect(profile_name, platform, predecessor, adapter, success):
  279. return
  280. except BaseException as exc:
  281. if adapter is not None:
  282. await self._safe_adapter_disconnect(adapter, platform)
  283. if self._profile_adapters.get(profile_name, {}).get(platform) is not adapter:
  284. hand_over_held_inbound(adapter, predecessor) # an installed adapter owns its queue
  285. if not isinstance(exc, Exception):
  286. raise # CancelledError (and other BaseExceptions) propagate after release
  287. logger.debug(
  288. "Secondary %s reconnect attempt failed (profile: %s)", platform.value,
  289. profile_name, exc_info=True,
  290. )
  291. if not self._running:
  292. return
  293. attempts += 1
  294. queue_info["attempts"] = attempts
  295. profile_home = self._routed_profile_home(profile_name)
  296. # The attempt above already hydrated this profile's secret sources off-loop. The re-arm runs in
  297. # this scope too: the plugin manager whose load failed is this profile's, not the launch's.
  298. with self._scope_or_null(
  299. functools.partial(_profile_runtime_scope, hydrate_secrets=False), profile_home):
  300. if rearm:
  301. with _log_suppressed(logging.DEBUG, "plugin load re-arm for %s failed (profile: %s): %s",
  302. platform.value, profile_name):
  303. await self._rearm_failed_plugin_load(platform, queue_info)
  304. flagged = self._flag_reconnect_needs_attention(
  305. platform, queue_info, time.monotonic(), status_key=f"{profile_name}:{platform.value}")
  306. if flagged and rearm: # the flag rewrote the entry without an error code: put the hint back
  307. self._mark_adapter_unavailable(f"{profile_name}:{platform.value}", platform, retrying=True)
  308. backoff = _reconnect_backoff(attempts)
  309. logger.info(
  310. "Secondary %s reconnect retry in %ds (profile: %s)", platform.value, backoff, profile_name
  311. )
  312. await asyncio.sleep(backoff)
  313. finally:
  314. # Release our slot unless a newer task already owns it.
  315. pending = self._profile_failed_platforms
  316. profile_pending = pending.get(profile_name) if isinstance(pending, dict) else None
  317. if isinstance(profile_pending, dict):
  318. task = profile_pending.get(platform)
  319. if not isinstance(task, asyncio.Task) or task is current_task:
  320. profile_pending.pop(platform, None)
  321. if not profile_pending:
  322. pending.pop(profile_name, None)
  323. def _schedule_secondary_profile_startup_reconnect(
  324. self, profile_name: str, platform: Platform, adapter: Optional[BasePlatformAdapter] = None
  325. ) -> None:
  326. """Queue a cold-start reconnect: startup failures happen BEFORE ``_running`` flips True (the
  327. regular scheduler would drop them), so park a task and hand off once live. ``adapter`` is None
  328. when nothing was built at all (a platform whose creation may still heal)."""
  329. if not getattr(adapter, "fatal_error_retryable", True):
  330. return
  331. if adapter is not None and is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)):
  332. # A live foreign token holder is an ownership conflict, not a blip: park it fatal.
  333. logger.error(
  334. # Park it fatal (like ``duplicate_credential``) instead of retry-storming the token every
  335. # backoff (#83183).
  336. "[MULTIPLEX] Profile '%s': %s credential is held by another "
  337. "gateway (%s) — parked, not retried. %s", profile_name, platform.value,
  338. adapter.fatal_error_code, adapter.fatal_error_message or "",
  339. )
  340. self._mark_platform_fatal(f"{profile_name}:{platform.value}", adapter)
  341. return
  342. # The bridge belongs to this serve of the profile: once it is unserved (its map popped) or served again
  343. # (a new map), a bridge still waiting for startup to finish hands nothing over.
  344. profile_map = (getattr(self, "_profile_adapters", None) or {}).get(profile_name)
  345. def _handoff() -> None:
  346. if (getattr(self, "_profile_adapters", None) or {}).get(profile_name) is not profile_map:
  347. return
  348. try:
  349. self._schedule_secondary_profile_reconnect(profile_name, platform, adapter)
  350. except Exception:
  351. # A raise here would die as an unretrieved-task exception logged only at GC; surface it.
  352. logger.exception(
  353. "secondary-startup-reconnect handoff failed (profile=%s platform=%s)",
  354. profile_name, platform.value,
  355. )
  356. async def _await_running_then_schedule() -> None:
  357. if self._running: # fast path (also the only path for bare runners without _shutdown_event)
  358. _handoff()
  359. return
  360. # Poll (startup completion has no event); bounded so a wedged startup cannot spin.
  361. while not self._running and not self._shutdown_event.is_set():
  362. await asyncio.sleep(0.1)
  363. if self._running and not self._shutdown_event.is_set():
  364. _handoff()
  365. bridge = self._retain_background_task(asyncio.create_task(
  366. _await_running_then_schedule(),
  367. name=f"secondary-startup-reconnect:{profile_name}:{platform.value}",
  368. ))
  369. if adapter is not None: # it owns its bot from the boot sweep on; the handoff passes the hold on
  370. self._hold_secondary_claims(profile_name, platform, bridge, self._adapter_claims(platform, adapter))
  371. def _schedule_secondary_profile_reconnect(
  372. self, profile_name: str, platform: Platform, adapter: Optional[BasePlatformAdapter] = None
  373. ) -> None:
  374. """Schedule one runner-owned reconnect without sharing primary secrets. ``adapter`` is the
  375. failed predecessor, or None when the platform has no adapter yet (creation returned None for a
  376. platform that can still heal): the queued entry stops the runtime re-scan from building the same
  377. platform twice, and the attempt claims the bot before it connects. A profile being unserved
  378. (``_retiring_profiles``) gets none: its unserve has already popped the reconnects it cancels."""
  379. if (
  380. not self._running
  381. or not getattr(adapter, "fatal_error_retryable", True)
  382. or profile_name in (getattr(self, "_retiring_profiles", None) or ())
  383. ):
  384. return
  385. pending = self._profile_failed_platforms
  386. if not isinstance(pending, dict):
  387. pending = self._profile_failed_platforms = {}
  388. profile_pending = pending.setdefault(profile_name, {})
  389. if platform in profile_pending:
  390. return
  391. task = profile_pending[platform] = self._retain_background_task(asyncio.create_task(
  392. self._run_secondary_profile_reconnect(profile_name, platform, adapter),
  393. name=f"secondary-reconnect:{profile_name}:{platform.value}",
  394. ))
  395. if adapter is not None: # the failed predecessor still owns its bot while it waits to reconnect
  396. self._hold_secondary_claims(profile_name, platform, task, self._adapter_claims(platform, adapter))