run_runtime_init.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. """Writable runner initialization, separate from process ownership."""
  2. from __future__ import annotations
  3. import logging
  4. logger = logging.getLogger("gateway.run") # log-record parity with gateway/run.py
  5. class GatewayRuntimeInitMixin:
  6. def _init_runtime_settings(self) -> None:
  7. """Load ephemeral per-call config (prefill, reasoning, busy modes, timeouts, routing)."""
  8. from gateway.run import (Dict, Optional, _load_gateway_config)
  9. self._prefill_messages = self._load_prefill_messages()
  10. self._reasoning_config = self._load_reasoning_config()
  11. self._service_tier = self._load_service_tier()
  12. self._show_reasoning = self._load_show_reasoning()
  13. self._busy_input_mode = self._load_busy_input_mode()
  14. self._busy_text_mode = self._load_busy_text_mode()
  15. # Secondary-profile busy modes snapshotted at multiplex startup; handlers never reread config.
  16. self._busy_input_modes_by_profile: Dict[str, str] = {}
  17. self._busy_text_modes_by_profile: Dict[str, str] = {}
  18. self._busy_text_timing = self._busy_text_timing_from_config(_load_gateway_config())
  19. self._busy_text_timing_by_profile: Dict[str, tuple[float, float]] = {}
  20. self._human_delay = self._human_delay_from_config(_load_gateway_config())
  21. self._human_delay_by_profile: Dict[str, Optional[tuple[int, int]]] = {}
  22. self._restart_drain_timeout = self._load_restart_drain_timeout()
  23. # Live launchd ``ExitTimeOut`` for this job (None when not launchd-owned). Read once at
  24. # boot — launchd fixes it at load — and applied only to signal-driven stops, which are the
  25. # only stops launchd times. See _load_launchd_exit_timeout().
  26. self._stop_requested_by_signal = False
  27. self._launchd_exit_timeout_s = self._load_launchd_exit_timeout(self._restart_drain_timeout)
  28. self._restart_after_turn_timeout = self._load_restart_after_turn_timeout()
  29. self._cron_drain_timeout = self._load_cron_drain_timeout()
  30. self._signal_interrupt_grace_timeout = self._load_signal_interrupt_grace_timeout()
  31. self._provider_routing = self._load_provider_routing()
  32. self._fallback_model = self._load_fallback_model()
  33. def _init_session_store(self) -> None:
  34. """Build the SessionStore (with process-registry reset guard), its async facade and the router."""
  35. from gateway.run import (AsyncSessionStore, DeliveryRouter, SessionStore)
  36. from tools.process_registry import process_registry
  37. self.session_store = SessionStore(
  38. self.config.sessions_dir, self.config,
  39. has_active_processes_fn=lambda key: process_registry.has_active_for_session(
  40. key))
  41. # Loop-side boundary: sync helpers use ``session_store`` directly; async handlers await this facade.
  42. self._async_session_store = AsyncSessionStore(self.session_store)
  43. self.delivery_router = DeliveryRouter(self.config)
  44. def _init_lifecycle_state(self) -> None:
  45. """Initialise run/exit/restart flags, per-session state, and completion-delivery bookkeeping."""
  46. from gateway.run import (Dict, List, MessageEvent, Optional, OrderedDict, Platform, SessionSource, SessionState, SessionTurnLeaseRegistry, asyncio, concurrent, threading, time)
  47. self._running = self._exit_cleanly = self._exit_with_failure = self._draining = False
  48. self._gateway_loop: Optional[asyncio.AbstractEventLoop] = None
  49. self._shutdown_event = asyncio.Event()
  50. self._exit_reason: Optional[str] = None
  51. self._exit_code: Optional[int] = None
  52. self._profile_failed_platforms: Dict[str, Dict[Platform, asyncio.Task]] = {}
  53. self._profile_bot_platforms: Dict[str, set[Platform]] = {}
  54. self._systemd_watchdog = None
  55. # External (NAS-driven) drain, distinct from one-way ``_draining``: set while ``.drain_request.json``
  56. # exists — NEW turns refused, process stays up, removing the marker reverts to ``running``.
  57. self._external_drain_active = False
  58. # ``_signal_initiated_shutdown``: SIGTERM/SIGINT with no planned-stop/takeover marker (container,
  59. # OOM, bare kill); _stop_impl must NOT persist gateway_state=stopped or container_boot won't restart.
  60. self._restart_requested = self._signal_initiated_shutdown = self._restart_task_started = False
  61. self._restart_detached = self._restart_via_service = self._detached_restart_helper_started = False
  62. self._restart_command_source: Optional[SessionSource] = None
  63. # Construction clock: bounds the /restart redelivery guard's window (missing dedup marker = stale).
  64. self._startup_time: float = time.time()
  65. # True when booted from a chat /restart (.restart_notify.json existed). One-shot signal so the
  66. # marker-missing fallback suppresses a /restart only when we KNOW we just restarted.
  67. self._booted_from_restart: bool = False
  68. self._stop_task: Optional[asyncio.Task] = None
  69. self._restart_task: Optional[asyncio.Task] = None
  70. self._executor_lock = threading.Lock()
  71. self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
  72. # Best-effort session housekeeping runs on its OWN pool; see _run_housekeeping_in_executor.
  73. self._housekeeping_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
  74. # Set on gateway stop so the recreate-on-shutdown path can't resurrect the pool.
  75. self._executor_closing = False
  76. # ALL per-session state lives here (gateway/session_state.py); use _session_state / _peek_session_state.
  77. self._sessions: Dict[str, SessionState] = {}
  78. # Per-SESSION_ID turn lease: serializes [load history → run → flush] when two ROUTING KEYS resolve
  79. # to one session_id (switch_session's many-to-one mapping), which routing-key guards cannot see.
  80. self._turn_leases = SessionTurnLeaseRegistry()
  81. # Stall-notified keys clear when pending clears / activity resumes / conversation boundary.
  82. # Held turn-lease tokens live on SessionState.turn.lease_tokens keyed by run generation, so a
  83. # stale unwind can never free a newer turn's lease (#28686). Runner-level queued interrupt text lives on
  84. # SessionState.persistent.pending_command_text (NOTE: distinct from the adapter-level
  85. # _pending_messages Dict[str, MessageEvent] in gateway/platforms/base.py, which shares the legacy
  86. # name). Last successfully-resolved (non-empty) model, keyed by session. Used as a fallback when a
  87. # fresh config read transiently returns an empty model (e.g. an mtime-keyed config-cache miss during
  88. # a post-interrupt recovery turn). Without this, the agent is built with model="" and every API call
  89. # fails HTTP 400 "No models provided" — the session goes silent until the user manually re-sends.
  90. # See #35314. The ``"*"`` session entry holds a process-wide last-known-good for sessions seen for
  91. # the first time. Lives on SessionState.conversation.last_resolved_model. Overflow buffer for
  92. # explicit /queue commands. The adapter-level _pending_messages dict is a single slot per session
  93. # (designed for "next-turn" follow-ups where repeated sends collapse into one event). /queue has
  94. # different semantics: each invocation must produce its own full agent turn, in FIFO order, with no
  95. # merging. When the slot is occupied, additional /queue items land here and are promoted
  96. # one-at-a-time after each run's drain. Cleared on /new and /reset. /model and other mid-session
  97. # operations preserve the queue. Lives on SessionState.conversation.queued_events; native image
  98. # paths, busy-ack debounce timestamps and the monotonic run-generation counter (#28686, NEVER reset)
  99. # live on SessionState too. See gateway.session_stall.
  100. self._session_stall_notified: Dict[str, bool] = {}
  101. # Consecutive "persisted transcript lagged live cached history" turns per session key; see
  102. # run_turn_runner._load_turn_history (#114266). Cleared on /new.
  103. self._transcript_lag_streaks: Dict[str, int] = {}
  104. # Startup restore gate: while restart-interrupted sessions auto-resume, real inbound messages
  105. # queue instead of competing with the synthetic resume turns; drained after all resume tasks end.
  106. self._startup_restore_in_progress = False
  107. self._startup_restore_queue: List[MessageEvent] = []
  108. self._startup_restore_tasks: List[asyncio.Task] = []
  109. # Set by start_gateway() only for an explicit ``--replace`` launch; scoped to each adapter's
  110. # cold-start connect and removed before any reconnect can run.
  111. self._platform_lock_takeover_on_start = False
  112. # Capped LRU of live SessionSources for fallback routing (shutdown notices, synthetic events) when
  113. # the persisted origin is missing and _parse_session_key can't recover thread_id.
  114. self._session_sources: OrderedDict[str, SessionSource] = OrderedDict()
  115. self._session_sources_max = 512
  116. # Lifecycle-scoped completion dedup: closes queue/watcher races inside one gateway without claiming
  117. # exactly-once across a crash; durable replay state stays owned by tools.async_delegation.
  118. self._completion_delivery_lock = threading.Lock()
  119. self._completion_deliveries_inflight: set[tuple[str, str, object]] = set()
  120. self._completion_deliveries_delivered: OrderedDict[tuple[str, str, object], None] = OrderedDict()
  121. self._completion_delivery_retention = 2048
  122. # Agent-triggered terminal completions from one conversation often land in the same scheduler
  123. # tick; hold them briefly so the agent gets one synthetic turn instead of one per process.
  124. # See #70300.
  125. self._completion_notification_batches: dict[tuple[str, ...], list[tuple[str, dict, asyncio.Future]]] = {}
  126. self._completion_notification_batch_tasks: dict[tuple[str, ...], asyncio.Task] = {}
  127. self._completion_notification_batch_flush_tasks: set[asyncio.Task] = set()
  128. self._completion_notification_batch_window = 0.1
  129. self._completion_notification_batches_stopping = False
  130. def _init_runtime_caches(self) -> None:
  131. """Agent cache, profile identity, Teams runtime, failed-platform tracking, slash-confirm counter."""
  132. from gateway.run import (Any, Dict, Optional, OrderedDict, Platform, threading)
  133. # AIAgent per session preserves prompt caching (fresh agent per message ~10x cost on Anthropic).
  134. # Value: (AIAgent, config_signature); LRU cap in _enforce_agent_cache_cap, TTL in expiry watcher.
  135. self._agent_cache: OrderedDict[str, tuple] = OrderedDict()
  136. self._agent_cache_lock = threading.Lock()
  137. # Launch-time identity of the profile that owns ``self.adapters``; ``_authorization_adapter``
  138. # compares against this rather than the per-turn ``_active_profile_name()``. A multiplex
  139. # host's primary map is always the default profile, even when a named profile launched
  140. # the process (that launcher is a secondary adapter owner).
  141. launch = self._active_profile_name()
  142. self._kanban_notifier_profile = launch
  143. self._primary_profile_name = (
  144. "default" if getattr(self.config, "multiplex_profiles", False) else launch
  145. )
  146. # Teams meeting pipeline runtime (bound later when msgraph_webhook adapter exists).
  147. self._teams_pipeline_runtime = None
  148. self._teams_pipeline_runtime_error: Optional[str] = None
  149. # Failed-to-connect platforms for background reconnection: Platform -> {config, attempts, next_retry}
  150. self._failed_platforms: Dict[Platform, Dict[str, Any]] = {}
  151. # Strong refs to detached fatal-error handler tasks so the loop can't GC them mid-run.
  152. self._fatal_handler_tasks: set = set()
  153. # Slash-confirm state lives in tools.slash_confirm (module-level) so adapters resolve callbacks
  154. # without a runner backref; local counter keeps confirm_ids compact (64-byte callback_data caps).
  155. import itertools
  156. self._slash_confirm_counter = itertools.count(1)
  157. def _init_session_db(self) -> None:
  158. """Open the session DB for the active scope and run opportunistic state.db / checkpoint maintenance."""
  159. from gateway.run import (Any, Dict, Path, _SESSION_DB_UNPINNED, _housekeeping_chore, _housekeeping_state_db_maintenance, _launch_sessions_dir, threading)
  160. # Session DB is a property caching one AsyncSessionDB per path (a handle bound here would pin the
  161. # root home under multiplex); priming here keeps startup diagnostics at init.
  162. # Initialize session database for session_search tool support. Same frozen-handle class of bug as
  163. # SessionStore._db (#88532): a handle bound here is pinned to the process's root home, but /resume,
  164. # /title, /history and session search all run inside _profile_runtime_scope on a multiplexed gateway
  165. # and must see that profile's own state.db.
  166. self._session_db_pinned: Any = _SESSION_DB_UNPINNED
  167. self._session_db_handles: Dict[Path, Any] = {}
  168. self._session_db_handles_lock = threading.Lock()
  169. from gateway.session_db_recovery import RecoverableHandleCache
  170. self._session_db_handle_cache = RecoverableHandleCache(
  171. handles=self._session_db_handles, lock=self._session_db_handles_lock)
  172. try:
  173. self._open_session_db_for_active_scope(raise_on_error=True)
  174. except Exception as e:
  175. # WARNING (not DEBUG) so it lands in errors.log; else an NFS HERMES_HOME silently loses /resume etc.
  176. logger.warning("SQLite session store not available: %s", e, exc_info=True)
  177. self._session_db_init_error = str(e) # surfaced on the home channel(s) once connected
  178. # Opportunistic state.db maintenance (prune + optional VACUUM), at most once per min_interval_hours.
  179. # A few blocking seconds per day is fine for a long-lived gateway; failures log, never raise.
  180. # Surface the failure to the user via their home channel(s) once the gateway connects. Without this,
  181. # state.db corruption or NFS/SMB lock failures silently degrade the entire gateway — messages may
  182. # flow but nothing is persisted, and the user has no indication until they try /resume and find
  183. # nothing (#88235).
  184. # Once per SERVED profile, each under its own scope: both the store and the ``sessions:``
  185. # config that governs it must be the profile's own. Bound to ``self._session_db`` this ran
  186. # against the construction-time launch home only, so a multiplexed secondary profile's
  187. # state.db was never pruned or vacuumed by anybody, and the launch profile's
  188. # retention_days/auto_prune decided whether it happened at all.
  189. from gateway.run_profile_reconcile import _for_each_served_profile
  190. _launch_sessions = _launch_sessions_dir(self.config) # resolved OUTSIDE any profile scope
  191. _housekeeping_chore(
  192. "state.db startup maintenance",
  193. lambda: _for_each_served_profile(
  194. self, lambda _label: _housekeeping_state_db_maintenance(_launch_sessions)))
  195. # Checkpoint store pruning is a housekeeping chore (``_housekeeping_checkpoint_prune``), not a
  196. # constructor step: its ``git gc`` repacks the whole store (tens of seconds on a GB store) and
  197. # here it ran before the control socket, adapters and the code_sha stamp — so the first
  198. # restart of the day (the ``hermes update`` one) looked hung and failed fleet verification.
  199. def _init_registries_and_clocks(self) -> None:
  200. """Pairing stores, hook registry, voice modes, background-task set, liveness and idle clocks."""
  201. from gateway.run import (Dict, List, Optional, asyncio, time)
  202. # ``pairing_store``: global/default store (CLI, callers without profile context); ``pairing_stores``:
  203. # per-profile map ``authz_mixin._is_user_authorized`` routes through (one whitelist per profile).
  204. from gateway.pairing import PairingStore
  205. from gateway.hooks import ProfileHookRegistries
  206. self.pairing_store = PairingStore()
  207. self.pairing_stores: Dict[str, PairingStore] = {}
  208. # One HookRegistry per served profile home, resolved from the active scope at emit time.
  209. self.hooks = ProfileHookRegistries()
  210. # Per-chat voice reply mode: "off" | "voice_only" | "all"
  211. self._voice_mode: Dict[str, str] = self._load_voice_modes()
  212. # Per-(bot, guild, text channel, user) transcript dedup: voice/STT can emit one utterance twice.
  213. self._recent_voice_transcripts: Dict[tuple, List[tuple[float, str]]] = {}
  214. # Background tasks kept referenced so they are not garbage-collected mid-execution.
  215. self._background_tasks: set = set()
  216. # Event-loop liveness heartbeat: rewritten every 30s while the loop dispatches; supervisors use
  217. # the file mtime / updated_at to tell "process alive" from "loop frozen".
  218. # See #66892.
  219. self._gateway_started_at: float = time.time()
  220. self._loop_heartbeat_task: Optional[asyncio.Task] = None
  221. self._loop_floor_timer_handle = self._loop_liveness_watchdog = None
  222. # scale-to-zero: gateway-scoped "last inbound seen" clock, stamped in _handle_message (the single
  223. # inbound chokepoint) and seeded to "now" so a fresh gateway isn't idle from epoch.
  224. self._last_inbound_at: float = time.time()
  225. # Re-arm cooldown after a wake so we don't go dormant again before the drained backlog updates
  226. # the clock; and a one-shot latch so the "platform owns the suspend" notice logs once.
  227. self._scale_to_zero_cooldown_until: float = 0.0
  228. self._scale_to_zero_no_suspend_logged: bool = False
  229. self._scale_to_zero_direct_platform_logged: bool = False