run_config_loaders.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. """Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner.
  2. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
  3. ``gateway.run`` internals are imported lazily inside method bodies (import cycle),
  4. so ``patch("gateway.run.X")`` keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import os
  10. import time
  11. from contextlib import suppress
  12. from pathlib import Path
  13. from typing import TYPE_CHECKING, Any, Dict, List, Optional
  14. from gateway.config import Platform
  15. from gateway.restart import (
  16. DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT,
  17. DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
  18. DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, parse_cron_drain_timeout,
  19. parse_restart_after_turn_timeout, parse_restart_drain_timeout,
  20. launchd_service_label, parse_signal_interrupt_grace_timeout, read_launchd_exit_timeout_s,
  21. resolve_launchd_capped_drain,
  22. )
  23. from gateway.session import SessionSource
  24. from gateway.session_state import SERVICE_TIER_UNSET as _SERVICE_TIER_UNSET
  25. from hermes_cli.config import cfg_get, resolve_ephemeral_system_prompt_from_config
  26. from hermes_cli.fallback_config import get_fallback_chain
  27. from utils import is_truthy_value
  28. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  29. from gateway.run import GatewayRunner
  30. from gateway.run_turn_runner import TurnRunner
  31. # Log-record parity with the origin module.
  32. logger = logging.getLogger("gateway.run")
  33. DEFAULT_HUMAN_DELAY_RANGE_MS: tuple[int, int] = (800, 2500)
  34. _BUSY_INPUT_MODES = {"interrupt", "queue", "steer"}
  35. class GatewayConfigLoadersMixin:
  36. """Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner."""
  37. @staticmethod
  38. def _cfg_str(section: str, key: str) -> str:
  39. """``<section>.<key>`` from the gateway runtime config as a stripped string ("" when unset)."""
  40. from gateway.run import _load_gateway_config
  41. return str(cfg_get(_load_gateway_config(), section, key, default="") or "").strip()
  42. @classmethod
  43. def _env_or_cfg_str(cls, env_var: str, section: str, key: str) -> str:
  44. """Non-empty stripped env var, else :meth:`_cfg_str`."""
  45. return os.getenv(env_var, "").strip() or cls._cfg_str(section, key)
  46. @staticmethod
  47. def _load_prefill_messages() -> list[dict[str, Any]]:
  48. """Load ephemeral prefill messages from config or env var.
  49. HERMES_PREFILL_MESSAGES_FILE env wins, then top-level prefill_messages_file in config.yaml,
  50. then legacy agent.prefill_messages_file. Relative paths resolve from ~/.hermes/.
  51. """
  52. from gateway.run import _gateway_config_home, _load_gateway_config
  53. file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "")
  54. if not file_path:
  55. cfg = _load_gateway_config()
  56. file_path = str(
  57. cfg.get("prefill_messages_file", "") or cfg_get(cfg, "agent", "prefill_messages_file", default="") or ""
  58. )
  59. if not file_path:
  60. return []
  61. path = Path(file_path).expanduser()
  62. if not path.is_absolute():
  63. path = _gateway_config_home() / path
  64. if not path.exists():
  65. logger.warning("Prefill messages file not found: %s", path)
  66. return []
  67. try:
  68. with open(path, "r", encoding="utf-8-sig") as f:
  69. data = json.load(f)
  70. if not isinstance(data, list):
  71. logger.warning("Prefill messages file must contain a JSON array: %s", path)
  72. return []
  73. return data
  74. except Exception as e:
  75. logger.warning("Failed to load prefill messages from %s: %s", path, e)
  76. return []
  77. @staticmethod
  78. def _load_ephemeral_system_prompt() -> str:
  79. """HERMES_EPHEMERAL_SYSTEM_PROMPT env first (a local session's: its client's), then
  80. ``display.personality`` / ``agent.system_prompt``."""
  81. from agent.session_client_env import session_env
  82. from gateway.run import _load_gateway_config
  83. prompt = session_env("HERMES_EPHEMERAL_SYSTEM_PROMPT")
  84. if prompt:
  85. return prompt
  86. return resolve_ephemeral_system_prompt_from_config(_load_gateway_config())
  87. def _channel_override(self, platform: Platform, chat_id: str, thread_id, parent_id):
  88. """``channel_overrides`` entry for this channel/thread, or None (also when no config is bound)."""
  89. from gateway.run import _get_channel_override
  90. config = getattr(self, "config", None)
  91. if not config:
  92. return None
  93. return _get_channel_override(config, platform, chat_id, thread_id=thread_id, parent_id=parent_id)
  94. def _silence_allowed_for_source(self, source, display_kind, reply_expected, agent_result) -> bool:
  95. """Evaluate existing exemptions before consulting the optional room permission."""
  96. from gateway.response_filters import silence_allowed
  97. return silence_allowed(display_kind, reply_expected) or silence_allowed(
  98. display_kind, reply_expected, allow_silence=self._room_allow_silence(source),
  99. agent_result=agent_result)
  100. def _room_allow_silence(self, source: SessionSource) -> Optional[bool]:
  101. """Resolve the room policy in the serving profile, never the launch snapshot for a route."""
  102. from gateway.config import GatewayConfig
  103. from gateway.config_loader import merge_platform_sections, platform_section
  104. from gateway.run import _get_channel_override, _load_gateway_config
  105. from gateway.session_identity import identity_of
  106. config = getattr(self, "config", None)
  107. identity = identity_of(source)
  108. runtime = identity.runtime_profile if identity is not None else (source.profile or None)
  109. # ``self.config`` (including a ``gateway --config`` file) serves the primary profile; only a
  110. # turn routed to another profile reads that profile's config.yaml.
  111. routed = bool(runtime) and runtime != (getattr(self, "_primary_profile_name", None) or "default")
  112. try:
  113. if config is None or routed:
  114. raw = _load_gateway_config(self._resolve_profile_home_for_source(source) / "config.yaml")
  115. # The loader's own two steps: merge every nested platform block, then the overrides of
  116. # the block platform_section picks (a top-level ``<platform>:`` block first) win.
  117. # ``GatewayConfig.from_dict(raw)`` alone reads only the nested ``platforms`` form.
  118. name, gateway_section = str(source.platform.value), raw.get("gateway")
  119. overrides = (merge_platform_sections(raw, gateway_section, {}).get(name) or {}).get("channel_overrides")
  120. section, _ = platform_section(raw, name, (
  121. gateway_section.get("platforms") if isinstance(gateway_section, dict) else None))
  122. if isinstance(section, dict) and isinstance(section.get("channel_overrides"), dict):
  123. overrides = section["channel_overrides"]
  124. config = GatewayConfig.from_dict({"platforms": {name: {
  125. "channel_overrides": overrides if isinstance(overrides, dict) else {}}}})
  126. override = _get_channel_override(
  127. config, source.platform, source.chat_id,
  128. thread_id=source.thread_id, parent_id=source.parent_chat_id)
  129. except Exception:
  130. # No permission means the default reply rule; a lookup failure must not become an error reply.
  131. logger.warning("Room allow_silence lookup failed for %s; keeping the reply rule",
  132. source.chat_id, exc_info=True)
  133. return None
  134. return override.allow_silence if override is not None else None
  135. def _resolve_model_for_channel(
  136. self, platform: Platform, chat_id: str, *, user_config: Optional[dict] = None,
  137. thread_id: Optional[str] = None, parent_id: Optional[str] = None,
  138. ) -> str:
  139. """Resolve model for this channel: channel_overrides else global default.
  140. Precedence lives in :func:`hermes_cli.model_switch.resolve_effective_model` (shared with the
  141. API server so the surfaces cannot diverge). No session tier here: session /model overrides
  142. are applied later by ``_apply_session_model_override``.
  143. """
  144. from gateway.run import _resolve_gateway_model
  145. from hermes_cli.model_switch import resolve_effective_model
  146. return resolve_effective_model(
  147. None, # session tier applied downstream (_apply_session_model_override)
  148. self._channel_override(platform, chat_id, thread_id, parent_id),
  149. _resolve_gateway_model(user_config),
  150. )
  151. def _get_system_prompt_for_channel(
  152. self, platform: Platform, chat_id: str, *, thread_id: Optional[str] = None,
  153. parent_id: Optional[str] = None,
  154. ) -> str:
  155. """Ephemeral system prompt for this channel/thread.
  156. ``channel_overrides`` when set, else the gateway prompt resolved from the CURRENT profile's
  157. config on every call (callers run inside ``_profile_runtime_scope``, so routed multiplex
  158. profiles get their own personality/system_prompt and ``/personality`` edits apply next turn).
  159. Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in ``run_sync``.
  160. Callers run inside ``_profile_runtime_scope`` (``run_sync`` under ``_run_agent``), so a routed
  161. multiplex profile gets its own ``display.personality`` / ``agent.system_prompt`` instead of a
  162. boot-time snapshot of the launch profile's (#89161); ``/personality`` edits take effect on the next
  163. turn for the same reason.
  164. """
  165. override = self._channel_override(platform, chat_id, thread_id, parent_id)
  166. if override and override.system_prompt:
  167. return (override.system_prompt or "").strip()
  168. return self._load_ephemeral_system_prompt()
  169. @staticmethod
  170. def _load_reasoning_config(model: str = "") -> dict | None:
  171. """Reasoning effort from config.yaml via :func:`hermes_constants.resolve_reasoning_config`.
  172. Per-model override > global ``agent.reasoning_effort``; YAML False = disabled. Empty
  173. ``model`` uses ``model.default``.
  174. Closes #21256.
  175. """
  176. from gateway.run import _load_gateway_config
  177. from hermes_constants import resolve_reasoning_config
  178. return resolve_reasoning_config(_load_gateway_config(), model)
  179. @staticmethod
  180. def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]:
  181. """Parse `/reasoning` args into `(value, persist_global)`; `--global` anywhere persists to config."""
  182. import shlex
  183. text = str(raw_args or "").strip().replace("—", "--")
  184. if not text:
  185. return "", False
  186. try:
  187. tokens = shlex.split(text)
  188. except ValueError:
  189. tokens = text.split()
  190. value = " ".join(token for token in tokens if token != "--global")
  191. return value.strip().lower(), "--global" in tokens
  192. def _resolve_session_reasoning_config(
  193. self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None,
  194. model: str = "",
  195. ) -> dict | None:
  196. """Session ``/reasoning --session`` > per-model ``agent.reasoning_overrides`` > global.
  197. ``model`` must be the session's *effective* model (session ``/model`` override included);
  198. empty uses ``model.default``.
  199. """
  200. resolved_session_key = self._resolve_session_key_or_none(source, session_key)
  201. if resolved_session_key:
  202. _r_state = self._peek_session_state(resolved_session_key)
  203. if _r_state is not None and _r_state.conversation.reasoning_override is not None:
  204. return _r_state.conversation.reasoning_override
  205. from gateway.session_setting_commands import local_policy
  206. if (policy := local_policy(self, source)) is not None:
  207. # A local session runs its frozen (or followed) level, not the profile's.
  208. return policy.current_reasoning_config(self, resolved_session_key)
  209. return self._load_reasoning_config(model)
  210. def _set_session_reasoning_override(self, session_key: str, reasoning_config: Optional[dict]) -> None:
  211. """Set or clear the session-scoped reasoning override."""
  212. if not session_key:
  213. return
  214. # Per-session field write: a lazy ``_session_reasoning_overrides = {}`` init replaced the
  215. # WHOLE dict, racing concurrent sessions; a SessionState field reset cannot cross sessions.
  216. self._session_state(session_key).conversation.reasoning_override = (
  217. None if reasoning_config is None else dict(reasoning_config)
  218. )
  219. def _resolve_session_service_tier(self, source=None, session_key: Optional[str] = None) -> Optional[str]:
  220. """Effective service tier: a session-scoped /fast override beats the config default.
  221. The override stores "priority" or None (explicit normal), so presence — not truthiness — decides.
  222. """
  223. resolved_session_key = self._resolve_session_key_or_none(source, session_key)
  224. if resolved_session_key:
  225. _t_state = self._peek_session_state(resolved_session_key)
  226. if _t_state is not None and _t_state.conversation.service_tier_override is not _SERVICE_TIER_UNSET:
  227. return _t_state.conversation.service_tier_override
  228. from gateway.session_setting_commands import local_live_for_route, local_policy
  229. live = local_live_for_route(self, resolved_session_key)
  230. if live is not None and (policy := local_policy(self, live.source)) is not None:
  231. return policy.current_service_tier(self, resolved_session_key) # its frozen tier, not config.yaml's
  232. return self._load_service_tier()
  233. def _set_session_service_tier_override(self, session_key: str, service_tier, clear: bool = False) -> None:
  234. """Set ("priority" / None = explicit normal) or ``clear`` the session-scoped /fast override."""
  235. if not session_key:
  236. return
  237. # Presence-sensitive: "priority" or None (explicit normal) both count as an override; the
  238. # sentinel means "no override". Per-session field write: a lazy dict replace races sessions.
  239. self._session_state(session_key).conversation.service_tier_override = (
  240. _SERVICE_TIER_UNSET if clear else service_tier
  241. )
  242. @classmethod
  243. def _load_service_tier(cls) -> str | None:
  244. """``agent.service_tier`` parsed like the CLI (``hermes_cli.cli_config_load``); None when unset/unknown."""
  245. from hermes_cli.cli_config_load import _parse_service_tier_config
  246. return _parse_service_tier_config(cls._cfg_str("agent", "service_tier"))
  247. def _load_daybreak(self, source) -> bool:
  248. """``agent.daybreak`` (off unless explicitly true) for a turn from ``source``. Read per turn so the
  249. Desktop switch reaches the next turn of a running chat without rebuilding its cached agent. Only the
  250. owner's own chats carry it: Daybreak is the approved account's own access (OpenAI: no third-party
  251. or downstream use), and a messaging platform's chats, a hosted room or an A2A peer can be other people.
  252. LOCAL is the canonical transport, not the owner, so the route's frozen launch policy decides."""
  253. from gateway.run import _load_gateway_config
  254. from gateway.session_policy import DAYBREAK_LABELS, policy_for_source
  255. if getattr(source, "platform", None) != Platform.LOCAL:
  256. return False
  257. try:
  258. policy = policy_for_source(self, source)
  259. except Exception: # health: allow BLE001 -- an unreadable policy fails closed to standard requests
  260. return False
  261. if getattr(policy, "source", None) not in DAYBREAK_LABELS:
  262. return False
  263. return cfg_get(_load_gateway_config(), "agent", "daybreak") is True
  264. @staticmethod
  265. def _load_show_reasoning() -> bool:
  266. """``display.show_reasoning`` toggle."""
  267. from gateway.run import _load_gateway_config
  268. return is_truthy_value(cfg_get(_load_gateway_config(), "display", "show_reasoning"), default=False)
  269. @classmethod
  270. def _load_busy_input_mode(cls) -> str:
  271. """Gateway drain-time busy-input behavior from env/config (default ``interrupt``)."""
  272. mode = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_INPUT_MODE", "display", "busy_input_mode").lower()
  273. return mode if mode in {"queue", "steer"} else "interrupt"
  274. @classmethod
  275. def _load_busy_text_mode(cls) -> str:
  276. """Normal busy TEXT follow-up behavior.
  277. ``busy_input_mode`` is the source of truth (default ``interrupt``); legacy ``busy_text_mode``
  278. is honored only when explicitly set so existing queue setups keep working.
  279. """
  280. from gateway.run import GatewayRunner
  281. legacy = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_TEXT_MODE", "display", "busy_text_mode").lower()
  282. if legacy in {"interrupt", "queue"}:
  283. return legacy
  284. return "queue" if GatewayRunner._load_busy_input_mode() == "queue" else "interrupt"
  285. @staticmethod
  286. def _busy_modes_from_config(config: dict, *, fallback_input: str, fallback_text: str) -> tuple[str, str]:
  287. """Resolve one profile's busy modes without consulting process env."""
  288. raw_input = str(cfg_get(config, "display", "busy_input_mode", default="") or "").strip().lower()
  289. input_mode = raw_input if raw_input in _BUSY_INPUT_MODES else fallback_input
  290. raw_text = str(cfg_get(config, "display", "busy_text_mode", default="") or "").strip().lower()
  291. if raw_text in {"interrupt", "queue"}:
  292. text_mode = raw_text
  293. elif raw_input in _BUSY_INPUT_MODES:
  294. text_mode = "queue" if input_mode == "queue" else "interrupt"
  295. else:
  296. text_mode = fallback_text
  297. return input_mode, text_mode
  298. @staticmethod
  299. def _busy_text_timing_from_config(config: dict) -> tuple[float, float]:
  300. """``display.busy_text_debounce_seconds`` / ``display.busy_text_hard_cap_seconds`` for one
  301. profile, without consulting process env (#116893). A non-numeric or negative value is
  302. rejected with a warning naming the key, never silently coerced."""
  303. from gateway.platforms.base import DEFAULT_BUSY_TEXT_DEBOUNCE_SECONDS, DEFAULT_BUSY_TEXT_HARD_CAP_SECONDS
  304. out = []
  305. for key, default in (("busy_text_debounce_seconds", DEFAULT_BUSY_TEXT_DEBOUNCE_SECONDS),
  306. ("busy_text_hard_cap_seconds", DEFAULT_BUSY_TEXT_HARD_CAP_SECONDS)):
  307. raw = cfg_get(config, "display", key, default=None)
  308. if raw is None or raw == "":
  309. out.append(default)
  310. continue
  311. try:
  312. value = float(raw)
  313. if value < 0 or value != value:
  314. raise ValueError(raw)
  315. except (TypeError, ValueError):
  316. logger.warning("display.%s=%r is not a non-negative number; using %s", key, raw, default)
  317. value = default
  318. out.append(value)
  319. return out[0], out[1]
  320. @staticmethod
  321. def _human_delay_from_config(config: dict) -> Optional[tuple[int, int]]:
  322. """``human_delay.{mode,min_ms,max_ms}`` for one profile as a ``(lo_ms, hi_ms)`` range, or
  323. ``None`` when off (#116895). ``natural`` is the fixed 800-2500 range; ``custom`` reads the
  324. bounds and rejects non-integer, negative or inverted values with a warning naming the key,
  325. falling back to the natural range instead of letting ``random.uniform`` raise at send."""
  326. mode = str(cfg_get(config, "human_delay", "mode", default="off") or "off").strip().lower()
  327. if mode == "off":
  328. return None
  329. lo, hi = DEFAULT_HUMAN_DELAY_RANGE_MS
  330. if mode != "natural":
  331. if mode != "custom":
  332. logger.warning("human_delay.mode=%r is not off/natural/custom; using natural", mode)
  333. return DEFAULT_HUMAN_DELAY_RANGE_MS
  334. bounds = []
  335. for key, default in (("min_ms", lo), ("max_ms", hi)):
  336. raw = cfg_get(config, "human_delay", key, default=None)
  337. try:
  338. value = default if raw is None or raw == "" else int(raw)
  339. if value < 0:
  340. raise ValueError(raw)
  341. except (TypeError, ValueError):
  342. logger.warning("human_delay.%s=%r is not a non-negative integer; using %s", key, raw, default)
  343. value = default
  344. bounds.append(value)
  345. lo, hi = bounds
  346. if lo > hi:
  347. logger.warning("human_delay.min_ms=%s exceeds human_delay.max_ms=%s; using natural range", lo, hi)
  348. lo, hi = DEFAULT_HUMAN_DELAY_RANGE_MS
  349. return lo, hi
  350. def _snapshot_profile_busy_modes(self, profile_name: str, config: dict) -> None:
  351. """Cache a routed profile's busy policy and pacing for this gateway lifetime."""
  352. input_mode, text_mode = self._busy_modes_from_config(
  353. config, fallback_input=getattr(self, "_busy_input_mode", "interrupt"),
  354. fallback_text=getattr(self, "_busy_text_mode", "interrupt"),
  355. )
  356. self.__dict__.setdefault("_busy_input_modes_by_profile", {})[profile_name] = input_mode
  357. self.__dict__.setdefault("_busy_text_modes_by_profile", {})[profile_name] = text_mode
  358. self.__dict__.setdefault("_busy_text_timing_by_profile", {})[profile_name] = (
  359. self._busy_text_timing_from_config(config))
  360. self.__dict__.setdefault("_human_delay_by_profile", {})[profile_name] = (
  361. self._human_delay_from_config(config))
  362. def _busy_profile_name_for_source(self, source: SessionSource) -> Optional[str]:
  363. """Return the routed profile whose busy policy applies, if any."""
  364. if not getattr(getattr(self, "config", None), "multiplex_profiles", False):
  365. return None
  366. name = str(getattr(source, "profile", "") or "").strip()
  367. if not name:
  368. try:
  369. name = str(self._profile_name_for_source(source) or "").strip()
  370. except Exception:
  371. name = ""
  372. return name or None
  373. def _effective_busy_mode(self, source: SessionSource, attr: str) -> str:
  374. """Busy mode from the routed profile snapshot (``attr``: ``_busy_input_mode`` / ``_busy_text_mode``)."""
  375. fallback = getattr(self, attr, "interrupt")
  376. profile_name = self._busy_profile_name_for_source(source)
  377. if not profile_name:
  378. return fallback
  379. modes = getattr(self, attr + "s_by_profile", None)
  380. return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback
  381. def _effective_busy_input_mode(self, source: SessionSource) -> str:
  382. """Resolve busy input mode from the routed profile startup snapshot."""
  383. return self._effective_busy_mode(source, "_busy_input_mode")
  384. def _effective_busy_text_mode(self, source: SessionSource) -> str:
  385. """Resolve legacy busy text mode from the routed profile snapshot."""
  386. return self._effective_busy_mode(source, "_busy_text_mode")
  387. @staticmethod
  388. def _warn_unparsable_timeout(cfg_key: str, raw: object, default: float) -> None:
  389. """Warn when a supplied timeout value is not a number (the parser already fell back to ``default``)."""
  390. try:
  391. float(raw)
  392. except (TypeError, ValueError):
  393. logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default)
  394. @classmethod
  395. def _load_restart_drain_timeout(cls) -> float:
  396. """Graceful gateway restart/stop drain timeout in seconds."""
  397. raw = cls._env_or_cfg_str("HERMES_RESTART_DRAIN_TIMEOUT", "agent", "restart_drain_timeout")
  398. value = parse_restart_drain_timeout(raw)
  399. if raw and value == DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT:
  400. cls._warn_unparsable_timeout("restart_drain_timeout", raw, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT)
  401. return value
  402. @staticmethod
  403. def _load_launchd_exit_timeout(drain_timeout: float) -> Optional[float]:
  404. """Read the live launchd ``ExitTimeOut`` this job runs under, if any.
  405. launchd is the one supervisor the gateway cannot size from config: the per-user (gui)
  406. domain clamps ``ExitTimeOut`` (measured 60s on macOS 26), and any signal-driven stop that
  407. drains past it is SIGKILLed mid-teardown — the unclean-exit half of the state.db
  408. corruption class. Returns ``None`` (fail-open, drain unchanged) when not launchd-owned or
  409. when ``launchctl print`` is unavailable. Logs a WARNING when the configured drain exceeds
  410. the live budget so the misconfiguration is visible at boot, not at the next SIGKILL.
  411. """
  412. label = launchd_service_label()
  413. if label is None:
  414. return None
  415. # read_launchd_exit_timeout_s is already fail-open (returns None on any probe failure).
  416. exit_timeout = read_launchd_exit_timeout_s(label)
  417. if exit_timeout is None:
  418. return None
  419. effective = resolve_launchd_capped_drain(drain_timeout, exit_timeout)
  420. if effective < drain_timeout:
  421. logger.warning(
  422. "restart_drain_timeout=%.0fs exceeds the live launchd exit timeout (%.0fs) for %s; "
  423. "signal-driven stops will drain at most %.0fs so teardown finishes before launchd "
  424. "SIGKILLs (launchd clamps ExitTimeOut in the per-user domain).",
  425. drain_timeout, exit_timeout, label, effective,
  426. )
  427. else:
  428. logger.info(
  429. "launchd exit timeout for %s is %.0fs (drain %.0fs fits)",
  430. label, exit_timeout, drain_timeout,
  431. )
  432. return exit_timeout
  433. @classmethod
  434. def _load_env_or_agent_cfg_timeout(cls, env_var: str, cfg_key: str, parse, default: float) -> float:
  435. """Env var (non-empty) else ``agent.<cfg_key>``; warn once when a supplied value fails to parse.
  436. ``0`` is a valid value; the parser falls back to ``default`` on garbage."""
  437. from gateway.run import _load_gateway_config
  438. env_raw = os.getenv(env_var)
  439. if env_raw is not None and str(env_raw).strip() != "":
  440. raw: object = env_raw
  441. else:
  442. raw = cfg_get(_load_gateway_config(), "agent", cfg_key, default=None)
  443. value = parse(raw)
  444. if raw is not None and str(raw).strip() != "":
  445. cls._warn_unparsable_timeout(cfg_key, raw, default)
  446. return value
  447. @classmethod
  448. def _load_restart_after_turn_timeout(cls) -> float:
  449. """In-band restart wait-for-idle timeout in seconds."""
  450. return cls._load_env_or_agent_cfg_timeout(
  451. "HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout",
  452. parse_restart_after_turn_timeout, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT,
  453. )
  454. @classmethod
  455. def _load_cron_drain_timeout(cls) -> float:
  456. """The cron and api_server (/v1) floor under the stop()/drain wait.
  457. See #82161.
  458. """
  459. return cls._load_env_or_agent_cfg_timeout(
  460. "HERMES_CRON_DRAIN_TIMEOUT", "cron_drain_timeout",
  461. parse_cron_drain_timeout, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT,
  462. )
  463. @classmethod
  464. def _load_signal_interrupt_grace_timeout(cls) -> float:
  465. """``gateway.signal_interrupt_grace_timeout``: unexpected-signal post-interrupt grace in seconds."""
  466. from gateway.run import _load_gateway_config
  467. raw = cfg_get(_load_gateway_config(), "gateway", "signal_interrupt_grace_timeout", default=None)
  468. value = parse_signal_interrupt_grace_timeout(raw)
  469. if raw is not None and raw != "":
  470. cls._warn_unparsable_timeout(
  471. "signal_interrupt_grace_timeout", raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT
  472. )
  473. return value
  474. def _post_interrupt_grace_timeout(self) -> float:
  475. """Grace before teardown after forcibly interrupting agents (longer on an unexpected signal)."""
  476. if getattr(self, "_signal_initiated_shutdown", False) and not getattr(self, "_restart_requested", False):
  477. grace = getattr(self, "_signal_interrupt_grace_timeout", DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT)
  478. return max(0.0, float(grace))
  479. return DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT
  480. @staticmethod
  481. def _load_background_notifications_mode() -> str:
  482. """Background process notification mode from env/config (default ``concise``), resolved for
  483. the AMBIENT profile — callers deciding for another profile's event enter its scope first
  484. (``_completion_event_scope``). The env override reads through the secret scope so a served
  485. secondary sees its own ``.env`` value, not the launch profile's ``os.environ``."""
  486. from gateway.run import _load_gateway_config
  487. from gateway.platforms._shared import platform_gate_env as _platform_gate_env
  488. mode = _platform_gate_env("HERMES_BACKGROUND_NOTIFICATIONS")
  489. if not mode:
  490. raw = cfg_get(_load_gateway_config(), "display", "background_process_notifications")
  491. if raw is False:
  492. mode = "off"
  493. elif raw not in {None, ""}:
  494. mode = str(raw)
  495. mode = (mode or "concise").strip().lower()
  496. if mode not in {"concise", "all", "result", "error", "off"}:
  497. logger.warning("Unknown background_process_notifications '%s', defaulting to 'concise'", mode)
  498. return "concise"
  499. return mode
  500. @staticmethod
  501. def _load_provider_routing() -> dict:
  502. """OpenRouter provider routing preferences (canonical fail-open loader: managed overlay + ${VAR})."""
  503. from gateway.run import _load_gateway_config
  504. try:
  505. return _load_gateway_config().get("provider_routing", {}) or {}
  506. except Exception:
  507. return {}
  508. @staticmethod
  509. def _load_fallback_model() -> list | None:
  510. """Fallback chain: ``fallback_providers`` (kept first) merged with legacy ``fallback_model``."""
  511. from gateway.run import _load_gateway_config
  512. try:
  513. # Canonical gateway loader (fail-open): managed overlay + ${VAR} expansion apply here too.
  514. return get_fallback_chain(_load_gateway_config()) or None
  515. except Exception:
  516. return None
  517. def _refresh_fallback_model(self) -> list | None:
  518. """Re-read fallback_providers from disk for the next agent create/reuse.
  519. Lets a chain edited after startup reach messaging sessions (cron already re-reads per job).
  520. A TRANSIENT read/parse failure (user mid-edit, non-atomic write) keeps the last known-good
  521. chain; only a successful read that genuinely lacks the key clears it.
  522. Cron already does this per job via ``get_fallback_chain``; the gateway previously froze
  523. ``self._fallback_model`` at process start, so a chain configured (or changed) after ``hermes
  524. gateway`` was running never reached messaging sessions even though the same process's cron jobs fell
  525. back correctly. Fixes #60955.
  526. Reads the ACTIVE gateway home (the routed profile under multiplexing, else the launch home)
  527. and keeps one last-known-good chain per home: a single runner-wide slot filled from the launch
  528. home handed every secondary profile the default profile's fallback chain.
  529. """
  530. from hermes_cli.config_backend import config_exists
  531. from gateway.run import _gateway_config_home
  532. from hermes_constants import hermes_home_key
  533. home = _gateway_config_home()
  534. by_home = getattr(self, "_fallback_model_by_home", None)
  535. if by_home is None:
  536. by_home = self._fallback_model_by_home = {}
  537. home_key = hermes_home_key(home)
  538. try:
  539. from hermes_cli.config_effective import load_user_config_effective
  540. cfg_path = home / "config.yaml"
  541. if not config_exists(cfg_path):
  542. by_home[home_key] = self._fallback_model = None
  543. return self._fallback_model
  544. # fail_closed: a torn mid-edit write must raise so the per-home last known-good chain
  545. # below survives, instead of being WIPED by an empty fail-open result.
  546. cfg = load_user_config_effective(cfg_path, fail_closed=True)
  547. except Exception:
  548. logger.debug("fallback_providers refresh: config.yaml read failed; keeping last known-good chain", exc_info=True)
  549. self._fallback_model = by_home.get(home_key, self._fallback_model)
  550. return self._fallback_model
  551. by_home[home_key] = self._fallback_model = get_fallback_chain(cfg) or None
  552. return self._fallback_model
  553. @staticmethod
  554. def _apply_fallback_chain_to_agent(agent: Any, chain: list | None) -> None:
  555. """Keep a cached agent's fallback chain aligned with current config.
  556. Skips the rewrite while a cooldown holds the agent on an activated fallback provider
  557. (``restore_primary_runtime`` owns that lifecycle); otherwise replaces the chain so
  558. mid-uptime ``fallback_providers`` edits apply without a restart.
  559. When primary is active (or cooldown expired), replace the chain so mid-uptime ``fallback_providers``
  560. edits take effect without requiring a gateway restart (#60955).
  561. """
  562. if agent is None:
  563. return
  564. new_chain = list(chain or [])
  565. rate_limited_until = getattr(agent, "_rate_limited_until", 0) or 0
  566. if getattr(agent, "_fallback_activated", False) and rate_limited_until > time.monotonic():
  567. return
  568. old_chain = list(getattr(agent, "_fallback_chain", []) or [])
  569. agent._fallback_chain = new_chain
  570. agent._fallback_model = new_chain[0] if new_chain else None
  571. if not getattr(agent, "_fallback_activated", False):
  572. agent._fallback_index = 0
  573. # A config edit means the user changed something — drop the session-scoped unavailability
  574. # memo so re-configured entries (e.g. credentials added mid-uptime) get retried. Only on real
  575. # content change, so the per-message no-op refresh keeps the memo's rate-limiting benefit.
  576. # See #60955.
  577. if new_chain != old_chain:
  578. unavailable = getattr(agent, "_unavailable_fallback_keys", None)
  579. if unavailable:
  580. unavailable.clear()