config_env.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. """Environment-variable overrides for the gateway config (``_apply_env_overrides``).
  2. Runs after ``GatewayConfig.from_dict`` so env always wins over config.yaml / gateway.json.
  3. Most platforms follow one shape — "credentials present in env ⇒ enable the platform and
  4. copy the values into ``extra``" — declared by each platform plugin as ``PlatformEntry.env_overrides``
  5. (``gateway.platform_env.EnvCred`` rows; see that module). Core-only platforms (api_server, webhook,
  6. msgraph_webhook, relay) keep their small functions here.
  7. """
  8. import copy
  9. import logging
  10. import os
  11. from contextvars import ContextVar
  12. from typing import Optional
  13. from gateway.config import (
  14. _BUILTIN_PLATFORM_VALUES,
  15. _discovered_registry_entry,
  16. GatewayConfig,
  17. HomeChannel,
  18. Platform,
  19. PlatformConfig,
  20. _getenv_str,
  21. _has_usable_api_server_key,
  22. SHARED_LISTENER_MIRROR_PLATFORMS,
  23. )
  24. from gateway.platform_env import INT, csv_extras, env_extras
  25. from utils import is_truthy_value
  26. # Logger name parity with the origin module: records stay under "gateway.config".
  27. logger = logging.getLogger("gateway.config")
  28. getenv = _getenv_str
  29. # Platforms already warned "explicitly disabled in config.yaml, but credentials present in
  30. # env": config reloads every turn, so the notice is one-time per platform per process.
  31. _EXPLICIT_DISABLE_WARNED: set = set()
  32. # Env var(s) whose presence drives a CORE platform's env-enable branch, named in the explicit-disable
  33. # WARNING. Plugin platforms declare theirs as ``PlatformEntry.env_enable_credentials``.
  34. _ENV_ENABLE_CREDENTIALS: dict = {
  35. Platform.RELAY: ("GATEWAY_RELAY_URL",),
  36. }
  37. def _enable_credential_names(platform: Platform) -> tuple:
  38. """WARNING credential names: the core row, else the platform entry's declaration. The entry is
  39. already materialized (its own env step is what is warning), so the lookup never imports."""
  40. if platform in _ENV_ENABLE_CREDENTIALS:
  41. return _ENV_ENABLE_CREDENTIALS[platform]
  42. from gateway.platform_registry import platform_registry
  43. entry = platform_registry.get(platform.value)
  44. names = tuple(getattr(entry, "env_enable_credentials", ()) or ())
  45. if not names:
  46. from gateway.platform_env import bundled_env_rows
  47. names = (bundled_env_rows(platform.value) or ((), ()))[1]
  48. return names
  49. # Set by builtin_platform_status: a status read for another profile must neither repeat the gateway's
  50. # operational logs on every dashboard poll nor use up its one-time warnings.
  51. _STATUS_READ: ContextVar[bool] = ContextVar("gateway_env_status_read", default=False)
  52. def _warn_explicit_disable_beats_env(platform: Platform) -> None:
  53. """One-time WARNING: ``platforms.<x>.enabled: false`` wins over env creds (which
  54. used to force-enable), so users relying on "creds in .env = on" learn why it went dark."""
  55. if platform in _EXPLICIT_DISABLE_WARNED or _STATUS_READ.get():
  56. return
  57. _EXPLICIT_DISABLE_WARNED.add(platform)
  58. names = _enable_credential_names(platform)
  59. present = [n for n in names if (os.environ.get(n) or "").strip()]
  60. creds = ", ".join(present or names) or "its credentials"
  61. logger.warning(
  62. "Platform '%s' is explicitly disabled by platforms.%s.enabled: false in "
  63. "config.yaml, so the credentials found in the environment (%s) will NOT "
  64. "start its adapter. Environment credentials no longer override an "
  65. "explicit disable. Remove the key or set platforms.%s.enabled: true to "
  66. "turn it back on.",
  67. platform.value, platform.value, creds, platform.value,
  68. )
  69. def _loading_secondary_under_multiplexer() -> bool:
  70. """True while a multiplexer loads a NON-default profile's config (``_profile_runtime_scope`` sets the
  71. home override; the runner sets the multiplex flag). Same signal ``gateway.config`` uses for scoped reads."""
  72. from agent.secret_scope import is_multiplex_active
  73. from hermes_constants import get_hermes_home_override, profile_name_for_home
  74. override = get_hermes_home_override()
  75. return bool(override) and is_multiplex_active() and profile_name_for_home(override) != "default"
  76. def _enable_from_env(
  77. config: GatewayConfig, platform: Platform, *, pop_marker: bool = False, warn: bool = True
  78. ) -> PlatformConfig:
  79. """Enable *platform* on env credentials unless config.yaml explicitly disabled it.
  80. A multiplex secondary profile may pin ``enabled: false`` yet inherit the process env; without
  81. this guard env presence would force-enable it. By default the ``_enabled_explicit`` marker is
  82. READ (the plugin-enable and relay passes still need it) and the disable is warned once;
  83. api_server/webhook POP it (terminal branch) and stay silent.
  84. """
  85. platform_config = config.platforms.setdefault(platform, PlatformConfig())
  86. extra = platform_config.extra
  87. explicit = extra.pop("_enabled_explicit", False) if pop_marker else extra.get("_enabled_explicit", False)
  88. if platform_config.enabled:
  89. return platform_config
  90. if not explicit and not (
  91. platform.value in SHARED_LISTENER_MIRROR_PLATFORMS and _loading_secondary_under_multiplexer()
  92. ):
  93. # A secondary's API_SERVER_KEY / WEBHOOK_ENABLED (the docs require the key in its .env for
  94. # /p/<profile>/ auth) must not turn into listener intent: the default profile's listener already
  95. # mirrors those two at /p/<profile>/ (#100397). The credential still lands in ``extra`` for it.
  96. # Every other inbound-port platform IS enabled for a secondary: it runs in shared-listener mode.
  97. platform_config.enabled = True
  98. elif warn:
  99. _warn_explicit_disable_beats_env(platform)
  100. return platform_config
  101. def _api_server(config: GatewayConfig) -> None:
  102. """Require a usable key: an unauthenticated adapter refuses to start and the reconnect watcher would spin."""
  103. key = getenv("API_SERVER_KEY")
  104. if not _has_usable_api_server_key(key):
  105. return
  106. extra = _enable_from_env(config, Platform.API_SERVER, pop_marker=True, warn=False).extra
  107. extra["key"] = key
  108. csv_extras(extra, (("cors_origins", getenv("API_SERVER_CORS_ORIGINS")),))
  109. env_extras(extra, (("port", "API_SERVER_PORT", INT), ("host", "API_SERVER_HOST"), ("model_name", "API_SERVER_MODEL_NAME")))
  110. def _webhook(config: GatewayConfig) -> None:
  111. enabled = is_truthy_value(getenv("WEBHOOK_ENABLED"))
  112. if not (enabled or Platform.WEBHOOK in config.platforms):
  113. return
  114. webhook_config = (
  115. _enable_from_env(config, Platform.WEBHOOK, pop_marker=True, warn=False)
  116. if enabled
  117. else config.platforms[Platform.WEBHOOK]
  118. )
  119. env_extras(webhook_config.extra, (("port", "WEBHOOK_PORT", INT), ("secret", "WEBHOOK_SECRET")))
  120. def _msgraph_webhook(config: GatewayConfig) -> None:
  121. enabled = is_truthy_value(getenv("MSGRAPH_WEBHOOK_ENABLED"))
  122. client_state = getenv("MSGRAPH_WEBHOOK_CLIENT_STATE")
  123. resources = getenv("MSGRAPH_WEBHOOK_ACCEPTED_RESOURCES")
  124. allowed_cidrs = getenv("MSGRAPH_WEBHOOK_ALLOWED_SOURCE_CIDRS")
  125. if not (enabled or Platform.MSGRAPH_WEBHOOK in config.platforms or getenv("MSGRAPH_WEBHOOK_PORT")
  126. or client_state or resources or allowed_cidrs):
  127. return
  128. msgraph_cfg = config.platforms.setdefault(Platform.MSGRAPH_WEBHOOK, PlatformConfig())
  129. if enabled: # same guard as webhook, but READ (don't pop) the marker: the relay pass still consults it
  130. _enable_from_env(config, Platform.MSGRAPH_WEBHOOK, warn=False)
  131. env_extras(msgraph_cfg.extra, (("port", "MSGRAPH_WEBHOOK_PORT", INT),))
  132. if client_state:
  133. msgraph_cfg.extra["client_state"] = client_state
  134. csv_extras(msgraph_cfg.extra, (("accepted_resources", resources), ("allowed_source_cidrs", allowed_cidrs)))
  135. def _plugin_probe_seed(entry) -> Optional[dict]:
  136. """``env_enablement_fn()`` result as a non-empty dict, else None."""
  137. if entry.env_enablement_fn is None:
  138. return None
  139. try:
  140. seed = entry.env_enablement_fn()
  141. except Exception as e:
  142. logger.debug("env_enablement_fn for %s raised: %s", entry.name, e)
  143. return None
  144. return seed if isinstance(seed, dict) and seed else None
  145. def _plugin_is_configured(entry, existing_extra: dict, seed: Optional[dict]) -> bool:
  146. """``entry.is_connected`` on a transient ``enabled=True`` view seeded with env extras (never the real config)."""
  147. try:
  148. for k, v in (seed or {}).items():
  149. if k != "home_channel":
  150. existing_extra.setdefault(k, v)
  151. configured = bool(entry.is_connected(PlatformConfig(enabled=True, extra=existing_extra)))
  152. except Exception as exc:
  153. logger.debug("is_connected for %s raised: %s — skipping enablement", entry.name, exc)
  154. return False
  155. if not configured:
  156. logger.debug(
  157. "Plugin platform '%s' available but not configured "
  158. "(is_connected returned False) — skipping enable",
  159. entry.name,
  160. )
  161. return configured
  162. def _enable_plugin_platform(config: GatewayConfig, entry) -> None:
  163. try:
  164. platform = Platform(entry.name)
  165. except Exception as e:
  166. logger.debug("unknown platform name %r: %s", entry.name, e)
  167. return
  168. existing_cfg = config.platforms.get(platform)
  169. already_enabled = existing_cfg is not None and existing_cfg.enabled
  170. existing_extra = dict(existing_cfg.extra or {}) if existing_cfg is not None else {}
  171. # Never re-enable a platform the user explicitly disabled (marker set by the YAML loader).
  172. if existing_cfg is not None and not already_enabled and existing_extra.get("_enabled_explicit", False):
  173. return
  174. seed = _plugin_probe_seed(entry)
  175. # is_connected gates only platforms not already enabled by YAML/env.
  176. if not already_enabled and entry.is_connected is not None and not _plugin_is_configured(entry, existing_extra, seed):
  177. return
  178. # Dependencies LAST — only for platforms already enabled or past the credential gate.
  179. try:
  180. # ``check_fn`` is a PASSIVE probe (never installs); a platform whose deps are missing but which
  181. # registered ``ensure_deps_fn`` still gets enabled here — the registry's ``create_adapter()`` runs
  182. # the active installer at gateway start, when the user actually wants the platform up. Historically
  183. # the ACTIVE installer was wired as ``check_fn`` and this sweep pip-installed
  184. # Discord/Telegram/Slack/Feishu/Dingtalk SDKs on every ``load_gateway_config()`` call — including
  185. # the desktop/dashboard readiness probe (``GET /api/status``) — blocking startup until every install
  186. # finished and boot-looping the desktop app at 94%. The check_fn/ensure_deps_fn split (#79812) makes
  187. # that impossible by construction.
  188. deps_ok = bool(entry.check_fn())
  189. except Exception as e:
  190. logger.debug("check_fn for %s raised: %s", entry.name, e)
  191. deps_ok = False
  192. if not deps_ok and entry.ensure_deps_fn is None:
  193. return
  194. platform_config = config.platforms.setdefault(platform, PlatformConfig())
  195. platform_config.enabled = True
  196. if seed: # commit the probe's env-seeded extras (env_enablement_fn is never called twice)
  197. seed = dict(seed)
  198. home = seed.pop("home_channel", None)
  199. platform_config.extra.update(seed)
  200. if isinstance(home, dict) and home.get("chat_id"):
  201. platform_config.home_channel = HomeChannel(
  202. platform=platform, chat_id=str(home["chat_id"]), name=str(home.get("name") or "Home"),
  203. thread_id=str(home["thread_id"]) if home.get("thread_id") else None,
  204. )
  205. def _configured_plugin_entries(config: GatewayConfig) -> list:
  206. """Plugin entries with activation evidence (config block, env/.env, credential pool); only those
  207. are imported — an unconfigured adapter has nothing to enable. See gateway.platform_activation.
  208. config.yaml blocks already materialized their adapters in load_yaml_layer (and every merged
  209. block is in config.platforms), so the YAML is not parsed a second time here."""
  210. try:
  211. from hermes_cli.plugins import discover_plugins
  212. discover_plugins() # idempotent
  213. from gateway.platform_registry import platform_registry
  214. from gateway.platform_activation import configured_plugin_entries
  215. return configured_plugin_entries(platform_registry, yaml_cfg={}, platform_names=config.platforms)
  216. except Exception as e: # health: allow BLE001 -- moved from the plugin pass: discovery never breaks config load
  217. logger.debug("Plugin platform discovery for env overrides failed: %s", e)
  218. return []
  219. def _bundled_env_fallbacks(config: GatewayConfig, entries: list, *, only: Optional[Platform] = None) -> list:
  220. """Env rows of bundled platforms with activation evidence but no registry entry (discovery
  221. suppressed or the plugin disabled), read from their ``gateway_env`` files; see
  222. ``gateway.platform_env.bundled_env_rows``. They only run the env steps, never the enable pass."""
  223. from types import SimpleNamespace
  224. from gateway.platform_activation import activation_predicate
  225. from gateway.platform_env import _BUNDLED_ENV_SIBLINGS, bundled_env_rows
  226. have = {getattr(e, "name", "") for e in entries}
  227. if only is not None: # bundled_env_rows() answers for one name without scanning every manifest
  228. names = {only.value} - have
  229. else:
  230. names = (set(Platform._scan_bundled_plugin_platforms()[0]) | set(_BUNDLED_ENV_SIBLINGS)) - have
  231. if not names:
  232. return []
  233. wanted = activation_predicate(yaml_cfg={}, platform_names=config.platforms)
  234. out = []
  235. for name in sorted(names):
  236. rows = bundled_env_rows(name) if wanted(name) else None
  237. if rows and rows[0]:
  238. out.append(SimpleNamespace(name=name, env_overrides=rows[0], env_enable_credentials=rows[1]))
  239. return out
  240. def _platform_rank(entry) -> int:
  241. # Static ``Platform`` members first, in enum order (the order the env rows always ran in), so
  242. # ``config.platforms`` insertion order stays deterministic and independent of registry load order.
  243. members = list(Platform.__members__.values())
  244. try:
  245. return members.index(Platform(entry.name))
  246. except (ValueError, KeyError):
  247. return len(members)
  248. def _apply_plugin_env_overrides(config: GatewayConfig, entries: list) -> None:
  249. """Run each plugin platform's declared ``env_overrides`` steps (credential enable, home channel,
  250. reply mode, ...). Every step touches only its own platform, so only the order WITHIN an entry is
  251. significant (e.g. Telegram's reply mode may create the entry its home channel then attaches to)."""
  252. for entry in sorted(entries, key=_platform_rank):
  253. for step in getattr(entry, "env_overrides", ()) or ():
  254. try:
  255. step(config)
  256. except Exception as e: # health: allow BLE001 -- a broken plugin env row must not break config load
  257. logger.debug("env override for %s raised: %s", entry.name, e)
  258. def _enable_plugin_platforms_from_env(config: GatewayConfig, entries: list) -> None:
  259. """Registry-driven enable for plugin platforms whose ``env_overrides`` did not already enable them.
  260. Enabled when credentials are configured (``is_connected`` MUST gate: ``check_fn`` alone would
  261. enable unconfigured platforms that retry-connect forever) and deps are present (``check_fn``) or
  262. installable later by ``create_adapter()`` — never here: installing in this sweep boot-looped the app.
  263. """
  264. for entry in entries:
  265. try:
  266. _enable_plugin_platform(config, entry)
  267. except Exception as e:
  268. logger.debug("Plugin platform enable pass failed for %s: %s", entry.name, e)
  269. def _relay(config: GatewayConfig) -> None:
  270. """Relay (connector-fronted, EXPERIMENTAL): enabled by GATEWAY_RELAY_URL or gateway.relay_url;
  271. the URL is mirrored into extra["relay_url"] for the connected-checker.
  272. Relay-exclusive: the GATEWAY_RELAY_URL env stamp means the connector owns every platform
  273. connection; a directly-connected adapter would be a second unmanaged ingress (duplicate
  274. deliveries, split sessions, a socket disarming scale-to-zero). So the env stamp disables all
  275. other messaging platforms — even explicitly enabled ones — except non-messaging surfaces (local,
  276. api_server, webhook; same set as the scale-to-zero arm gate). relay_url from YAML only keeps the
  277. additive behavior. Opt out with GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true.
  278. """
  279. relay_url_env = getenv("GATEWAY_RELAY_URL").strip()
  280. existing_relay = config.platforms.get(Platform.RELAY)
  281. relay_url_yaml = str(existing_relay.extra.get("relay_url") or "").strip() if existing_relay else ""
  282. relay_url_val = relay_url_env or relay_url_yaml
  283. if relay_url_val:
  284. relay_config = _enable_from_env(config, Platform.RELAY)
  285. relay_config.extra["relay_url"] = relay_url_val.rstrip("/")
  286. # An opted-out relay does not own this profile's native connections.
  287. if not relay_config.enabled:
  288. return
  289. if not relay_url_env or is_truthy_value(getenv("GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS")):
  290. return
  291. skip = {Platform.RELAY, Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK}
  292. for platform, platform_config in config.platforms.items():
  293. if platform in skip or not platform_config.enabled:
  294. continue
  295. if _STATUS_READ.get():
  296. pass # status reads repeat this on every poll
  297. elif platform_config.extra.get("_enabled_explicit"):
  298. logger.warning(
  299. "Relay connector is configured via GATEWAY_RELAY_URL; "
  300. "disabling directly-connected platform '%s' even though "
  301. "it is explicitly enabled in this profile's configuration. "
  302. "All messaging goes through the connector on this "
  303. "deployment. Set GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true "
  304. "to keep direct platforms alongside the relay.",
  305. platform.value,
  306. )
  307. else:
  308. logger.info(
  309. "Relay connector is configured via GATEWAY_RELAY_URL; "
  310. "disabling directly-connected platform '%s'.",
  311. platform.value,
  312. )
  313. platform_config.enabled = False
  314. platform_config.disabled_by_relay = True
  315. def _scrub_explicit_markers(config: GatewayConfig) -> None:
  316. for platform_config in config.platforms.values():
  317. platform_config.extra.pop("_enabled_explicit", None)
  318. def _apply_env_overrides(config: GatewayConfig, *, only: Optional[Platform] = None) -> None:
  319. """Apply environment variable overrides to *config*.
  320. Order: plugin platforms' declared env rows, then the core-only rows (api_server, webhook,
  321. msgraph_webhook; each touches only its own platform), then the registry enable pass, relay
  322. disabling (needs every platform's final ``enabled``), and the marker scrub, which must be last.
  323. ``only`` limits the plugin passes to that built-in platform's own entry and bundled env rows
  324. (no per-call manifest scan; see ``builtin_platform_status``).
  325. """
  326. if only is None:
  327. entries = _configured_plugin_entries(config)
  328. else:
  329. entry = _discovered_registry_entry(only)
  330. entries = [entry] if entry is not None else []
  331. _apply_plugin_env_overrides(config, entries + _bundled_env_fallbacks(config, entries, only=only))
  332. _api_server(config)
  333. _webhook(config)
  334. _msgraph_webhook(config)
  335. _enable_plugin_platforms_from_env(config, entries)
  336. _relay(config)
  337. _scrub_explicit_markers(config)
  338. def builtin_platform_status(platform_id: str, yaml_cfg: dict) -> Optional[tuple[bool, bool]]:
  339. """``(enabled, connected)`` as the gateway decides them for one built-in platform of the profile
  340. whose config.yaml is *yaml_cfg*, with env read through the bound secret scope; None for a
  341. plugin-only id.
  342. For judging one platform of a profile that is not the process's own (the dashboard's Channels
  343. page): ``load_gateway_config()`` there enumerates that profile's platform plugins and constructs
  344. ``Platform(<plugin id>)`` for them, process-wide pseudo-members every other profile then sees.
  345. So only this platform's and the relay's blocks (the relay can disable it) go through the
  346. loader's own steps (section merge, administrator-pinned keys, top-level bridge; see
  347. ``load_yaml_layer``), which also record an explicit ``enabled`` for the env pass. The env pass's
  348. logs and one-time warnings are held back.
  349. """
  350. if platform_id not in _BUILTIN_PLATFORM_VALUES:
  351. return None
  352. from gateway.config_loader import (
  353. _apply_managed_extra, bridge_platform_shared_keys, merge_platform_sections, snapshot_authored_extra,
  354. )
  355. from hermes_cli.managed_scope import apply_managed_overlay
  356. platform = Platform(platform_id)
  357. names = {platform_id, Platform.RELAY.value}
  358. top, nested = _platform_layers(yaml_cfg, names)
  359. platforms = merge_platform_sections(top, nested, gw_data={})
  360. authored = snapshot_authored_extra(platforms) # no gateway.json layer here: the merge is config.yaml only
  361. managed_top, managed_nested = _platform_layers(apply_managed_overlay({}), names)
  362. _apply_managed_extra(authored, {**managed_top, "gateway": managed_nested}, platforms)
  363. bridge_platform_shared_keys(top, nested["platforms"], {}, platforms, [platform, Platform.RELAY], authored=authored)
  364. for block in platforms.values():
  365. block.pop("home_channel", None) # status needs none; HomeChannel.from_dict can raise or name another platform
  366. config = GatewayConfig.from_dict({"platforms": platforms})
  367. token = _STATUS_READ.set(True)
  368. try:
  369. _apply_env_overrides(config, only=platform)
  370. platform_config = config.platforms.get(platform)
  371. if platform_config is None:
  372. return False, False
  373. return bool(platform_config.enabled), config._is_platform_connected(platform, platform_config)
  374. finally:
  375. _STATUS_READ.reset(token)
  376. def _platform_layers(yaml_cfg: dict, names: set) -> tuple[dict, dict]:
  377. """*yaml_cfg*'s platform blocks for *names* only, as the loader's two inputs: the root layer
  378. (``platforms.<id>`` and top-level ``<id>:``) and the ``gateway:`` layer."""
  379. gateway_section = yaml_cfg.get("gateway") if isinstance(yaml_cfg.get("gateway"), dict) else {}
  380. top = {"platforms": _only_keys(yaml_cfg.get("platforms"), names), **_only_keys(yaml_cfg, names)}
  381. nested = {"platforms": _only_keys(gateway_section.get("platforms"), names), **_only_keys(gateway_section, names)}
  382. return top, nested
  383. def _only_keys(section, names: set) -> dict:
  384. """Copies of *section*'s entries named in *names* (no others reach the loader's ``Platform(key)``
  385. probes, and the merge and bridge, which write into blocks they reuse, never touch the caller's)."""
  386. if not isinstance(section, dict):
  387. return {}
  388. return {key: copy.deepcopy(value) for key, value in section.items() if key in names}