gateway_multiplex_mode.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. """Boot-time verdict for ``gateway.multiplex_profiles`` (the default is on, and there is no opt-out).
  2. ``GatewayConfig.from_dict`` leaves the flag ``None`` when neither config.yaml nor
  3. ``GATEWAY_MULTIPLEX_PROFILES`` set it. Turning the default on must not make a default gateway
  4. double-bind a fleet that still runs per-profile gateways (two pollers on one bot token, port
  5. fights), so the implicit default is a *request*: the gateway runs the same preflight
  6. ``hermes gateway migrate --multiplex`` runs and multiplexes only when the fold would have been
  7. safe. An explicit ``true`` is never second-guessed.
  8. An explicit ``false`` is RETIRED (multiplex-only ruling): it parses, it is logged, and it is then
  9. resolved exactly like an unset key. The key itself survives because it is still the RUNTIME
  10. mode flag every scoped code path reads (``config.multiplex_profiles``) — what it can no longer do
  11. is pin a second gateway process onto this host.
  12. The refusal is logged, never fatal: the gateway comes up standalone exactly as before the
  13. default flipped, and the log names the blocker plus ``hermes gateway migrate --multiplex``.
  14. """
  15. from __future__ import annotations
  16. import logging
  17. from dataclasses import dataclass
  18. from pathlib import Path
  19. from typing import Optional
  20. logger = logging.getLogger(__name__)
  21. SINGLE_PROFILE_REASON = "only one profile exists (nothing to multiplex)"
  22. STANDALONE_PROFILE_REASON = "this profile is standalone (gateway.standalone: true); it serves only itself"
  23. #: A single-profile boot that ``gateway.auto_multiplex_migration: false`` keeps standalone once a second
  24. #: profile appears: the in-place switch is an automatic migration, so the opt-out holds it back.
  25. AUTO_MIGRATION_OPTED_OUT_REASON = (
  26. "gateway.auto_multiplex_migration is false, so this gateway serves only its own profile; "
  27. "run `hermes gateway migrate --multiplex` to serve every profile")
  28. #: ``gateway.standalone: true`` is a TEMPORARY backwards-compatibility shim, not a supported topology.
  29. #: It exists so fleets that lost per-profile gateways in the multiplex-only switch keep working while
  30. #: the remaining multiplexing gaps (per-profile stop/restart, WhatsApp bridge/relay on secondaries,
  31. #: dashboard scoping) are closed; it is removed once they are. Every surface that names the key
  32. #: prints this so nobody builds on it.
  33. STANDALONE_DEPRECATION_NOTICE = (
  34. "gateway.standalone is a temporary compatibility shim while multiplexing gaps are fixed; "
  35. "it will be removed once they are — plan to fold this profile with `hermes gateway migrate --multiplex`."
  36. )
  37. #: ``gateway.multiplex_profiles: false`` is no longer an opt-out from the one-gateway-per-host
  38. #: topology; it parses, it is reported, and it is ignored.
  39. RETIRED_OPT_OUT_REASON = (
  40. "gateway.multiplex_profiles: false is retired and was rewritten to true; one gateway per host "
  41. "serves every profile. A per-profile gateway is `gateway.standalone: true` in that profile's "
  42. "config (temporary shim) or `--force`.")
  43. #: One-time marker the gateway leaves after rewriting a retired ``false``; ``hermes update``'s summary
  44. #: prints the notice from it and clears it, so the flip is never silent on either surface.
  45. REWRITTEN_MARKER_NAME = ".multiplex_opt_out_rewritten"
  46. def explicit_multiplex_flag(default_home: Path) -> Optional[bool]:
  47. """The operator's explicit choice for the DEFAULT profile's gateway: a recognized
  48. ``GATEWAY_MULTIPLEX_PROFILES``, else ``gateway.multiplex_profiles`` (or the top-level alias) as
  49. written in its config.yaml; ``None`` when neither is set. Raw read on purpose: the callers are
  50. other processes (``hermes -p X ...`` has X's config loaded) asking about the default's file."""
  51. from hermes_cli.config_backend import config_exists
  52. from gateway.config import _bool_token, _env_multiplex_profiles_override
  53. env = _env_multiplex_profiles_override()
  54. if env is not None:
  55. return env
  56. cfg_path = Path(default_home) / "config.yaml"
  57. if not config_exists(cfg_path):
  58. return None
  59. from hermes_cli.config import read_user_config_raw
  60. cfg = read_user_config_raw(cfg_path) or {}
  61. gateway_section = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {}
  62. value = cfg.get("multiplex_profiles")
  63. if value is None:
  64. value = gateway_section.get("multiplex_profiles")
  65. if value is None:
  66. return None
  67. if isinstance(value, str):
  68. parsed = _bool_token(value)
  69. return True if parsed is None else parsed
  70. return bool(value)
  71. def settled_explicit_flag(value: Optional[bool]) -> Optional[bool]:
  72. """How boot reads an explicit ``multiplex_profiles``: ``true`` is final; the retired ``false``
  73. settles exactly like an unset key (``None``: the boot preflight decides)."""
  74. return True if value else None
  75. def host_serves_named_profile(default_home: Path, home: Path) -> Optional[bool]:
  76. """Offline twin of the boot verdict for named profile *home* under *default_home*'s gateway:
  77. ``False`` for a ``gateway.standalone: true`` secondary (``profiles_to_serve`` never serves it),
  78. ``True`` for an explicit ``true`` on the default, ``None`` when boot settles it (unset or the
  79. retired ``false``, see :func:`resolve_multiplex_mode`)."""
  80. from hermes_cli.profiles import profile_is_standalone
  81. if profile_is_standalone(home):
  82. return False
  83. return settled_explicit_flag(explicit_multiplex_flag(default_home))
  84. def default_gateway_multiplexes(default_home: Optional[Path] = None) -> bool:
  85. """Does the default profile's gateway serve every profile? For CLI/dashboard processes: the LIVE
  86. gateway's ``served_profiles`` record when one runs (it settled the unset default itself), else
  87. the explicit flag, else False — an unset flag is decided by the gateway at boot, never guessed
  88. here.
  89. The one thing that can no longer report "standalone" is an explicit ``false``: it is RETIRED
  90. (warned about and ignored at boot, see :func:`resolve_multiplex_mode`), so answering False from
  91. it made every CLI surface contradict the gateway that was about to multiplex anyway.
  92. """
  93. from hermes_constants import get_default_hermes_root
  94. from hermes_cli.gateway_multiplex_served import recorded_served_profiles
  95. root = Path(default_home) if default_home is not None else get_default_hermes_root()
  96. recorded = recorded_served_profiles(root)
  97. if recorded is not None:
  98. return bool(recorded)
  99. flag = explicit_multiplex_flag(root)
  100. return False if flag is None else True
  101. @dataclass(frozen=True)
  102. class MultiplexDecision:
  103. enabled: bool
  104. # "config" (config.yaml / env override — explicit), "default" (implicit default applied),
  105. # "guard" (implicit default refused; ``reason`` names the blocker).
  106. source: str
  107. reason: str = ""
  108. def _standalone_launcher() -> bool:
  109. from hermes_constants import get_hermes_home, profile_name_for_home
  110. from hermes_cli.profiles import profile_is_standalone
  111. home = get_hermes_home()
  112. return profile_name_for_home(home) not in (None, "default") and profile_is_standalone(home)
  113. def standalone_launcher_decision(config) -> Optional[MultiplexDecision]:
  114. """The per-profile opt-out also binds callers supplying an explicit GatewayConfig."""
  115. if not _standalone_launcher():
  116. return None
  117. config.multiplex_profiles = False
  118. return MultiplexDecision(False, "guard", STANDALONE_PROFILE_REASON)
  119. def implicit_multiplex_blocker() -> Optional[str]:
  120. """Why THIS process must not multiplex right now, or None when it may.
  121. Mirrors what makes ``hermes gateway migrate --multiplex`` refuse or leave a per-profile gateway
  122. in place: hosts whose per-profile gateways the preflight cannot see (s6 slots) stay standalone;
  123. a secondary that still runs its own gateway (live process or installed service) or a preflight
  124. blocker (duplicate bot credential, port binder without a ``/p/<profile>/`` ingress) keeps this
  125. gateway standalone.
  126. Every blocker here is a TRANSIENT, fixable condition, which is why this function is now also
  127. the whole answer for an explicit ``gateway.multiplex_profiles: false`` (see
  128. :func:`resolve_multiplex_mode`): the host converges the moment the blocker is gone.
  129. The launching profile's IDENTITY is deliberately not a blocker: multiplex-only means "the one
  130. host process", whichever profile started it. Gating on ``active == 'default'`` made a host
  131. whose only gateway runs under a named profile permanently standalone — and every lifecycle
  132. verb built on "the default's multiplexer" blind to the process actually serving the host.
  133. """
  134. from hermes_cli.profiles import profiles_to_serve
  135. if _standalone_launcher():
  136. return STANDALONE_PROFILE_REASON
  137. # Cheap and first: a single-profile install has nothing to multiplex, and the fail-closed secret
  138. # scope the multiplexer arms buys it nothing. (Also keeps every embedded/test runner off the
  139. # service-manager probes below.) Create a second profile and restart to start serving it.
  140. # Parking is reversible without a host restart, so keep the reconcile watcher alive.
  141. if len(profiles_to_serve(multiplex=True, include_parked=True)) < 2:
  142. return SINGLE_PROFILE_REASON
  143. from hermes_cli.gateway_migrate import MIGRATE_COMMAND, _host_supports_migration, build_migration_plan
  144. host_reason = _host_supports_migration()
  145. if host_reason:
  146. return host_reason
  147. plan = build_migration_plan()
  148. if plan.standalone_secondaries:
  149. owned = ", ".join(
  150. f"'{p.name}' ({'pid ' + str(p.pid) if p.pid else p.service_label()})"
  151. for p in plan.standalone_secondaries)
  152. return f"profile(s) {owned} still run their own gateway; fold them with `{MIGRATE_COMMAND}`"
  153. if plan.blocked:
  154. return "; ".join(plan.blockers)
  155. return None
  156. def _default_profile_home() -> Path:
  157. from hermes_constants import get_default_hermes_root
  158. return get_default_hermes_root()
  159. def persist_resolved_default(decision: MultiplexDecision, default_home: Optional[Path] = None) -> bool:
  160. """Write ``gateway.multiplex_profiles: true`` into the DEFAULT profile's config.yaml so the file
  161. reads as the gateway behaves. The key has ONE valid value right now (Teknium ruling): an unset key
  162. is made explicit ("left unset" was read as "off"), a retired ``false`` is rewritten in place and
  163. leaves a one-time marker for the boxed notice. Comment-preserving writer, once, and NEVER on a guard
  164. refusal (the file must not say true while the runtime is standalone). Returns True on a write."""
  165. from hermes_cli.config_backend import config_exists
  166. if not decision.enabled or decision.source == "guard":
  167. return False
  168. default_home = Path(default_home) if default_home is not None else _default_profile_home()
  169. cfg_path = default_home / "config.yaml"
  170. try:
  171. from hermes_cli.config import read_user_config_raw
  172. cfg = read_user_config_raw(cfg_path) or {} if config_exists(cfg_path) else {}
  173. section = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {}
  174. in_file = cfg.get("multiplex_profiles", section.get("multiplex_profiles"))
  175. if in_file is True:
  176. return False
  177. from hermes_cli.gateway_migrate import _write_multiplex_flag
  178. _write_multiplex_flag(default_home, True)
  179. if decision.source == "retired-opt-out":
  180. (default_home / REWRITTEN_MARKER_NAME).write_text(RETIRED_OPT_OUT_REASON + "\n", encoding="utf-8")
  181. except Exception:
  182. logger.debug("could not persist gateway.multiplex_profiles: true", exc_info=True)
  183. return False
  184. logger.info("Wrote gateway.multiplex_profiles: true to %s (was %s).", cfg_path,
  185. "unset" if in_file is None else repr(in_file))
  186. return True
  187. def retired_opt_out_notice_lines() -> list[str]:
  188. """The one-time boxed notice for a rewritten ``false`` (same box as the guard warning)."""
  189. return _box(["⚠ gateway.multiplex_profiles: false is retired and was rewritten to true;",
  190. "one gateway per host serves every profile.",
  191. "A per-profile gateway is `gateway.standalone: true` in that profile's config",
  192. "(temporary shim) or `--force`."])
  193. def consume_rewritten_notice(default_home: Optional[Path] = None) -> list[str]:
  194. """``hermes update``'s summary: print the rewrite notice ONCE more, then clear the marker."""
  195. default_home = Path(default_home) if default_home is not None else _default_profile_home()
  196. marker = default_home / REWRITTEN_MARKER_NAME
  197. if not marker.exists():
  198. return []
  199. try:
  200. marker.unlink()
  201. except OSError:
  202. return []
  203. return retired_opt_out_notice_lines()
  204. def _box(body: list[str]) -> list[str]:
  205. width = max(len(line) for line in body) + 2
  206. return ["┌" + "─" * width + "┐",
  207. *[f"│ {line.ljust(width - 1)}│" for line in body],
  208. "└" + "─" * width + "┘"]
  209. def resolve_multiplex_mode(config) -> MultiplexDecision:
  210. """Settle ``config.multiplex_profiles`` for one gateway boot; the config is updated in place.
  211. ``gateway.multiplex_profiles: false`` is RETIRED as a topology opt-out (multiplex-only ruling).
  212. The key still parses and still drives the runtime mode this function writes back, but an
  213. explicit ``false`` no longer pins a per-profile fleet: it is warned about and resolved exactly
  214. like an unset key. That is safe because the unset path is not optimistic — it refuses to
  215. multiplex while any real blocker holds (an s6 container, a secondary that still owns a gateway,
  216. a duplicate bot credential), so a host that genuinely cannot fold still comes up standalone and
  217. says why, and it converges by itself once ``hermes gateway migrate --multiplex`` has run.
  218. """
  219. current = getattr(config, "multiplex_profiles", None)
  220. standalone = standalone_launcher_decision(config)
  221. if standalone is not None:
  222. return standalone
  223. if settled_explicit_flag(current):
  224. return MultiplexDecision(True, "config")
  225. retired_opt_out = current is False
  226. try:
  227. blocker = implicit_multiplex_blocker()
  228. except Exception as exc: # a broken preflight must not take the gateway down with it
  229. logger.warning("Multiplex preflight failed; starting standalone: %s", exc, exc_info=True)
  230. blocker = f"preflight failed ({exc})"
  231. if blocker:
  232. decision = MultiplexDecision(False, "guard", blocker)
  233. elif retired_opt_out:
  234. decision = MultiplexDecision(True, "retired-opt-out", RETIRED_OPT_OUT_REASON)
  235. else:
  236. decision = MultiplexDecision(True, "default", "gateway.multiplex_profiles unset; default applies")
  237. config.multiplex_profiles = decision.enabled
  238. return decision
  239. _pending_decision: list[MultiplexDecision] = []
  240. def record_multiplex_decision(decision: MultiplexDecision) -> None:
  241. """Persist a guard refusal into ``gateway_state.json`` so `hermes gateway status` can show why this
  242. gateway serves one profile while the default says multiplex; any other verdict clears the field.
  243. Only the runtime-lock owner writes that file. The decision is made while the config loads, before
  244. the lock claim, and a pre-lock write published this process's pid as a live ``starting`` gateway:
  245. a launch racing it read that as an owner and refused, leaving no gateway at all. Until this process
  246. holds the lock the decision stays pending; ``publish_pending_multiplex_decision`` writes it once
  247. the claim succeeds, and a launch that loses the claim never writes it."""
  248. try:
  249. from gateway.status import owns_gateway_runtime_lock, write_runtime_status
  250. if not owns_gateway_runtime_lock():
  251. _pending_decision[:] = [decision]
  252. return
  253. _pending_decision.clear()
  254. write_runtime_status(multiplex_standalone_reason=decision.reason if decision.source == "guard" else None)
  255. except Exception:
  256. logger.debug("could not record the multiplex decision", exc_info=True)
  257. def publish_pending_multiplex_decision() -> None:
  258. """Write the decision held back before the runtime-lock claim (called right after the claim)."""
  259. if _pending_decision:
  260. record_multiplex_decision(_pending_decision[-1])
  261. def log_multiplex_decision(decision: MultiplexDecision) -> None:
  262. record_multiplex_decision(decision)
  263. if decision.source == "retired-opt-out":
  264. logger.warning("%s", RETIRED_OPT_OUT_REASON)
  265. persist_resolved_default(decision)
  266. for line in retired_opt_out_notice_lines():
  267. print(line)
  268. elif decision.source == "guard" and decision.reason == SINGLE_PROFILE_REASON:
  269. logger.info("Single-profile install: gateway.multiplex_profiles unset, serving the default profile only.")
  270. elif decision.source == "guard":
  271. logger.warning(
  272. "This gateway stays standalone: %s. It serves only the launching profile.",
  273. decision.reason)
  274. for line in standalone_warning_lines(decision):
  275. print(line)
  276. elif decision.source == "default":
  277. logger.info("Serving every profile on this host (gateway.multiplex_profiles unset; default on).")
  278. persist_resolved_default(decision)
  279. def unserved_profiles() -> list[str]:
  280. """Named profiles a standalone gateway leaves without a bot (the whole point of the warning)."""
  281. from hermes_constants import get_hermes_home, profile_name_for_home
  282. from hermes_cli.profiles import profiles_to_serve
  283. me = profile_name_for_home(get_hermes_home()) or "default"
  284. return [name for name, _home in profiles_to_serve(multiplex=True, include_parked=True) if name != me]
  285. def standalone_warning_lines(decision: MultiplexDecision, unserved: Optional[list[str]] = None) -> list[str]:
  286. """The boxed warning a multi-profile host prints when a guard keeps its gateway standalone.
  287. Empty for anything but a guard refusal on a host with other profiles to serve: a single-profile
  288. install has nothing unserved, so there is nothing to shout about. The same box appears at
  289. gateway start, in the ``hermes update`` summary and (as text) in the dashboard banner.
  290. """
  291. if decision.source != "guard" or decision.reason == SINGLE_PROFILE_REASON:
  292. return []
  293. if unserved is None:
  294. try:
  295. unserved = unserved_profiles()
  296. except Exception:
  297. unserved = []
  298. if not unserved:
  299. return []
  300. from hermes_cli.gateway_migrate import MIGRATE_COMMAND
  301. body = [
  302. "⚠ This gateway is STANDALONE: it serves only its own profile.",
  303. "Profiles NOT served (their bots stay silent): " + ", ".join(unserved),
  304. f"Why: {decision.reason}",
  305. f"Fix: {MIGRATE_COMMAND}",
  306. ]
  307. return _box(body)
  308. def recorded_standalone_reason(state_path: Path | None = None) -> str | None:
  309. """The guard refusal a LIVE standalone gateway published in its ``gateway_state.json`` (this
  310. home's, or *state_path*), else ``None``. A stopped, failed or dead record's reason is history."""
  311. try:
  312. from gateway.status import read_runtime_status, runtime_status_pid_is_live
  313. record = read_runtime_status(state_path) or {}
  314. if record.get("gateway_state") in (None, "stopped", "startup_failed"):
  315. return None
  316. # Liveness is the state + a live PID, never the heartbeat: a paused or wedged gateway is
  317. # still the standalone process the box warns about (#120991).
  318. if not runtime_status_pid_is_live(record):
  319. return None
  320. reason = record.get("multiplex_standalone_reason")
  321. # The readers already absorb a missing/garbled file; what is left is a hostile record shape
  322. # (a non-object, a PID too large for the OS) or a liveness probe's OS error.
  323. except (OSError, ValueError, TypeError, AttributeError, OverflowError):
  324. return None
  325. return str(reason) if isinstance(reason, str) and reason else None
  326. def recorded_standalone_warning_lines() -> list[str]:
  327. """Same box, rebuilt from the live gateway's ``gateway_state.json`` for processes that did not
  328. make the decision (``hermes update``'s summary, ``hermes gateway status``)."""
  329. reason = recorded_standalone_reason()
  330. if not reason:
  331. return []
  332. return standalone_warning_lines(MultiplexDecision(False, "guard", reason))