gateway_migrate.py 64 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289
  1. """``hermes gateway migrate --multiplex``: converge a per-profile-gateway install onto the ONE
  2. host gateway, with a table-driven preflight.
  3. Multiplex-only (Teknium ruling): exactly one ``hermes gateway run`` per host, serving every
  4. profile. This command is the supported convergence path, and it is defined by TOPOLOGY, not by a
  5. config flag — a host is converged when no secondary profile owns a gateway process or a supervisor
  6. unit any more. That makes it re-runnable: a half-migrated host (flag flipped, a unit left behind, a
  7. crash between the two) converges on the next run instead of being reported "already multiplexed".
  8. There is no ``--standalone`` rollback command: reinstalling per-profile services is no longer a
  9. supported topology. The rollback machinery survives as the COMPENSATOR inside a single failed
  10. apply (:func:`rollback_migration`) — which restores exactly ONE gateway, the default's, because
  11. the host gateway lock now refuses the fleet it used to rebuild — and the recorded manifest is what
  12. the next re-run resumes from. A manifest on disk outranks every preflight gate: a resume
  13. compensates an already-destructive state instead of initiating one.
  14. The preflight reuses the gateway's own conflict logic (``GatewayRunner._adapter_credential_fingerprint``,
  15. ``platform_binds_port``, the adapters' ``serves_profile_prefix`` declaration) so its verdict matches
  16. what the multiplexer would do at startup. ``hermes update`` calls :func:`maybe_auto_migrate_after_update`.
  17. """
  18. from __future__ import annotations
  19. import contextlib
  20. import json
  21. import logging
  22. import os
  23. import sys
  24. import time
  25. from dataclasses import dataclass, field
  26. from pathlib import Path
  27. from types import SimpleNamespace
  28. from typing import Callable, Iterator, Optional
  29. from agent.deadline import poll_until
  30. logger = logging.getLogger(__name__)
  31. MANIFEST_NAME = "gateway_migration.json"
  32. MIGRATE_COMMAND = "hermes gateway migrate --multiplex"
  33. _SERVED_WAIT_SECONDS = 90.0
  34. # --------------------------------------------------------------------------- data
  35. @dataclass
  36. class ProfileGateway:
  37. """One profile's standalone gateway footprint: live PID and/or installed service."""
  38. name: str
  39. home: Path
  40. pid: Optional[int] = None
  41. # EVERY installed unit: [("systemd", system), ("launchd", False)]. A profile can carry a user AND a
  42. # system unit at once; recording only the first found left the other live beside the multiplexer.
  43. services: list[tuple[str, bool]] = field(default_factory=list)
  44. run_as_user: Optional[str] = None # User= recorded by a system-scope systemd unit
  45. uid: Optional[int] = None # owner of the gateway process/unit; None = unknown (never "different")
  46. runtime_home: Optional[Path] = None # HERMES_HOME the installed unit pins, when it differs from ``home``
  47. @property
  48. def is_default(self) -> bool:
  49. return self.name == "default"
  50. @property
  51. def service(self) -> Optional[tuple[str, bool]]:
  52. """The unit the default gateway is (re)started through; None when nothing is installed."""
  53. return self.services[0] if self.services else None
  54. @property
  55. def has_gateway(self) -> bool:
  56. return self.pid is not None or bool(self.services)
  57. @property
  58. def has_system_unit(self) -> bool:
  59. return ("systemd", True) in self.services
  60. def service_label(self) -> str:
  61. return " + ".join(_service_label(s) for s in self.services) if self.services else "none"
  62. def to_dict(self) -> dict:
  63. return {
  64. "profile": self.name, "home": str(self.home), "pid": self.pid,
  65. "service": None if self.service is None else _service_dict(self.service),
  66. "services": [_service_dict(s) for s in self.services],
  67. "run_as_user": self.run_as_user,
  68. "uid": self.uid, "runtime_home": None if self.runtime_home is None else str(self.runtime_home),
  69. }
  70. def _service_label(service: tuple[str, bool]) -> str:
  71. kind, system = service
  72. if kind == "systemd":
  73. return f"systemd ({'system' if system else 'user'})"
  74. if kind == "s6":
  75. return "s6 slot"
  76. return "Windows scheduled task" if kind == "windows" else kind
  77. def _remove_verb(service: tuple[str, bool]) -> str:
  78. """An s6 slot is parked down (it stays registered as the `hermes -p X gateway start` target), every
  79. other unit is uninstalled."""
  80. return "park" if service[0] == "s6" else "uninstall"
  81. def _service_dict(service: tuple[str, bool]) -> dict:
  82. return {"kind": service[0], "system": service[1]}
  83. @dataclass
  84. class MigrationPlan:
  85. default_home: Path
  86. profiles: list[ProfileGateway]
  87. multiplex_flag_on: bool
  88. live_served: Optional[list[str]] # served_profiles the live default gateway recorded, if any
  89. # The migration manifest on disk, when one exists. Its PRESENCE is the "this host is already
  90. # mid-migration, with units removed" signal that turns every later gate from a refusal into a
  91. # finding (see :func:`apply_migration`): a resume compensates a destructive state, it never
  92. # initiates one.
  93. manifest: Optional[dict] = None
  94. # A manifest with the flag on and no LIVE default gateway: an earlier apply died between flipping
  95. # the flag and the multiplexer confirming it is up (#110850). An installed unit is not proof of
  96. # anything — `systemd_install` writes the unit before the start that can still fail or be killed.
  97. # Not "already multiplexed" — resumable.
  98. interrupted: bool = False
  99. blockers: list[str] = field(default_factory=list)
  100. notices: list[str] = field(default_factory=list)
  101. # Profiles that authored `gateway.standalone: true`: they keep their own gateway and are neither
  102. # a blocker nor a fold target — the plan names them so the operator knows they were left alone.
  103. standalone_by_config: tuple[str, ...] = ()
  104. @property
  105. def secondaries(self) -> list[ProfileGateway]:
  106. return [p for p in self.profiles if not p.is_default]
  107. @property
  108. def default(self) -> ProfileGateway:
  109. return next(p for p in self.profiles if p.is_default)
  110. @property
  111. def already_multiplexed(self) -> bool:
  112. """Converged: nothing is left for this command to do.
  113. TOPOLOGY, not the flag. A host whose flag is on while a secondary still owns a gateway
  114. process or a supervisor unit is HALF-migrated; reporting that as "already multiplexed"
  115. made the re-run a no-op on exactly the host that needed it most (#100896).
  116. """
  117. if self.interrupted or self.standalone_secondaries:
  118. return False
  119. return self.multiplex_flag_on or bool(self.live_served and len(self.live_served) > 1)
  120. @property
  121. def standalone_secondaries(self) -> list[ProfileGateway]:
  122. return [p for p in self.secondaries if p.has_gateway]
  123. @property
  124. def expected_served_names(self) -> set[str]:
  125. from hermes_cli.profiles import profile_is_parked
  126. return {p.name for p in self.profiles if p.is_default or not profile_is_parked(p.home)}
  127. @property
  128. def blocked(self) -> bool:
  129. return bool(self.blockers)
  130. def target_service_kind(self) -> Optional[tuple[str, bool]]:
  131. """Service manager the default gateway should end up on: its own, else the one the
  132. secondaries used (so a systemd-managed fleet stays systemd-managed)."""
  133. if self.default.service is not None:
  134. return self.default.service
  135. return next((p.service for p in self.secondaries if p.service is not None), None)
  136. def target_run_as_user(self) -> Optional[str]:
  137. """Preserve the system unit identity that the default unit replaces."""
  138. if self.default.run_as_user:
  139. return self.default.run_as_user
  140. return next((p.run_as_user for p in self.secondaries if p.run_as_user), None)
  141. def to_dict(self) -> dict:
  142. return {
  143. "default_home": str(self.default_home),
  144. "profiles": [p.to_dict() for p in self.profiles],
  145. "standalone_by_config": list(self.standalone_by_config),
  146. "multiplex_flag_on": self.multiplex_flag_on,
  147. "live_served": self.live_served,
  148. "already_multiplexed": self.already_multiplexed,
  149. "interrupted": self.interrupted,
  150. "blockers": list(self.blockers),
  151. "notices": list(self.notices),
  152. "eligible": self.eligible_for_migration(),
  153. "command": MIGRATE_COMMAND,
  154. }
  155. def eligible_for_migration(self) -> bool:
  156. """>= 2 profiles, at least one secondary with its own gateway, multiplex off, no blockers.
  157. This is the AUTO-migration (``hermes update``) bar; the explicit command also proceeds with
  158. zero standalone secondaries (see :func:`cmd_migrate`)."""
  159. return (
  160. len(self.profiles) >= 2 and bool(self.standalone_secondaries)
  161. and not self.already_multiplexed and not self.blocked
  162. )
  163. # --------------------------------------------------------------------------- home / env plumbing
  164. @contextlib.contextmanager
  165. def _home_env(home: Path) -> Iterator[None]:
  166. """Run service-manager helpers as if ``home`` were the active HERMES_HOME. Both the contextvar
  167. override (``get_hermes_home``) and ``os.environ`` (``gateway.status`` identity files, unit
  168. generation) are switched, then restored.
  169. The process keeps its LAUNCH identity while swapped (``pin_process_hermes_home``): ``os.environ``
  170. still holds the launch profile's ``.env``, so a child spawned in here for ``home`` must see it as
  171. a routed home and take only ``home``'s own secrets — with the env var swapped, the routed-home
  172. checks read ``home`` as the launch profile and handed the launch profile's credentials to the
  173. default gateway."""
  174. from hermes_constants import (
  175. get_routing_process_hermes_home, pin_process_hermes_home, process_hermes_home_is_pinned,
  176. reset_hermes_home_override, set_hermes_home_override,
  177. )
  178. import hermes_constants
  179. previous = os.environ.get("HERMES_HOME")
  180. pinned_here = not process_hermes_home_is_pinned()
  181. if pinned_here:
  182. pin_process_hermes_home(get_routing_process_hermes_home())
  183. token = set_hermes_home_override(str(home))
  184. os.environ["HERMES_HOME"] = str(home)
  185. hermes_constants._default_hermes_root_memo = None
  186. try:
  187. yield
  188. finally:
  189. if pinned_here:
  190. pin_process_hermes_home(None)
  191. reset_hermes_home_override(token)
  192. if previous is None:
  193. os.environ.pop("HERMES_HOME", None)
  194. else:
  195. os.environ["HERMES_HOME"] = previous
  196. hermes_constants._default_hermes_root_memo = None
  197. def _default_home() -> Path:
  198. from hermes_constants import get_default_hermes_root
  199. return get_default_hermes_root()
  200. def _profile_homes() -> list[tuple[str, Path]]:
  201. from hermes_cli.profiles import profiles_to_serve
  202. return list(profiles_to_serve(multiplex=True, include_parked=True))
  203. def _live_gateway_pid(home: Path) -> Optional[int]:
  204. """Verified PID of a gateway SERVING ``home``, else None (never raises: a probe failure must
  205. not abort a migration plan).
  206. Topology REPORTING, not ownership: ``live_gateway_pid_for_home`` deliberately answers with the
  207. host multiplexer's PID for every home it serves (``gateway.status._host_gateway_serves_home``),
  208. so "this profile is being served" reads as a live PID. Use :func:`_own_gateway_pid` for the
  209. question this command acts on.
  210. """
  211. from gateway.status import live_gateway_pid_for_home
  212. with contextlib.suppress(Exception):
  213. return live_gateway_pid_for_home(home)
  214. return None
  215. def _host_gateway_owner():
  216. """The ONE live host gateway (``gateway.host_attach.HostGateway``) or None. Never raises."""
  217. with contextlib.suppress(Exception):
  218. from gateway.host_attach import host_gateway
  219. return host_gateway()
  220. return None
  221. def _own_gateway_pid(home: Path, owner) -> Optional[int]:
  222. """PID of a gateway ``home`` OWNS, else None.
  223. Ownership is what this command may stop, and the host multiplexer is not owned by the profiles
  224. it serves: it is one process, launched from one home. Deriving ownership from
  225. :func:`_live_gateway_pid` made every secondary on a CONVERGED host report the host gateway's
  226. PID, so the plan called the host half-migrated and each `hermes update` SIGTERMed the only
  227. gateway it had. The host process counts only for the home it was actually launched from; every
  228. other profile it serves owns nothing.
  229. """
  230. pid = _live_gateway_pid(home)
  231. if pid is None or owner is None or pid != owner.pid:
  232. return pid
  233. with contextlib.suppress(Exception):
  234. from gateway.status import _same_hermes_home
  235. return pid if _same_hermes_home(Path(owner.home), Path(home)) else None
  236. return None
  237. def _gateway_identity(home: Path, pid: Optional[int], services: list[tuple[str, bool]]) -> tuple[Optional[int], Path]:
  238. from hermes_cli.gateway_migrate_guards import gateway_identity
  239. return gateway_identity(home, pid, services)
  240. def _installed_services(home: Path) -> list[tuple[str, bool]]:
  241. """Every installed service for ``home``'s gateway (units / plist / scheduled task), user scope first.
  242. Under s6 the footprint is the SLOT: the root slot always (it is what a restart goes through),
  243. a named profile's slot only while it is UP — a registered-down slot is what the container's
  244. boot leaves behind for every named profile and is not a gateway."""
  245. from hermes_cli import gateway as gw
  246. found: list[tuple[str, bool]] = []
  247. if gw._running_under_s6():
  248. from hermes_cli.gateway_multiplex_s6 import named_slot_name, slot_is_up
  249. from hermes_cli.service_manager import S6ServiceManager
  250. from hermes_constants import profile_name_for_home
  251. name = profile_name_for_home(home) or "default"
  252. slot_dir = S6ServiceManager().scandir / named_slot_name(name)
  253. if slot_dir.is_dir() and (name == "default" or slot_is_up(name)):
  254. found.append(("s6", False))
  255. return found
  256. with _home_env(home):
  257. if gw.supports_systemd_services():
  258. found.extend(("systemd", system) for system in (False, True) if gw.get_systemd_unit_path(system=system).exists())
  259. if gw.is_macos() and gw.get_launchd_plist_path().exists():
  260. found.append(("launchd", False))
  261. if gw.is_windows() and _windows_task_installed():
  262. found.append(("windows", False))
  263. return found
  264. def _windows_task_installed() -> bool:
  265. """Is a per-profile Windows gateway installed for the ACTIVE ``HERMES_HOME``?
  266. ``get_task_name()`` is home-suffixed, so this answers per profile exactly the way the systemd
  267. unit path does. Either half counts: ``hermes gateway install`` falls back to a Startup-folder
  268. entry when it cannot register a scheduled task, and a migration that removed only the task
  269. would leave the fallback launching a second gateway at the next logon.
  270. """
  271. from hermes_cli import gateway_windows as gww
  272. with contextlib.suppress(Exception):
  273. return bool(gww.is_task_registered() or gww.is_startup_entry_installed())
  274. return False
  275. def _systemd_service_user(home: Path, services: list[tuple[str, bool]]) -> Optional[str]:
  276. """Read ``User=`` before migration removes a system-scope unit."""
  277. if ("systemd", True) not in services:
  278. return None
  279. from hermes_cli import gateway as gw
  280. with _home_env(home):
  281. return gw._read_systemd_user_from_unit(gw.get_systemd_unit_path(system=True))
  282. def _service_op(kind: str, system: bool, verb: str, home: Path, *, run_as_user: Optional[str] = None) -> None:
  283. """``stop`` / ``uninstall`` / ``start`` / ``restart`` / ``install`` / ``enable`` on ``home``'s service."""
  284. if kind == "s6":
  285. return _s6_slot_op(verb, home)
  286. from hermes_cli import gateway as gw
  287. with _home_env(home):
  288. if verb == "install":
  289. if kind == "launchd":
  290. gw.launchd_install()
  291. elif kind == "windows":
  292. # Non-interactive: the migration already asked; prompting here would hang a
  293. # supervised/`--yes` run on a console that has no operator.
  294. from hermes_cli import gateway_windows as gww
  295. gww.install(start_now=True, start_on_login=True)
  296. else:
  297. gw.systemd_install(system=system, run_as_user=run_as_user, non_interactive=True)
  298. return
  299. if verb == "enable":
  300. # Boot enablement only: the migration's uninstall of every secondary destroys their
  301. # boot enablement (systemd: disable + default.target.wants unlink), so the survivor's is
  302. # load-bearing for the whole host — and `restart` on a disabled unit succeeds, so a
  303. # reboot came up with no gateway while the migration reported success (#124922).
  304. # launchd (RunAtLoad plist) and the Windows logon task are their own enablement and the
  305. # migration never removes the survivor's.
  306. if kind != "systemd":
  307. return
  308. enabled = gw._run_systemctl(["enable", gw.get_service_name()], system=system, check=False, timeout=30)
  309. if getattr(enabled, "returncode", 0) != 0:
  310. raise RuntimeError(f"could not enable {gw.get_service_name()} at boot (systemctl enable "
  311. f"exited {enabled.returncode}); a reboot would come up with no gateway")
  312. return
  313. gw._service_call(kind, verb, system)
  314. def _stop_gateway_process(home: Path) -> None:
  315. from hermes_cli.profiles import _stop_gateway_process
  316. _stop_gateway_process(home)
  317. def _s6_slot_op(verb: str, home: Path) -> None:
  318. """The s6 leg of :func:`_service_op`. A named profile's slot is never uninstalled: it stays
  319. registered DOWN (``down`` file) as the target of a later ``hermes -p X gateway start``, exactly
  320. the shape the container's boot produces. The root slot is (re)started so it re-reads its config."""
  321. from hermes_cli.gateway_multiplex_s6 import bring_root_slot_up, park_named_slot
  322. from hermes_constants import profile_name_for_home
  323. name = profile_name_for_home(home) or "default"
  324. if name == "default":
  325. if verb in ("start", "restart"):
  326. bring_root_slot_up()
  327. return # install/uninstall/stop of the root slot are not migration steps
  328. if verb == "stop":
  329. park_named_slot(name)
  330. def _spawn_detached_gateway(home: Path) -> bool:
  331. from hermes_cli import gateway as gw
  332. with _home_env(home):
  333. return gw._spawn_detached_gateway()
  334. def _read_multiplex_flag(default_home: Path) -> bool:
  335. """The operator's EXPLICIT opt-in only. The unset default (on) is settled by the default gateway at
  336. boot and refused while a secondary runs its own gateway — exactly the fleet this command folds —
  337. so the plan reads it as "not yet multiplexed" and the migration proceeds."""
  338. from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag
  339. return explicit_multiplex_flag(default_home) is True
  340. def _write_multiplex_flag(default_home: Path, value: bool) -> None:
  341. """Set ``gateway.multiplex_profiles`` in the DEFAULT profile's config.yaml through the config API
  342. (same read-guard + nested-set + atomic write ``hermes config set`` uses; no raw YAML edits)."""
  343. from hermes_cli.config import _set_nested, _write_user_config, require_readable_config_before_write
  344. cfg_path = default_home / "config.yaml"
  345. user_config = require_readable_config_before_write(cfg_path)
  346. # A stale top-level alias would shadow the nested key the docs describe.
  347. user_config.pop("multiplex_profiles", None)
  348. _set_nested(user_config, "gateway.multiplex_profiles", value)
  349. _write_user_config(cfg_path, user_config)
  350. # --------------------------------------------------------------------------- preflight checks
  351. def _profile_gateway_config(home: Path):
  352. """This profile's ``GatewayConfig`` read exactly the way the multiplexer reads it: under the
  353. profile's own secret scope with multiplexing active, so a missing token stays missing instead of
  354. borrowing the CLI process's ``os.environ`` (which holds the launch profile's ``.env``).
  355. A PARKED profile stays in the inventory and the duplicate-credential guard (parking must not hide
  356. a conflict), but it is inert for the host: its config is read without discovering its plugins,
  357. whose ``register()`` would otherwise run in the live gateway on every boot (#123386). Only its
  358. plugin-defined platforms fall out of the guard — their credentials cannot be evaluated without
  359. importing the plugin; builtin bot tokens still collide.
  360. """
  361. from gateway.config import load_gateway_config
  362. from gateway.run import _profile_runtime_scope
  363. from hermes_cli.plugins_discovery import suppress_plugin_discovery
  364. from hermes_cli.profiles import profile_is_parked
  365. scope = suppress_plugin_discovery() if profile_is_parked(home) else contextlib.nullcontext()
  366. with _profile_runtime_scope(home), scope:
  367. return load_gateway_config()
  368. @contextlib.contextmanager
  369. def _multiplex_read_mode() -> Iterator[None]:
  370. from agent.secret_scope import is_multiplex_active, set_multiplex_active
  371. previous = is_multiplex_active()
  372. set_multiplex_active(True)
  373. try:
  374. yield
  375. finally:
  376. set_multiplex_active(previous)
  377. def _credential_probe(platform_config) -> SimpleNamespace:
  378. """Config-shaped stand-in for ``GatewayRunner._adapter_credential_fingerprint`` (which probes
  379. adapter attributes): token/api_key plus the id-style credentials adapters expose from ``extra``."""
  380. extra = getattr(platform_config, "extra", None) or {}
  381. return SimpleNamespace(
  382. token=getattr(platform_config, "token", None) or getattr(platform_config, "api_key", None),
  383. _app_id=extra.get("app_id"), _client_id=extra.get("client_id"), _bot_id=extra.get("bot_id"),
  384. _project_secret=extra.get("project_secret"), config=platform_config,
  385. )
  386. def _credential_claims(config) -> dict[tuple, str]:
  387. """``(platform, fingerprint)`` for every enabled platform with a discoverable credential."""
  388. from gateway.run import GatewayRunner
  389. claims: dict[tuple, str] = {}
  390. for platform, platform_config in config.platforms.items():
  391. if not platform_config.enabled:
  392. continue
  393. fp = GatewayRunner._adapter_credential_fingerprint(_credential_probe(platform_config))
  394. if fp is not None:
  395. claims[(platform.value, fp)] = platform.value
  396. return claims
  397. def _credential_key_names(platform_value: str) -> str:
  398. """Env key NAMES (never values) that make ``platform_value`` connect as a bot, e.g.
  399. ``TELEGRAM_BOT_TOKEN``; the platform id when no key is registered (config.yaml-only token)."""
  400. from hermes_cli.profile_channels import credential_env_keys
  401. names = sorted(key for key, pid in credential_env_keys().items() if pid == platform_value)
  402. return "/".join(names) or f"the {platform_value} token"
  403. def duplicate_credential_lines(configs: list[tuple[str, object]]) -> list[str]:
  404. """One finding per platform credential two profiles both hold, with the remedy. The SINGLE
  405. source for the migrate preflight, ``hermes doctor`` and ``hermes gateway status``, so all three
  406. name the same duplicates the same way: profile names + key names only, never a value or hash."""
  407. owners: dict[tuple, str] = {}
  408. lines: list[str] = []
  409. for name, cfg in configs: # default first: it wins the claim, like at multiplexer startup
  410. for claim, platform_value in _credential_claims(cfg).items():
  411. owner = owners.setdefault(claim, name)
  412. if owner == name:
  413. continue
  414. key = _credential_key_names(platform_value)
  415. lines.append(
  416. f"Profiles '{owner}' and '{name}' both hold the same {platform_value} credential ({key}): "
  417. f"one platform token can serve only one gateway, so the bot answers from whichever profile "
  418. f"claims it first and the other's adapter is parked. Give '{name}' its own bot token, or remove "
  419. f"{key} from the profile that should not own it (or keep it in {owner} and route {name}'s "
  420. f"chats with profile_routes — gateway.profile_routes in {owner}'s config.yaml), then run "
  421. f"{MIGRATE_COMMAND}."
  422. )
  423. return lines
  424. def duplicate_credential_findings() -> list[str]:
  425. """The preflight's duplicate-credential check read straight from the local profile homes, for
  426. diagnostics that have no migration plan (doctor, gateway status). A profile whose gateway config
  427. does not load is skipped here — ``build_migration_plan`` reports that one as its own blocker."""
  428. configs: list[tuple[str, object]] = []
  429. with _multiplex_read_mode():
  430. for name, home in _profile_homes():
  431. with contextlib.suppress(Exception):
  432. configs.append((name, _profile_gateway_config(home)))
  433. return duplicate_credential_lines(configs)
  434. def _check_duplicate_credentials(plan: MigrationPlan, configs: dict[str, object]) -> None:
  435. """BLOCKER: the same bot credential configured on two profiles — the multiplexer would park
  436. the duplicate adapter, so one profile's bot would go silent after migration."""
  437. plan.blockers.extend(duplicate_credential_lines(
  438. [(p.name, configs[p.name]) for p in plan.profiles if p.name in configs]))
  439. def platform_serves_profile_prefix(platform_value: str) -> bool:
  440. """True when the adapter for ``platform_value`` declares ``serves_profile_prefix`` (it answers
  441. ``/p/<profile>/...`` on the default listener). Read from the adapter CLASS — builtin table or the
  442. plugin registry entry — never from a hand-kept list, so new ingress adapters count automatically."""
  443. from gateway.platforms.base import BasePlatformAdapter
  444. def _declares(cls) -> bool:
  445. return isinstance(cls, type) and issubclass(cls, BasePlatformAdapter) and bool(
  446. getattr(cls, "serves_profile_prefix", False))
  447. with contextlib.suppress(Exception):
  448. from gateway.config import Platform
  449. from gateway.run import _BUILTIN_ADAPTERS, _builtin_adapter_import
  450. spec = _BUILTIN_ADAPTERS.get(Platform(platform_value))
  451. if spec is not None:
  452. adapter_cls, _ok = _builtin_adapter_import(spec[0], spec[1], spec[2])
  453. return _declares(adapter_cls)
  454. with contextlib.suppress(Exception):
  455. # Plugin-shipped adapters (sms, line, teams, feishu, wecom, ...) only exist in the registry
  456. # after discovery; a bare CLI process has not run it yet.
  457. from hermes_cli.plugins import discover_plugins
  458. discover_plugins() # idempotent
  459. from gateway.platform_registry import platform_registry
  460. entry = platform_registry.get(platform_value)
  461. if entry is not None:
  462. factory = entry.adapter_factory
  463. if _declares(factory):
  464. return True
  465. # Lambda factories: the adapter class lives in the factory's module.
  466. import importlib
  467. module = importlib.import_module(factory.__module__)
  468. return any(_declares(getattr(module, name)) for name in dir(module))
  469. return False
  470. def _listener_url(default_cfg, platform_value: str, profile: str) -> str:
  471. from gateway.config import Platform
  472. extra = {}
  473. with contextlib.suppress(Exception):
  474. extra = (default_cfg.platforms.get(Platform(platform_value)) or SimpleNamespace(extra={})).extra or {}
  475. defaults = {"api_server": ("127.0.0.1", 8642), "webhook": ("0.0.0.0", 8644)}
  476. host, port = defaults.get(platform_value, ("<host>", "<port>"))
  477. host = extra.get("host") or host
  478. port = extra.get("port") or port
  479. tail = {"api_server": "/v1/...", "webhook": "/webhooks/<route>"}.get(platform_value, "/...")
  480. from hermes_cli.url_utils import format_url_host
  481. return f"http://{format_url_host(host)}:{port}/p/{profile}{tail}"
  482. def _check_secondary_port_binders(plan: MigrationPlan, configs: dict[str, object]) -> None:
  483. """BLOCKER when a secondary enables a port-binding platform with no ``/p/<profile>/`` ingress
  484. (the multiplexer skips the whole profile); NOTICE (URL changes) when the ingress exists."""
  485. from gateway.config import platform_binds_port
  486. default_cfg = configs.get("default")
  487. for profile in plan.secondaries:
  488. cfg = configs.get(profile.name)
  489. if cfg is None:
  490. continue
  491. for platform, platform_config in cfg.platforms.items():
  492. if not platform_config.enabled or not platform_binds_port(platform.value, platform_config.extra):
  493. continue
  494. if platform_serves_profile_prefix(platform.value):
  495. plan.notices.append(
  496. f"Profile '{profile.name}': {platform.value} moves onto the default listener at "
  497. f"{_listener_url(default_cfg, platform.value, profile.name)} (its key/secret is "
  498. f"unchanged; update clients that call the old per-profile port)."
  499. )
  500. else:
  501. plan.blockers.append(
  502. f"Profile '{profile.name}' enables {platform.value}, which binds its own port and has no "
  503. f"/p/{profile.name}/ ingress on the default listener yet; the multiplexer would skip "
  504. f"the whole profile. Disable it there (platforms.{platform.value}.enabled: false) or "
  505. f"add a /p/<profile>/ ingress for it first."
  506. )
  507. _PREFLIGHT_CHECKS: tuple[Callable[[MigrationPlan, dict[str, object]], None], ...] = (
  508. _check_duplicate_credentials,
  509. _check_secondary_port_binders,
  510. )
  511. def _load_profile_configs(plan: MigrationPlan) -> dict[str, object]:
  512. configs: dict[str, object] = {}
  513. with _multiplex_read_mode():
  514. for profile in plan.profiles:
  515. try:
  516. configs[profile.name] = _profile_gateway_config(profile.home)
  517. except Exception as exc: # unreadable config is itself a blocker, not a crash
  518. plan.blockers.append(f"Profile '{profile.name}': could not load its gateway config ({exc}).")
  519. return configs
  520. def build_migration_plan() -> MigrationPlan:
  521. """Enumerate profiles + their OWN gateway footprint, then run every preflight check."""
  522. from hermes_cli.gateway_multiplex_served import recorded_served_profiles
  523. default_home = _default_home()
  524. # Probed ONCE: every profile's ownership verdict is relative to the same host process.
  525. owner = _host_gateway_owner()
  526. profiles = []
  527. for name, home in _profile_homes():
  528. pid, services = _own_gateway_pid(home, owner), _installed_services(home)
  529. uid, runtime_home = _gateway_identity(home, pid, services)
  530. profiles.append(ProfileGateway(name=name, home=home, pid=pid, services=services,
  531. run_as_user=_systemd_service_user(home, services), uid=uid,
  532. runtime_home=None if runtime_home == home else runtime_home))
  533. plan = MigrationPlan(
  534. default_home=default_home, profiles=profiles,
  535. multiplex_flag_on=_read_multiplex_flag(default_home),
  536. live_served=recorded_served_profiles(default_home),
  537. manifest=_read_manifest(default_home),
  538. )
  539. from hermes_cli.profiles import profiles_to_serve
  540. foldable = {name for name, _home in _profile_homes()}
  541. plan.standalone_by_config = tuple(
  542. name for name, _home in profiles_to_serve(True, include_standalone=True)
  543. if name != "default" and name not in foldable)
  544. plan.interrupted = plan.multiplex_flag_on and _manifest_not_yet_served(plan.manifest, plan.live_served)
  545. if len(plan.profiles) < 2:
  546. plan.notices.append("Only one profile exists: nothing to multiplex.")
  547. return plan
  548. configs = _load_profile_configs(plan)
  549. for check in _PREFLIGHT_CHECKS:
  550. check(plan, configs)
  551. from hermes_cli.gateway_migrate_guards import auto_migration_blockers
  552. # Notices, not blockers: the explicit command is the operator's decision; only the update hook
  553. # refuses to cross these boundaries on its own.
  554. plan.notices.extend(f"Not migrated automatically by `hermes update`: {b}" for b in auto_migration_blockers(plan))
  555. plan.notices.append(
  556. "Profiles created after the migration are served by the running multiplexer as soon as "
  557. "they exist (it rescans profiles/ on create/delete and every 30s)."
  558. )
  559. return plan
  560. # --------------------------------------------------------------------------- printing
  561. def _print(lines: list[str]) -> None:
  562. for line in lines:
  563. print(line)
  564. def _signalled_gateways(plan: MigrationPlan) -> list[str]:
  565. """Every RUNNING gateway process ``apply_migration`` will signal, in apply order.
  566. Derived from the steps the apply actually takes, not from "secondaries that happen to expose a
  567. PID". Two classes were silently missing and both killed live processes: a secondary with an
  568. installed unit and no readable PID is stopped through ``_service_op`` (which SIGTERMs whatever
  569. the supervisor is running), and the DEFAULT's own live gateway is replaced by
  570. :func:`_restart_default`. A dry run that promises nothing will be stopped and then stops the
  571. host's only gateway breaks the contract this block exists to keep.
  572. """
  573. def _who(p: ProfileGateway, suffix: str = "") -> str:
  574. where = f"pid {p.pid}" if p.pid else f"supervised by {p.service_label()}"
  575. return f"{p.name} ({where}{suffix})"
  576. signalled = [_who(p) for p in plan.standalone_secondaries if p.pid or p.services]
  577. default = plan.default
  578. if default.services or default.pid:
  579. signalled.append(_who(default, ", restarted onto the new flag"))
  580. return signalled
  581. def format_plan(plan: MigrationPlan, *, dry_run: bool) -> list[str]:
  582. head = "Migration plan (dry run — nothing changed)" if dry_run else "Migration plan"
  583. lines = [head, f" default home: {plan.default_home}", "", " profile gateway pid service"]
  584. for p in plan.profiles:
  585. lines.append(f" {p.name:<12} {p.pid or '-'!s:<13} {p.service_label()}")
  586. if plan.standalone_by_config:
  587. lines.append(f" Standalone by config (gateway.standalone: true), left alone: "
  588. f"{', '.join(plan.standalone_by_config)}")
  589. lines.append(" (temporary compatibility shim; remove the key and re-run once the gaps it "
  590. "covers for you are fixed)")
  591. lines.append("")
  592. if plan.already_multiplexed:
  593. lines.append(" ✓ The default gateway is already multiplexing"
  594. + (f" (serving {', '.join(plan.live_served)})" if plan.live_served else " (flag on)") + ".")
  595. return lines
  596. if plan.interrupted:
  597. lines.append(f" ↻ An earlier migration was interrupted before the default gateway came up "
  598. f"(flag on, no live multiplexer; manifest {plan.default_home / MANIFEST_NAME}); this run resumes it.")
  599. elif plan.multiplex_flag_on and plan.standalone_secondaries:
  600. lines.append(" ↻ Half-migrated host: the flag is on, but the profile(s) below still own a "
  601. "gateway. This run converges them.")
  602. steps = []
  603. signalled = _signalled_gateways(plan)
  604. for p in plan.standalone_secondaries:
  605. what = " + ".join(x for x in (f"stop pid {p.pid}" if p.pid else "", " + ".join(f"{_remove_verb(s)} {_service_label(s)}" for s in p.services)) if x)
  606. steps.append(f" - {p.name}: {what}")
  607. if len(plan.profiles) < 2: # the notice already says "only one profile exists"
  608. return lines + _plan_tail(plan)
  609. if not steps:
  610. lines.append(" No secondary profile runs its own gateway; the only step is turning the flag on:")
  611. else:
  612. lines += [" Steps:", *steps]
  613. lines.append(f" - default: set gateway.multiplex_profiles: true in {plan.default_home / 'config.yaml'}")
  614. # A resume installs through the units the MANIFEST recorded (the live ones are already gone),
  615. # which is the mechanism ``apply_migration`` picks — printing this plan's guess contradicted it.
  616. target = _resume_target(plan)[0] if plan.manifest is not None else plan.target_service_kind()
  617. lines.append(f" - default: {'restart' if plan.default.has_gateway else 'start'} the gateway"
  618. + (f" via {target[0]}" if target else " (detached)") + f", verify it serves {len(plan.expected_served_names)} profiles")
  619. lines.append(f" - record the previous state in {plan.default_home / MANIFEST_NAME} "
  620. f"(used to undo a FAILED apply, and to resume this command after a crash)")
  621. if signalled:
  622. # Never stop a running gateway without saying so first, and say it in the imperative
  623. # tense the operator can still act on: this block prints BEFORE anything is signalled.
  624. lines += ["",
  625. " ⚠ This SIGTERMs running gateway process(es): " + ", ".join(signalled) + ".",
  626. " They drain in-flight turns and exit; their profiles are served by the host "
  627. "gateway afterwards."]
  628. return lines + _plan_tail(plan)
  629. def _plan_tail(plan: MigrationPlan) -> list[str]:
  630. lines: list[str] = []
  631. if plan.blockers:
  632. if plan.manifest is None:
  633. lines += ["", " ✗ Blockers (fix these first, nothing will be changed):"]
  634. else:
  635. # This host is already mid-migration with units removed, so a blocker cannot be a
  636. # refusal any more: refusing would leave it with nobody serving and no way forward.
  637. lines += ["",
  638. f" ⚠ Blockers found, but a migration is already in progress on this host "
  639. f"(manifest {_manifest_path(plan.default_home)}).",
  640. " This run RESUMES it and converges anyway — refusing would strand the "
  641. "profiles whose",
  642. " gateways an earlier attempt already removed. Fix these afterwards:"]
  643. lines += [f" • {b}" for b in plan.blockers]
  644. if plan.notices:
  645. lines += ["", " Notices:"]
  646. lines += [f" • {n}" for n in plan.notices]
  647. return lines
  648. def _manifest_secondaries(manifest: dict) -> Optional[list[dict]]:
  649. """The manifest's secondary records, or None when the (hand-edited) manifest is malformed."""
  650. recs = manifest.get("secondaries", [])
  651. if not isinstance(recs, list) or not all(isinstance(r, dict) and r.get("profile") and r.get("home") for r in recs):
  652. return None
  653. return recs
  654. def _service_from_dict(service) -> Optional[tuple[str, bool]]:
  655. if isinstance(service, dict) and service.get("kind"):
  656. return str(service["kind"]), bool(service.get("system"))
  657. return None
  658. def _recorded_services(rec: dict) -> list[tuple[str, bool]]:
  659. """Every service a manifest record names: ``services`` (all installed units) or, in a manifest
  660. written before that key existed, the single ``service``."""
  661. recorded = rec.get("services")
  662. if isinstance(recorded, list):
  663. return [s for s in (_service_from_dict(r) for r in recorded) if s is not None]
  664. service = _service_from_dict(rec.get("service"))
  665. return [service] if service is not None else []
  666. def _recorded_run_as_user(rec: dict) -> Optional[str]:
  667. user = rec.get("run_as_user")
  668. return user if isinstance(user, str) and user else None
  669. def _target_from_manifest(manifest: dict) -> tuple[Optional[tuple[str, bool]], Optional[str]]:
  670. """Service manager + ``User=`` the resumed default install should use, read from the recorded
  671. footprint (the units themselves are gone by the time an interrupted apply is re-run)."""
  672. recs = [r for r in (manifest.get("default"), *(_manifest_secondaries(manifest) or [])) if isinstance(r, dict)]
  673. target = next((s for r in recs for s in _recorded_services(r)), None)
  674. user = next((u for r in recs if (u := _recorded_run_as_user(r)) is not None), None)
  675. return target, user
  676. _NOTHING_RECORDED = "no gateway was recorded; nothing to restore"
  677. def format_update_warning(plan: MigrationPlan, auto_blockers: list[str]) -> list[str]:
  678. return [
  679. "⚠ Your profiles each run their own gateway. A single multiplexed gateway is the recommended",
  680. " setup, but this install cannot be migrated automatically yet:",
  681. *[f" • {b}" for b in (*plan.blockers, *auto_blockers)],
  682. f" After fixing the above, run: {MIGRATE_COMMAND}",
  683. " (`hermes update` will migrate automatically once nothing blocks it.)",
  684. ]
  685. # --------------------------------------------------------------------------- apply / rollback
  686. def _manifest_path(default_home: Path) -> Path:
  687. return default_home / MANIFEST_NAME
  688. def _read_manifest(default_home: Path) -> Optional[dict]:
  689. path = _manifest_path(default_home)
  690. if not path.exists():
  691. return None
  692. try:
  693. data = json.loads(path.read_text(encoding="utf-8-sig"))
  694. except (OSError, ValueError):
  695. return None
  696. return data if isinstance(data, dict) else None
  697. def _manifest_not_yet_served(manifest: Optional[dict], live_served: Optional[list[str]]) -> bool:
  698. """The postcondition ``apply_migration`` waits for, re-derived from live state: a LIVE default that
  699. recorded serving every unparked profile the manifest migrated. Anything less — no live gateway, an
  700. installed-but-dead unit (``systemd_install`` writes the unit before the start that can still fail),
  701. a standalone default never restarted — is a half-applied migration, not "already multiplexed".
  702. Profiles created after the migration are not in the manifest, so they cannot flag it as interrupted."""
  703. if manifest is None:
  704. return False
  705. from hermes_cli.profiles import profile_is_parked
  706. recs = [r for r in (manifest.get("default"), *(_manifest_secondaries(manifest) or [])) if isinstance(r, dict)]
  707. migrated = {str(r.get("profile") or "default") for r in recs
  708. if not r.get("home") or not profile_is_parked(Path(r["home"]))} | {"default"}
  709. return not migrated <= set(live_served or [])
  710. def _write_manifest(default_home: Path, data: dict) -> None:
  711. from utils import atomic_json_write
  712. atomic_json_write(_manifest_path(default_home), data)
  713. def _reconcile_standalone_runtime(default_home: Path, secondary_names: set[str]) -> None:
  714. """Remove only status owned by multiplexing, without re-stamping gateway identity."""
  715. from gateway.status import read_runtime_status
  716. from utils import atomic_json_write
  717. path = default_home / "gateway_state.json"
  718. runtime = read_runtime_status(path)
  719. if runtime is None:
  720. return
  721. runtime["served_profiles"] = []
  722. platforms = runtime.get("platforms")
  723. if isinstance(platforms, dict):
  724. prefixes = tuple(f"{name}:" for name in secondary_names if name)
  725. runtime["platforms"] = {
  726. key: value for key, value in platforms.items()
  727. if not (isinstance(key, str) and key.startswith(prefixes))
  728. }
  729. atomic_json_write(path, runtime, indent=None, separators=(",", ":"))
  730. def _wait_for_served(default_home: Path, expected: set[str], timeout: float) -> Optional[list[str]]:
  731. """Poll the default's ``gateway_state.json`` until ``served_profiles`` covers ``expected``."""
  732. from hermes_cli.gateway_multiplex_served import recorded_served_profiles
  733. deadline = time.monotonic() + timeout
  734. served: Optional[list[str]] = None
  735. while time.monotonic() < deadline:
  736. with _home_env(default_home):
  737. served = recorded_served_profiles(default_home)
  738. if served is not None and expected <= set(served):
  739. return served
  740. time.sleep(0.5)
  741. return served
  742. def _restart_default(
  743. plan_default: ProfileGateway,
  744. target: Optional[tuple[str, bool]],
  745. default_home: Path,
  746. *,
  747. run_as_user: Optional[str] = None,
  748. ) -> str:
  749. """Bring the default gateway up on the new flag value; returns a one-line description.
  750. Boot enablement is NOT this function's job: an existing survivor unit is enabled by
  751. :func:`_enable_default_at_boot` before the first removal, and the install branch enables
  752. through ``systemd_install``'s own enable-on-startup.
  753. """
  754. if plan_default.service is not None:
  755. kind, system = plan_default.service
  756. _service_op(kind, system, "restart", default_home)
  757. return f"restarted the default gateway via {kind}"
  758. if target is not None:
  759. kind, system = target
  760. _service_op(kind, system, "install", default_home, run_as_user=run_as_user)
  761. _service_op(kind, system, "start", default_home)
  762. return f"installed and started the default gateway via {kind}"
  763. verb = "restarted" if plan_default.pid is not None else "started"
  764. if plan_default.pid is not None:
  765. _stop_gateway_process(default_home)
  766. if not _spawn_detached_gateway(default_home):
  767. raise RuntimeError("could not spawn the default gateway (detached)")
  768. return f"{verb} the default gateway (detached; no service manager was in use)"
  769. def _enable_default_at_boot(plan: MigrationPlan) -> Optional[str]:
  770. """Make the survivor boot-startable BEFORE anything is removed. Every secondary unit the fold
  771. uninstalls takes its boot enablement with it, and the pre-existing default unit may have been
  772. disabled all along; ``restart`` on a disabled unit succeeds, so the old order (remove, then
  773. restart) turned N boot-startable gateways into 0 and reported success (#124922). ``enable``
  774. never starts the unit, so doing it first cannot double-bind a platform while secondaries run,
  775. and an apply interrupted anywhere after it still leaves a host that comes back after a reboot.
  776. Returns the failure line, or None: a fresh apply refuses on it (nothing has been removed yet),
  777. a resume reports it and converges anyway (rollback could do no better).
  778. """
  779. if plan.default.service is None:
  780. return None
  781. kind, system = plan.default.service
  782. try:
  783. _service_op(kind, system, "enable", plan.default_home)
  784. except RuntimeError as exc:
  785. return f"default: {exc}"
  786. return None
  787. def _remove_secondary_gateways(plan: MigrationPlan) -> None:
  788. for p in plan.standalone_secondaries:
  789. for kind, system in p.services:
  790. _service_op(kind, system, "stop", p.home)
  791. _service_op(kind, system, "uninstall", p.home)
  792. done = "parked (down file)" if kind == "s6" else "removed"
  793. print(f" ✓ {p.name}: stopped and {done} its {_service_label((kind, system))}")
  794. if p.pid is not None:
  795. _stop_gateway_process(p.home)
  796. print(f" ✓ {p.name}: stopped standalone gateway (pid {p.pid})")
  797. def _preflight_apply(plan: MigrationPlan, target: Optional[tuple[str, bool]], run_as_user: Optional[str]) -> Optional[str]:
  798. """A failure of the destructive phase that is knowable from the plan alone, refused BEFORE any
  799. working per-profile gateway is stopped: rollback is the fallback for surprises, not the plan.
  800. Mirrors the checks ``systemd_install``/``_service_call`` make on a system unit (root, resolvable
  801. ``User=``) and the config write's read-guard."""
  802. from hermes_cli import gateway as gw
  803. from hermes_cli.config import require_readable_config_before_write
  804. try:
  805. require_readable_config_before_write(plan.default_home / "config.yaml")
  806. except Exception as exc:
  807. return f"default: config.yaml cannot be updated ({exc})"
  808. touches_system_unit = target == ("systemd", True) or any(p.has_system_unit for p in plan.standalone_secondaries)
  809. if touches_system_unit:
  810. try:
  811. gw._require_root_for_system_service("migration")
  812. except Exception as exc:
  813. return str(exc)
  814. if plan.default.service is None and target == ("systemd", True):
  815. if run_as_user is None:
  816. try:
  817. gw._system_service_identity() # the #110850 refusal (implicit root), before anything is removed
  818. except ValueError as exc:
  819. return f"default: {exc}"
  820. else:
  821. import pwd
  822. try:
  823. pwd.getpwnam(run_as_user)
  824. except KeyError:
  825. return f"default: the recorded service user '{run_as_user}' does not exist on this host"
  826. return None
  827. def _resume_target(plan: MigrationPlan) -> tuple[Optional[tuple[str, bool]], Optional[str]]:
  828. """Service manager + ``User=`` an apply will use: the manifest's record wins on a resume (the
  829. live units it describes are already gone), else what the plan can still see."""
  830. target, run_as_user = plan.target_service_kind(), plan.target_run_as_user()
  831. if plan.manifest is None:
  832. return target, run_as_user
  833. m_target, m_user = _target_from_manifest(plan.manifest)
  834. return m_target or target, m_user or run_as_user
  835. def _resume_findings(plan: MigrationPlan, target, run_as_user) -> list[str]:
  836. """Safety checks re-run on a RESUME and reported as findings instead of refusals.
  837. Both halves matter. Skipping them (the resume branch used to return before ``_preflight_apply``
  838. ran at all) let a duplicate bot credential, a ``/p/`` ingress gap or an unresolvable service
  839. user reach the multiplexer unannounced. Enforcing them as blockers is worse: a resume runs on
  840. a host whose secondary units are ALREADY removed, so refusing does not prevent damage — it
  841. makes the damage permanent, which is exactly how a stranded host lost its only documented
  842. recovery. So: run every check, name every finding, converge anyway.
  843. """
  844. findings = [*plan.blockers]
  845. apply_blocker = _preflight_apply(plan, target, run_as_user)
  846. if apply_blocker is not None:
  847. findings.append(apply_blocker)
  848. if not findings:
  849. return []
  850. return [" ⚠ Preflight findings — resuming anyway (this host is mid-migration; refusing would",
  851. " leave its profiles with no gateway at all). Fix these once it is converged:",
  852. *[f" • {f}" for f in findings]]
  853. def apply_migration(plan: MigrationPlan, *, served_wait: float = _SERVED_WAIT_SECONDS) -> bool:
  854. """Flip the flag, stop/uninstall every secondary gateway, bring up the multiplexer, verify.
  855. Returns True when the multiplexer verifiably serves every profile.
  856. Every step after the manifest write is fallible (a config write, a secondary's stop or its
  857. unit's daemon-reload, a system unit that needs ``--run-as-user``, an unreachable user bus) and
  858. runs inside ONE compensating boundary: on failure the manifest written before the first
  859. destructive step brings ONE gateway back (:func:`rollback_migration`), so the host never ends
  860. with nobody serving. The flag goes on first so an apply killed anywhere after it is resumable
  861. from the manifest (flag on + manifest + no live multiplexer = interrupted).
  862. **Resume beats preflight.** The manifest is read BEFORE the blocker gate: a manifest means an
  863. earlier apply already removed units, so this run is a compensation, not an initiation, and a
  864. gate that refuses it traps the host forever (no ``--standalone``, no rollback command). The
  865. checks still run — as findings, see :func:`_resume_findings`. With no manifest nothing has been
  866. destroyed yet and both gates refuse exactly as before.
  867. """
  868. if plan.already_multiplexed:
  869. print("✓ Already multiplexed — nothing to do.")
  870. return True
  871. target, run_as_user = _resume_target(plan)
  872. manifest = plan.manifest
  873. if manifest is not None:
  874. # A manifest on disk means an earlier apply got past its first destructive step: it either
  875. # died mid-way (``interrupted``) or its compensator did not finish. Either way the manifest
  876. # is the ONLY record of the units that existed, so resume from it and never overwrite it.
  877. # Re-running this command IS the recovery.
  878. print(f" ↻ resuming the migration recorded in {_manifest_path(plan.default_home)}")
  879. _print(_resume_findings(plan, target, run_as_user))
  880. enable_failure = _enable_default_at_boot(plan)
  881. if enable_failure is not None:
  882. print(f" ⚠ {enable_failure} — resuming anyway")
  883. else:
  884. if plan.blocked:
  885. _print(["✗ Migration refused:", *[f" • {b}" for b in plan.blockers]])
  886. return False
  887. blocker = _preflight_apply(plan, target, run_as_user) or _enable_default_at_boot(plan)
  888. if blocker is not None:
  889. _print(["✗ Migration refused before changing anything:", f" • {blocker}"])
  890. return False
  891. manifest = {
  892. "version": 1, "migrated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
  893. "flag_was": plan.multiplex_flag_on,
  894. "default": plan.default.to_dict(),
  895. "secondaries": [p.to_dict() for p in plan.standalone_secondaries],
  896. }
  897. # Recovery metadata must exist before the first destructive operation; the manifest never
  898. # changes afterwards, so this is the only write it needs.
  899. _write_manifest(plan.default_home, manifest)
  900. try:
  901. _write_multiplex_flag(plan.default_home, True)
  902. print(f" ✓ default: gateway.multiplex_profiles: true ({plan.default_home / 'config.yaml'})")
  903. _remove_secondary_gateways(plan) # on resume: whatever an apply killed mid-removal left installed
  904. print(f" ✓ {_restart_default(plan.default, target, plan.default_home, run_as_user=run_as_user)}")
  905. except Exception as exc:
  906. _print([f" ✗ migration failed ({exc})",
  907. " ↩ Restoring one host gateway so this host is not left without one..."])
  908. rolled_back = rollback_migration(plan.default_home)
  909. if not rolled_back:
  910. print(f" Re-run {MIGRATE_COMMAND} to resume from the manifest.")
  911. return False
  912. expected = plan.expected_served_names
  913. served = _wait_for_served(plan.default_home, expected, served_wait)
  914. if served is not None and expected <= set(served):
  915. # Manifest present == migration UNFINISHED. That is the whole resume/half-migrated signal
  916. # (`MigrationPlan.interrupted`), so a CONFIRMED convergence must clear it -- otherwise the
  917. # host reports itself interrupted forever and never says "already multiplexing".
  918. _manifest_path(plan.default_home).unlink(missing_ok=True)
  919. _print(["", f"✓ Migrated: the default gateway now serves {len(served)} profiles: {', '.join(served)}",
  920. *[f" • {n}" for n in plan.notices]])
  921. return True
  922. missing = sorted(expected - set(served or []))
  923. _print(["", f"⚠ Migration applied, but the default gateway has not confirmed serving: {', '.join(missing)}",
  924. " Check `hermes gateway status` and the gateway log; the flag and manifest are in place.",
  925. f" Re-run {MIGRATE_COMMAND} once it is healthy — it resumes from the manifest."])
  926. return False
  927. _COMPENSATOR_WAIT_SECONDS = 30.0
  928. def _wait_for_live_gateway(home: Path, timeout: float) -> Optional[int]:
  929. """Bounded wait for a gateway that is actually UP for ``home``; None on timeout.
  930. A service-manager ``start`` that returns is not proof: systemd returns after its own start
  931. timeout on a unit that keeps failing, and taking that for success is how the compensator
  932. printed "✓ Restored" over a unit respawning every 5 s at ``ExecMainStatus=75``.
  933. """
  934. return poll_until(lambda: _live_gateway_pid(home), timeout, 0.5)
  935. def rollback_migration(default_home: Optional[Path] = None) -> bool:
  936. """COMPENSATOR for a failed apply: put ONE gateway back on this host.
  937. Not a user-facing rollback and deliberately NOT a fleet restore. It used to reinstall and
  938. start every recorded per-profile gateway, which the host gateway lock now forbids by
  939. construction: this function clears the multiplex-owned served record first, so
  940. ``gateway.host_attach`` tells each secondary to "start normally" and the flock loser exits 75
  941. into a supervisor respawn loop (observed live: a secondary won the race while the default's
  942. unit sat at ``NRestarts=38, ExecMainStatus=75``, respawning every 5 s — ``StartLimitIntervalSec=0``
  943. defeats the limiter). Under multiplex-only a restored fleet is not a state we want to be able
  944. to reach, so the compensation is the one gateway the topology allows: the DEFAULT's, on the
  945. recorded flag. The secondaries whose units this attempt already removed are named in the
  946. output and are served by that gateway as soon as it multiplexes (``flag_was`` is either unset
  947. or the retired ``false``; both resolve to multiplex at boot once nothing blocks it).
  948. Returns True only when a gateway is VERIFIABLY live again; the manifest is kept otherwise so
  949. the next ``hermes gateway migrate --multiplex`` resumes.
  950. """
  951. default_home = default_home or _default_home()
  952. manifest = _read_manifest(default_home)
  953. if manifest is None:
  954. print(f"✗ No migration manifest at {_manifest_path(default_home)}; nothing to restore.")
  955. return False
  956. incomplete = (f"⚠ Compensation incomplete; manifest kept at {_manifest_path(default_home)}.\n"
  957. f" Re-run {MIGRATE_COMMAND} — it resumes from the manifest.")
  958. secondaries = _manifest_secondaries(manifest)
  959. if secondaries is None:
  960. # Refuse before touching anything: a hand-edited manifest is not a rollback authority.
  961. print(f"✗ Malformed secondary records in {_manifest_path(default_home)}; fix or delete the manifest.")
  962. return False
  963. try:
  964. _write_multiplex_flag(default_home, bool(manifest.get("flag_was", False)))
  965. print(" ✓ default: gateway.multiplex_profiles restored")
  966. except Exception as exc:
  967. print(f" ✗ default: could not restore gateway.multiplex_profiles ({exc})")
  968. print(incomplete)
  969. return False
  970. default_rec = manifest.get("default")
  971. default_gw = ProfileGateway(
  972. "default", default_home, pid=_own_gateway_pid(default_home, _host_gateway_owner()),
  973. services=(_recorded_services(default_rec) if isinstance(default_rec, dict) else []) or _installed_services(default_home),
  974. )
  975. names = [str(rec["profile"]) for rec in secondaries]
  976. # The live multiplexer's record still claims every secondary, and every lifecycle verb reads it;
  977. # clear it before the default comes back up on the restored flag.
  978. try:
  979. _reconcile_standalone_runtime(default_home, set(names))
  980. print(" ✓ default: cleared multiplex-owned runtime status")
  981. except Exception as exc:
  982. print(f" ✗ default: could not clear multiplex-owned runtime status ({exc})")
  983. print(incomplete)
  984. return False
  985. if not default_gw.has_gateway and not any(rec.get("pid") or _recorded_services(rec) for rec in secondaries):
  986. _manifest_path(default_home).unlink(missing_ok=True)
  987. print(f"✓ Nothing was running before this attempt ({_NOTHING_RECORDED}); the flag is restored.")
  988. return True
  989. target, run_as_user = _target_from_manifest(manifest)
  990. try:
  991. print(f" ✓ {_restart_default(default_gw, target, default_home, run_as_user=run_as_user)}")
  992. except Exception as exc:
  993. # The recorded service manager is exactly what the failed apply could not drive (a refused
  994. # system-unit install, a read-only unit dir). Falling back to a detached gateway still
  995. # honours one-gateway-per-host, and a host with a running gateway beats a correct unit.
  996. print(f" ✗ default: could not bring the host gateway back up via the recorded service "
  997. f"manager ({exc}); falling back to a detached gateway")
  998. try:
  999. spawned = _spawn_detached_gateway(default_home)
  1000. except Exception as spawn_exc:
  1001. print(f" ✗ default: the detached fallback also failed ({spawn_exc})")
  1002. spawned = False
  1003. if not spawned:
  1004. print(incomplete)
  1005. return False
  1006. print(" ✓ default: started the host gateway detached (no service manager)")
  1007. live = _wait_for_live_gateway(default_home, _COMPENSATOR_WAIT_SECONDS)
  1008. if live is None:
  1009. print(f" ✗ default: no gateway confirmed serving this host within {_COMPENSATOR_WAIT_SECONDS:.0f}s "
  1010. f"(check `hermes gateway status` and the gateway log)")
  1011. print(incomplete)
  1012. return False
  1013. _manifest_path(default_home).unlink(missing_ok=True)
  1014. print(f"✓ Compensated: one host gateway is running again (pid {live}) on the recorded flag.")
  1015. if names:
  1016. print(f" Per-profile gateways this attempt had already removed: {', '.join(names)}. They are "
  1017. f"NOT reinstalled — one gateway per host is the only supported topology — and the host "
  1018. f"gateway serves them as soon as it multiplexes.\n"
  1019. f" Run {MIGRATE_COMMAND} to finish converging.")
  1020. return True
  1021. # --------------------------------------------------------------------------- CLI + update hook
  1022. def _host_supports_migration() -> Optional[str]:
  1023. """Reason the host cannot be converged by this command, else None.
  1024. Windows IS handled (per-profile Scheduled Tasks and the Startup-folder fallback are removed
  1025. like any other unit). s6 IS handled too, in-process: a named profile's slot that is UP is
  1026. parked (``s6-svc -d`` + ``down`` file) and its autostart intent folded into the root slot the
  1027. same way the container's boot does it (``hermes_cli.gateway_multiplex_s6``). What cannot be
  1028. done from here is register a slot the boot never created — that is the one refusal left.
  1029. """
  1030. from hermes_cli import gateway as gw
  1031. if not gw._running_under_s6():
  1032. return None
  1033. from hermes_cli.gateway_multiplex_s6 import named_slot_name
  1034. from hermes_cli.service_manager import S6ServiceManager
  1035. scandir = S6ServiceManager().scandir
  1036. if not (scandir / named_slot_name("default")).is_dir():
  1037. return (f"s6-supervised container without a root gateway slot ({scandir / named_slot_name('default')}); "
  1038. "the container's boot registers it — restart the container.")
  1039. return None
  1040. def cmd_migrate(args) -> None:
  1041. """``hermes gateway migrate [--multiplex] [--dry-run] [--yes]``."""
  1042. reason = _host_supports_migration()
  1043. if reason:
  1044. print(f"✗ {reason}")
  1045. sys.exit(1)
  1046. plan = build_migration_plan()
  1047. dry_run = getattr(args, "dry_run", False)
  1048. _print(format_plan(plan, dry_run=dry_run))
  1049. if dry_run:
  1050. return
  1051. if plan.already_multiplexed:
  1052. return
  1053. # A manifest means this host is already mid-migration with units removed: the blocked/too-few-
  1054. # profiles gates describe a migration that has not started yet, and applying them here is what
  1055. # left a stranded host refusing its own documented recovery forever.
  1056. if plan.manifest is None and (plan.blocked or len(plan.profiles) < 2):
  1057. sys.exit(1 if plan.blocked else 0)
  1058. # Zero standalone secondaries is still a migration when the user asks for it explicitly: the flag
  1059. # goes on and the default gateway restarts (the update hook keeps treating that case as a no-op).
  1060. if not getattr(args, "yes", False) and sys.stdin.isatty():
  1061. from hermes_cli.setup import prompt_yes_no
  1062. if not prompt_yes_no("Apply this migration now?", True):
  1063. print("Aborted; nothing changed.")
  1064. return
  1065. print()
  1066. sys.exit(0 if apply_migration(plan) else 1)
  1067. def maybe_auto_migrate_after_update() -> None:
  1068. """``hermes update`` hook: with >= 2 profiles, per-profile gateways present and multiplex off,
  1069. migrate automatically when unblocked (deterministic, never prompts) or print the blocker block.
  1070. ``gateway.auto_multiplex_migration: false`` on the default profile opts out; a secondary behind a
  1071. service-domain / UNIX-user / HERMES_HOME boundary blocks this path only (the explicit command decides)."""
  1072. from hermes_cli.gateway_migrate_guards import auto_migration_blockers, auto_migration_opted_out
  1073. if _host_supports_migration() is not None or auto_migration_opted_out(_default_home()):
  1074. return
  1075. plan = build_migration_plan()
  1076. if plan.already_multiplexed or len(plan.profiles) < 2:
  1077. return
  1078. # A stranded host (manifest on disk, units already removed, nobody serving) has no standalone
  1079. # secondaries left to find, so the "nothing to fold" exit skipped the one host that needs this
  1080. # most. A manifest makes the update hook a RESUME, and a resume is not gated on blockers.
  1081. resuming = plan.manifest is not None
  1082. if not plan.standalone_secondaries and not resuming:
  1083. return
  1084. from hermes_cli.gateway_migrate_guards import masked_default_unit
  1085. masked = None if resuming else masked_default_unit(plan.default_home)
  1086. if masked is not None:
  1087. print(f"\n ℹ Per-profile gateways kept: the default gateway unit is masked ({masked}).\n"
  1088. f" Unmask it and run {MIGRATE_COMMAND} to fold them into one gateway.")
  1089. return
  1090. print()
  1091. auto_blockers = auto_migration_blockers(plan)
  1092. if not resuming and (plan.blocked or auto_blockers):
  1093. _print(format_update_warning(plan, auto_blockers))
  1094. return
  1095. print("→ Migrating per-profile gateways onto one multiplexed default gateway...")
  1096. _print(format_plan(plan, dry_run=False))
  1097. apply_migration(plan)