host_attach.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425
  1. """Is there ONE live host gateway, and does it already serve this profile?
  2. Multiplex-only (Teknium ruling): exactly one MULTIPLEXING ``hermes gateway run`` per host, serving
  3. every profile; standalone per-profile gateways coexist until that migration is forced (#109417).
  4. The lifecycle verbs therefore answer a different question than they used to — not "does THIS home
  5. hold a ``gateway.pid``?" but "is the host process live, and is this profile in its served set?" —
  6. and when it is not, they ask that process to serve the profile instead of starting a second one.
  7. Five outcomes, in order:
  8. * ``ATTACH`` — a live host gateway already serves this profile. Nothing to start; exit 0.
  9. * ``RESCAN``→ATTACH — it does not serve it yet: ask it to reconcile ``profiles/`` now (control
  10. socket ``rescan-profiles``) and attach once the answer includes us.
  11. * ``REPLACE_HOST`` — ``--replace`` targets the host process when it serves this profile, whichever
  12. home launched it. An owner that has not published its served set yet is targeted too, but the
  13. ownership guard (``run._replace_target_belongs_to_other_profile``) can then only prove it from
  14. THIS home's pid record, so from another home the replace is refused (exit 1).
  15. * ``REFUSE`` — a live MULTIPLEXING gateway exists and cannot be made to serve this profile.
  16. Never start a second one silently.
  17. * ``START`` — no live owner, or the owner answers ``multiplex: False``: it is another
  18. profile's standalone gateway (the documented one-process-per-profile topology), not a
  19. multiplexer that excluded us, so this profile runs its own gateway beside it as it always did.
  20. **The attach channel is the OWNER's control socket, never ours.** Ordering matters: the owner
  21. publishes its rendezvous record when it claims its PID file and binds its control socket a moment
  22. later (``gateway/run.py``: claim → socket), so for a short window the record exists and the channel
  23. does not. A reader that took "no socket" for "no owner" would start exactly the second gateway this
  24. module prevents — but a reader that took the RECORD's word for the served set is worse: the
  25. claim-time record is published before the process knows what it will serve, so a supervised unit
  26. for a profile nobody serves would stand down forever. Hence the split:
  27. * the record proves an OWNER exists (PID + createTime), and that alone never yields ATTACH;
  28. * the served set comes ONLY from a live ``identify`` answer, waited for a bounded
  29. :data:`ATTACH_CHANNEL_WAIT_S`;
  30. * owner present + served set unknown is a TRANSIENT verdict — do not start, do not park.
  31. Nothing here depends on the *calling* process having started anything.
  32. """
  33. from __future__ import annotations
  34. import logging
  35. import os
  36. import time
  37. from dataclasses import dataclass
  38. from pathlib import Path
  39. from typing import Optional
  40. from agent.deadline import poll_until
  41. logger = logging.getLogger(__name__)
  42. #: How long a caller waits for the owner's control socket after seeing its record (see module doc).
  43. ATTACH_CHANNEL_WAIT_S = 5.0
  44. _CHANNEL_POLL_S = 0.25
  45. START = "start"
  46. ATTACH = "attach"
  47. REFUSE = "refuse"
  48. REPLACE_HOST = "replace-host"
  49. def _normalize(name: str) -> str:
  50. try:
  51. from hermes_cli.profiles import normalize_profile_name
  52. return normalize_profile_name(name or "default")
  53. except Exception:
  54. return (name or "default").strip().lower()
  55. def profile_name_for_home(home: Path | str) -> str:
  56. """Profile a home belongs to; the root/default home is ``'default'`` (not ``None``)."""
  57. from gateway.status import _profile_name_for_home
  58. return _profile_name_for_home(Path(home)) or "default"
  59. @dataclass(frozen=True)
  60. class HostGateway:
  61. """The one live host gateway: who it is, where it was launched from, what it serves."""
  62. pid: int
  63. home: Path
  64. profiles: tuple[str, ...]
  65. #: False when the owner has not answered ``identify`` yet: an owner exists, but which profiles
  66. #: it serves is UNKNOWN. Never conflate that with "serves nothing" — see the module doc.
  67. served_known: bool = True
  68. #: True once the owner has said ``multiplex: False``: a per-profile gateway that cannot be asked
  69. #: to serve anyone else — see ``START`` in the module doc.
  70. standalone: bool = False
  71. def serves(self, profile: str) -> bool:
  72. if not self.served_known:
  73. return False
  74. wanted = _normalize(profile)
  75. return any(_normalize(p) == wanted for p in self.profiles)
  76. @property
  77. def profile_label(self) -> str:
  78. return profile_name_for_home(self.home)
  79. def describe(self) -> str:
  80. if not self.served_known:
  81. served = "not published yet (its control socket has not answered)"
  82. else:
  83. served = ", ".join(self.profiles) if self.profiles else "nothing"
  84. return f"PID {self.pid} (launched by profile '{self.profile_label}'; serves: {served})"
  85. def record_home(record) -> Path:
  86. """Home the owner was launched from. Records written before the field existed fall back to the
  87. default root — the home every pre-record multiplexer ran under."""
  88. from hermes_constants import get_default_hermes_root
  89. return Path(record.home) if getattr(record, "home", "") else Path(get_default_hermes_root())
  90. def launched_by_other_tenant(owner_home: Path | str, our_home: Path | str) -> bool:
  91. """Was the owner launched from ANOTHER Hermes root (a second tenant on this host)?
  92. The host record and lock are per OS user, and two tenants (separate ``HERMES_HOME`` roots)
  93. each expose a profile named ``default``: tenant A's multiplexer "serving default" is a name
  94. collision for tenant B, never coverage, so it is not this tenant's host gateway (#121352).
  95. A legacy record with no ``home`` proves nothing and counts as ours.
  96. """
  97. if not str(owner_home or ""):
  98. return False
  99. from gateway.status import _same_hermes_home
  100. from hermes_constants import get_default_hermes_root
  101. return not _same_hermes_home(get_default_hermes_root(home=owner_home),
  102. get_default_hermes_root(home=our_home))
  103. def _identify(home: Path) -> Optional[dict]:
  104. try:
  105. from gateway.control_socket import identify_gateway
  106. return identify_gateway(home)
  107. except Exception:
  108. logger.debug("host gateway identify failed for %s", home, exc_info=True)
  109. return None
  110. def _served_from_identity(identity: dict) -> tuple[str, ...]:
  111. """Served set from a live ``identify``. A STANDALONE gateway publishes no ``served_profiles``;
  112. it serves its own profile and nothing else, which is not the same as "unknown".
  113. The unified runtime publishes each served profile as ``{"profile_id", "home"}`` (what
  114. ``hermes_cli.gateway_runtime`` matches homes against); older gateways published bare names.
  115. Both spell the same roster, so a dict entry is reduced to its profile name here."""
  116. served = identity.get("served_profiles")
  117. if isinstance(served, list) and served:
  118. names = []
  119. for entry in served:
  120. if isinstance(entry, dict):
  121. home = entry.get("home") or entry.get("profile_id")
  122. names.append(profile_name_for_home(str(home)) if home else "default")
  123. else:
  124. names.append(str(entry))
  125. return tuple(names)
  126. return (str(identity.get("profile") or "default"),)
  127. def _identity_matches(identity, record, home: Path) -> bool:
  128. """Is this ``identify`` answer really the record's owner?
  129. PID alone is not enough: a record naming an arbitrary home makes us dial whatever listens
  130. there, so the answer must also agree about the home it was launched from.
  131. """
  132. if not isinstance(identity, dict) or identity.get("pid") != record.pid:
  133. return False
  134. reported = identity.get("hermes_home")
  135. if not reported:
  136. return True # older gateway: PID + a socket keyed by this home is all it can prove
  137. try:
  138. from gateway.status import _same_hermes_home
  139. return bool(_same_hermes_home(Path(str(reported)), home))
  140. except Exception:
  141. return str(reported) == str(home)
  142. #: A CLI invocation asks this question once per profile (``gateway status`` across N profiles,
  143. #: doctor, the lifecycle guards); a gateway PROCESS asks it for the life of the process, so the
  144. #: memo is time-bounded rather than permanent. Writes invalidate it eagerly.
  145. HOST_GATEWAY_CACHE_TTL_S = 2.0
  146. _cached_probe: Optional[tuple[float, Optional[HostGateway]]] = None
  147. def invalidate_host_gateway_cache() -> None:
  148. """Forget the memoized probe (called by ``host_rendezvous`` on every record write)."""
  149. global _cached_probe
  150. _cached_probe = None
  151. def _probe_host_gateway(wait_for_channel: float) -> Optional[HostGateway]:
  152. from gateway import host_rendezvous as hr
  153. record = hr.read_record(hr.ROLE_GATEWAY)
  154. if record is None:
  155. return None
  156. # Liveness BEFORE the dial. A record we cannot prove live must not make us open a socket at an
  157. # address it chose; proving the PID first is also what keeps a stale record from naming a peer.
  158. # A free host lock falsifies a pid that merely looks alive: treating it as a silent-but-live
  159. # gateway is the never-expiring transient refuse (#138343, #133744).
  160. if not hr.record_names_live_owner(record):
  161. return None
  162. home = record_home(record)
  163. def _matching_identity():
  164. identity = _identify(home)
  165. return identity if _identity_matches(identity, record, home) else None
  166. identity = poll_until(_matching_identity, max(0.0, wait_for_channel), _CHANNEL_POLL_S)
  167. if identity is not None:
  168. return HostGateway(record.pid, home, _served_from_identity(identity))
  169. # An owner exists and has not answered: the served set is UNKNOWN, never the record's word.
  170. return HostGateway(record.pid, home, (), served_known=False)
  171. def host_gateway(*, wait_for_channel: float = 0.0) -> Optional[HostGateway]:
  172. """The one live host gateway, or ``None``.
  173. The served set comes from the owner's control socket and nowhere else; a record with no live
  174. answer behind it yields ``served_known=False`` — an owner whose served set nobody knows yet.
  175. """
  176. global _cached_probe
  177. now = time.monotonic()
  178. if wait_for_channel <= 0 and _cached_probe is not None and now - _cached_probe[0] < HOST_GATEWAY_CACHE_TTL_S:
  179. return _cached_probe[1]
  180. result = _probe_host_gateway(wait_for_channel)
  181. _cached_probe = (time.monotonic(), result)
  182. return result
  183. def host_gateway_serving(profile: str, *, wait_for_channel: float = 0.0) -> Optional[HostGateway]:
  184. """The host gateway when it is live AND serves ``profile`` — true for ``default`` too."""
  185. gateway = host_gateway(wait_for_channel=wait_for_channel)
  186. return gateway if gateway is not None and gateway.serves(profile) else None
  187. def request_serve_profile(profile: str, *, timeout: float = 8.0,
  188. owner: Optional[HostGateway] = None) -> Optional[HostGateway]:
  189. """Ask the live host gateway to reconcile ``profiles/`` now; return it once it serves
  190. ``profile``. ``None`` when nobody answered or a multiplexer's roster still excludes the profile.
  191. An owner that answers ``multiplex: False`` comes back flagged ``standalone``: it cannot take the
  192. profile, and it is not a multiplexer that refused — the caller runs beside it, as before."""
  193. gateway = owner if owner is not None else host_gateway(wait_for_channel=ATTACH_CHANNEL_WAIT_S)
  194. if gateway is None or gateway.serves(profile):
  195. return gateway
  196. try:
  197. from gateway.control_socket import rescan_gateway_profiles
  198. answer = rescan_gateway_profiles(gateway.home, timeout=timeout)
  199. except Exception:
  200. logger.debug("host gateway rescan failed", exc_info=True)
  201. return None
  202. if not isinstance(answer, dict):
  203. return None
  204. served = answer.get("served_profiles")
  205. rescanned = HostGateway(
  206. gateway.pid, gateway.home,
  207. tuple(str(p) for p in served) if isinstance(served, list) else (),
  208. standalone=answer.get("multiplex") is False)
  209. return rescanned if rescanned.standalone or rescanned.serves(profile) else None
  210. @dataclass(frozen=True)
  211. class HostAttachDecision:
  212. outcome: str
  213. message: str
  214. owner: Optional[HostGateway] = None
  215. #: True when the verdict is a RUNTIME observation ("someone else serves me right now", "the
  216. #: owner has not answered yet") rather than a config-derived permanent refusal. A supervisor
  217. #: must RETRY a transient verdict; parking the unit on one strands the profile forever.
  218. transient: bool = False
  219. def attach_message(gateway: HostGateway, profile: str) -> str:
  220. return (
  221. f"✓ The host gateway already serves profile '{profile}' — nothing to start.\n"
  222. f" {gateway.describe()}\n"
  223. f" One gateway per host serves every profile; manage it with "
  224. f"`hermes -p {gateway.profile_label} gateway restart`.")
  225. def _unknown_served_message(gateway: HostGateway, profile: str) -> str:
  226. return (
  227. f"⏳ A gateway already owns this host and has not published its served set yet.\n"
  228. f" {gateway.describe()}\n"
  229. f" Whether it will serve profile '{profile}' is unknown, so starting a second gateway\n"
  230. f" now could double-bind this profile's platforms. Nothing was started; this is a\n"
  231. f" transient state and a service supervisor will retry.\n"
  232. f" Take the host over (only from the home that launched it): hermes gateway run --replace\n"
  233. f" Start anyway: hermes gateway run --force")
  234. def _refuse_message(gateway: HostGateway, profile: str) -> str:
  235. from hermes_cli.gateway_migrate import MIGRATE_COMMAND
  236. return (
  237. f"❌ A gateway already owns this host and will not serve profile '{profile}'.\n"
  238. f" {gateway.describe()}\n"
  239. f" Exactly one gateway per host serves every profile, so starting a second one\n"
  240. f" would double-bind this profile's platforms.\n"
  241. f" Fold this profile into it: {MIGRATE_COMMAND}\n"
  242. f" Or start one anyway: hermes gateway run --force\n"
  243. f" (--replace only replaces an owner that serves this profile, so it would not take this one over.)")
  244. def standalone_rescan_message(profile: str) -> str:
  245. return (
  246. f"The host gateway still serves profile '{profile}'; gateway.standalone is not live yet. "
  247. "Wait for the host gateway to rescan (<=30s), or send the rescan-profiles control verb "
  248. "to the host gateway before starting this profile's gateway.")
  249. def _coexisting_gateways(owner: Optional[HostGateway]):
  250. """A standalone lock owner can hide a multiplexer launched beside it.
  251. Use the existing per-home liveness and control channels, not the single host
  252. record, to ask every running profile gateway what it actually serves.
  253. """
  254. from gateway.status import live_gateway_pid_for_home
  255. from hermes_cli.profiles import profiles_to_serve
  256. seen = {os.getpid()}
  257. if owner is not None:
  258. seen.add(owner.pid)
  259. yield owner
  260. for _name, home in profiles_to_serve(True, include_standalone=True, include_parked=True):
  261. pid = live_gateway_pid_for_home(home)
  262. if pid is None or pid in seen:
  263. continue
  264. seen.add(pid)
  265. peer = HostGateway(pid, home, (), served_known=False)
  266. identity = _identify(home)
  267. if isinstance(identity, dict) and _identity_matches(identity, peer, home):
  268. peer = HostGateway(pid, home, _served_from_identity(identity),
  269. standalone=identity.get("multiplex") is False)
  270. yield peer
  271. def standalone_attach_decision(our_home: Path, owner: Optional[HostGateway]) -> Optional[HostAttachDecision]:
  272. """An opt-out permits coexistence only after every live gateway confirms we are unserved.
  273. Shared by the initial attach check and the lock-losing race check.
  274. """
  275. from hermes_cli.profiles import profile_is_standalone
  276. if not profile_is_standalone(our_home):
  277. return None
  278. profile = profile_name_for_home(our_home)
  279. for peer in _coexisting_gateways(owner):
  280. if not peer.served_known:
  281. return HostAttachDecision(REFUSE, _unknown_served_message(peer, profile), peer, transient=True)
  282. if peer.serves(profile):
  283. return HostAttachDecision(REFUSE, standalone_rescan_message(profile), peer, transient=True)
  284. logger.info("Profile '%s' is standalone by config; starting beside the host multiplexer", profile)
  285. return HostAttachDecision(START, "", owner)
  286. def decide(our_home: Path, *, replace: bool = False) -> HostAttachDecision:
  287. """Attach, rescan-then-attach, replace or refuse; configured standalone profiles may coexist.
  288. Never raises: a broken probe degrades to ``START``, i.e. exactly the pre-rendezvous behaviour.
  289. """
  290. profile = profile_name_for_home(our_home)
  291. try:
  292. gateway = host_gateway()
  293. except Exception:
  294. logger.debug("host gateway probe failed; starting as before", exc_info=True)
  295. return HostAttachDecision(START, "")
  296. if gateway is None or gateway.pid == os.getpid() or launched_by_other_tenant(gateway.home, our_home):
  297. return standalone_attach_decision(our_home, None) or HostAttachDecision(START, "")
  298. if replace and (gateway.serves(profile) or not gateway.served_known):
  299. # An owner known not to serve us is another profile's gateway: replacing it is always refused
  300. # (_replace_target_belongs_to_other_profile fails closed) and the gateway exits, so on a
  301. # one-process-per-profile fleet, whose generated units all carry --replace, every unit but
  302. # the lock holder respawn-storms. Such an owner takes the non-replace path below instead.
  303. return HostAttachDecision(REPLACE_HOST, "", gateway)
  304. if gateway.served_known:
  305. standalone = standalone_attach_decision(our_home, gateway)
  306. if standalone is not None:
  307. return standalone
  308. if gateway.serves(profile):
  309. return HostAttachDecision(ATTACH, attach_message(gateway, profile), gateway, transient=True)
  310. if not gateway.served_known:
  311. # Give the owner its bounded window to answer before judging it: during the boot race the
  312. # record lands a moment before the control socket binds.
  313. waited = host_gateway(wait_for_channel=ATTACH_CHANNEL_WAIT_S)
  314. if waited is None:
  315. return HostAttachDecision(START, "")
  316. gateway = waited
  317. standalone = standalone_attach_decision(our_home, gateway)
  318. if standalone is not None:
  319. return standalone
  320. if gateway.serves(profile):
  321. return HostAttachDecision(ATTACH, attach_message(gateway, profile), gateway, transient=True)
  322. try:
  323. attached = request_serve_profile(profile, owner=gateway)
  324. except Exception:
  325. logger.debug("host gateway rescan request failed", exc_info=True)
  326. attached = None
  327. if attached is not None and attached.serves(profile):
  328. return HostAttachDecision(ATTACH, attach_message(attached, profile), attached, transient=True)
  329. if attached is not None and attached.standalone:
  330. # One-process-per-profile fleet: the owner is another profile's standalone gateway. Refusing
  331. # here exits 78, which every supervisor treats as permanent — on a launchd fleet that parked
  332. # every unit but the first to claim the host lock. Start beside it. decide() runs twice per
  333. # start (CLI guard + start_gateway), so this is INFO; the host-lock claim in run.py logs the
  334. # one WARNING with the `gateway migrate --multiplex` converge hint.
  335. from hermes_cli.gateway_migrate import MIGRATE_COMMAND
  336. logger.info(
  337. "Another profile's standalone gateway owns this host (%s); starting profile '%s' beside it. "
  338. "Fold every profile onto one gateway with: %s",
  339. attached.describe(), profile, MIGRATE_COMMAND)
  340. return HostAttachDecision(START, "")
  341. if not gateway.served_known:
  342. # The owner never answered, so we know only that it exists. ATTACH here (on the record's
  343. # word) parked a supervised unit against a served set nobody had committed to yet.
  344. return HostAttachDecision(
  345. REFUSE, _unknown_served_message(gateway, profile), gateway, transient=True)
  346. return HostAttachDecision(REFUSE, _refuse_message(gateway, profile), gateway)