gateway_runtime.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495
  1. """Validated, credential-free gateway endpoint discovery for local clients."""
  2. from __future__ import annotations
  3. from dataclasses import dataclass
  4. import ipaddress
  5. import math
  6. import os
  7. from pathlib import Path
  8. from typing import TYPE_CHECKING, Literal
  9. from urllib.parse import urlsplit
  10. if TYPE_CHECKING:
  11. from gateway.runtime_contract import RuntimeState
  12. # Total startup deadline a client waits for a cold daemon. A cold boot imports the whole
  13. # runtime, seeds the skill library and warms the turn machinery: ~10-20 s on a loaded
  14. # 32-core CI runner with dozens of sibling boots, where 30 s tipped every attaching client
  15. # of a parallel e2e shard into `starting: deadline`. A minute is still a verdict, not a hang.
  16. DEFAULT_ENSURE_TIMEOUT = 60.0
  17. @dataclass(frozen=True)
  18. class GatewayEndpoint:
  19. profile_id: str
  20. instance_id: str
  21. authority_epoch: int
  22. runtime_protocol: int
  23. api_origin: str
  24. supervisor: Literal["none", "systemd", "launchd", "windows", "external"]
  25. capabilities: frozenset[str]
  26. # Home whose control socket answers for this endpoint: the profile's own home, or the
  27. # default multiplexer's root when the profile is served by it.
  28. control_home: str | None = None
  29. # Commit the owner BOOTED from (its ``identify`` ``code_sha``); None when it cannot name one.
  30. # A gateway that outlived ``hermes update`` keeps serving old code, so an attaching client
  31. # compares this with its own checkout and restarts the owner instead of re-attaching it.
  32. code_sha: str | None = None
  33. @dataclass(frozen=True)
  34. class GatewayDiscovery:
  35. state: RuntimeState
  36. endpoint: GatewayEndpoint | None = None
  37. reason_code: str | None = None
  38. # Human-readable cause behind a bounded ``reason_code`` when the owner published one (the
  39. # multiplexer's park reason), or the redacted output tail of a start this call made that died
  40. # (``runtime_exited``), or the unsafe local control node a discovery refused (path, owner uid or
  41. # mode, and the fix); never raw peer data.
  42. detail: str | None = None
  43. def attachable(result: GatewayDiscovery) -> bool:
  44. """Whether a client may attach to this discovery/ensure verdict.
  45. ``ready`` owners, plus a restart's after-turn wait (``draining``/``restart_wait``): that owner
  46. still owns its turns and must stay reachable to answer a parked approval/clarify (#139041).
  47. Shared by every client bootstrap (``connect_gateway`` and Ink's ``gateway_bootstrap.py``)."""
  48. if result.endpoint is None:
  49. return False
  50. from gateway.drain_control import RESTART_WAIT_REASON
  51. return result.state == "ready" or (result.state == "draining" and result.reason_code == RESTART_WAIT_REASON)
  52. def _canonical_home(home: str | Path) -> str:
  53. return os.path.normcase(str(Path(home).expanduser().resolve()))
  54. def gateway_profile_home(home: str | Path) -> Path:
  55. """The profile home whose gateway serves *home*: the home itself, or for a private overlay home
  56. (a task dir mirroring a profile through symlinks, its ``gateway.lock`` among them) the profile its
  57. lock names. A client compares the owner's ``profile_id`` with this, never the overlay path.
  58. Not normcased: ``profile_id`` keeps the owner's own spelling (Windows)."""
  59. return _overlay_owner_home(Path(home).expanduser().resolve())
  60. def _overlay_owner_home(home: Path) -> Path:
  61. """Return the owner named by a private overlay's gateway lock, if it has one."""
  62. if os.name == "nt":
  63. return home
  64. from hermes_cli.gateway_runtime_discovery import DiscoveryError, _private_lock, _private_node
  65. lock = home / "gateway.lock"
  66. try:
  67. # An overlay home intentionally shares only the owner's lock. Do not turn an arbitrary
  68. # link into a profile alias: both homes and the target lock must meet ordinary discovery's
  69. # private-control checks.
  70. _private_node(home, kind="directory", home=True)
  71. target = lock.resolve(strict=True)
  72. if not lock.is_symlink() or target.name != "gateway.lock":
  73. return home
  74. _private_node(target.parent, kind="directory", home=True)
  75. _private_lock(target)
  76. except (FileNotFoundError, OSError, DiscoveryError):
  77. return home
  78. return target.parent
  79. def _endpoint(payload: dict, home: Path, control_home: Path | None = None) -> GatewayDiscovery:
  80. if type(payload.get("runtime_protocol")) is not int or payload["runtime_protocol"] != 1:
  81. return GatewayDiscovery("incompatible", reason_code="runtime_protocol")
  82. profiles = payload.get("served_profiles")
  83. if not isinstance(profiles, list):
  84. return GatewayDiscovery("inaccessible", reason_code="profile_mismatch")
  85. # Canonicalize BOTH sides: on Windows normcase lower-cases the served spelling, so a
  86. # caller's mixed-case Path (WorkerRPC) never matched and every worker saw owner_unavailable.
  87. wanted = _canonical_home(home)
  88. matches = [p for p in profiles if isinstance(p, dict)
  89. and isinstance(p.get("home"), str) and _canonical_home(p["home"]) == wanted]
  90. if len(matches) != 1 or not isinstance(matches[0].get("profile_id"), str) or not matches[0]["profile_id"]:
  91. return GatewayDiscovery("inaccessible", reason_code="profile_mismatch")
  92. state = payload.get("state")
  93. # A graceful restart's after-turn wait still owns its in-flight turns (#139041): it stays
  94. # ``draining`` (exit 6, never a fresh-start target) but carries its endpoint, so an interactive
  95. # client can attach to answer a turn parked on approval/clarify. New input queues for the next owner.
  96. from gateway.drain_control import RESTART_WAIT_REASON
  97. restart_wait = state == "draining" and payload.get("drain_reason") == RESTART_WAIT_REASON
  98. if state in {"starting", "draining", "conflict"} and not restart_wait:
  99. # An auto-started gateway ending through its idle exit names that, so its clients wait
  100. # for it to go and start a fresh one instead of reporting a refusal.
  101. idle = state == "draining" and payload.get("drain_reason") == "idle_exit"
  102. return GatewayDiscovery(state, reason_code="idle_exit" if idle else None)
  103. if state != "ready" and not restart_wait:
  104. return GatewayDiscovery("inaccessible", reason_code="invalid_runtime_state")
  105. capabilities = payload.get("capabilities")
  106. if (not isinstance(capabilities, list) or not all(isinstance(c, str) for c in capabilities)
  107. or "session-authority-v1" not in capabilities):
  108. return GatewayDiscovery("incompatible", reason_code="session_authority_unavailable")
  109. epoch, instance = payload.get("authority_epoch"), payload.get("instance_id")
  110. if type(epoch) is not int or epoch <= 0 or not isinstance(instance, str) or not instance:
  111. return GatewayDiscovery("inaccessible", reason_code="invalid_runtime_identity")
  112. origin = payload.get("api_origin")
  113. if not isinstance(origin, str):
  114. return GatewayDiscovery("inaccessible", reason_code="invalid_api_origin")
  115. try:
  116. parsed = urlsplit(origin)
  117. if (parsed.scheme not in {"http", "https"} or not parsed.hostname
  118. or parsed.username is not None or parsed.password is not None
  119. or parsed.path or parsed.query or parsed.fragment or not parsed.port
  120. or not ipaddress.ip_address(parsed.hostname).is_loopback):
  121. return GatewayDiscovery("inaccessible", reason_code="invalid_api_origin")
  122. except ValueError:
  123. return GatewayDiscovery("inaccessible", reason_code="invalid_api_origin")
  124. supervisor = payload.get("supervisor")
  125. if supervisor not in {"none", "systemd", "launchd", "windows", "external"}:
  126. return GatewayDiscovery("inaccessible", reason_code="invalid_supervisor")
  127. return GatewayDiscovery("draining" if restart_wait else "ready", GatewayEndpoint(
  128. profile_id=matches[0]["profile_id"], instance_id=instance,
  129. authority_epoch=epoch, runtime_protocol=1, api_origin=origin,
  130. supervisor=supervisor, capabilities=frozenset(capabilities),
  131. control_home=str(control_home) if control_home is not None else None,
  132. code_sha=code_sha if isinstance(code_sha := payload.get("code_sha"), str) and code_sha else None,
  133. ), reason_code=RESTART_WAIT_REASON if restart_wait else None)
  134. def control_home_for(home: Path, endpoint: GatewayEndpoint | None) -> Path:
  135. """Home whose control socket mints tickets for *endpoint* (the multiplexer's for a served profile)."""
  136. if endpoint is not None and endpoint.control_home:
  137. return Path(endpoint.control_home)
  138. return home
  139. def _starting_multiplexer_root(home: Path) -> Path | None:
  140. """The root whose reservation is pending AND whose boot policy would serve *home*, else None.
  141. A ``gateway.standalone: true`` secondary is never served by it, so a reserved (or ready) root
  142. must not make its client wait out the deadline instead of starting the profile's own owner.
  143. One uncached config parse plus a lock probe: call it once per ensure iteration."""
  144. from hermes_cli.gateway_runtime_discovery import missing_owner_state
  145. from hermes_cli.gateway_runtime_multiplex import implied_host_root
  146. root = implied_host_root(home)
  147. return root if root is not None and missing_owner_state(root) == "starting" else None
  148. def _standalone_multiplexer_reason(root: Path) -> str | None:
  149. """A live host's reason for not serving its named profiles, if it published one.
  150. A held root lock ordinarily means that a named profile should wait for its
  151. multiplexer. A host which published a guard refusal is different: it is a
  152. standalone gateway and can never add the named profile during that boot. Read
  153. with the same liveness rules as ``hermes gateway status``'s standalone box.
  154. """
  155. from hermes_cli.gateway_multiplex_mode import recorded_standalone_reason
  156. reason = recorded_standalone_reason(root / "gateway_state.json")
  157. return reason[:500] if reason else None
  158. def _parked_by_multiplexer(payload: dict, home: Path) -> GatewayDiscovery | None:
  159. """A secondary the multiplexer could not serve (unusable store, home owned elsewhere) is
  160. published under ``parked_profiles`` (name -> reason). That is a terminal verdict for its
  161. clients: waiting for a ``served_profiles`` entry that will never appear is the wrong answer."""
  162. parked = payload.get("parked_profiles")
  163. if not isinstance(parked, dict):
  164. return None
  165. from hermes_cli.profiles import normalize_profile_name
  166. name = normalize_profile_name(home.name)
  167. reason = next((r for n, r in parked.items() if normalize_profile_name(str(n)) == name), None)
  168. if reason is None:
  169. return None
  170. return GatewayDiscovery("inaccessible", reason_code="profile_parked", detail=str(reason)[:500] or None)
  171. def _refused_owner_verdict(exc: PermissionError) -> GatewayDiscovery | None:
  172. """The named refusal when the client proved the boundary from the control pipe, else None."""
  173. from gateway.runtime_bootstrap_windows import RefusedOwner
  174. if isinstance(exc, RefusedOwner):
  175. return GatewayDiscovery("inaccessible", reason_code=exc.reason_code, detail=str(exc))
  176. return None
  177. def _served_by_multiplexer(home: Path, *, timeout: float) -> GatewayDiscovery | None:
  178. """A served secondary has no socket of its own: the default multiplexer's control socket
  179. answers for it, and its ``identify`` lists the home under ``served_profiles``. Only the
  180. multiplexer's live descriptor proves service; a held ``gateway.lock`` alone does not."""
  181. from hermes_cli.gateway_runtime_discovery import DiscoveryError, query_identify
  182. from hermes_cli.gateway_runtime_multiplex import multiplexer_root_for
  183. root = multiplexer_root_for(home)
  184. if root is None:
  185. return None
  186. try:
  187. payload = query_identify(root, timeout=timeout)
  188. except PermissionError as exc:
  189. # The multiplexer's pipe proved the boundary: name it for the secondary too (#140706), unless
  190. # the boot policy never serves this home (a standalone secondary starts its own gateway). Any
  191. # other refusal is "not served" before a config is read, as on POSIX, where none is proven.
  192. named = _refused_owner_verdict(exc)
  193. if named is None:
  194. return None
  195. from hermes_cli.gateway_multiplex_mode import host_serves_named_profile
  196. if host_serves_named_profile(root, home) is False:
  197. return None
  198. return named
  199. except (FileNotFoundError, ConnectionRefusedError, TimeoutError, DiscoveryError,
  200. OSError, ValueError, TypeError):
  201. return None
  202. result = _endpoint(payload, home, control_home=root)
  203. if result.state == "inaccessible" and result.reason_code == "profile_mismatch":
  204. return _parked_by_multiplexer(payload, home) # else: the multiplexer does not serve this profile
  205. return result
  206. _NATIVE_WINDOWS = os.name == "nt"
  207. def _owner_pid(home: Path) -> int | None:
  208. """The PID a refusal may name: one of *home*'s gateway records whose process is provably that
  209. gateway, else None (the caller keeps the generic ``authorization``).
  210. Proof is the shared identity chain: alive, its recorded start time known and equal to the live
  211. one (a recycled PID differs), its recorded home not another profile's, and its live command line
  212. (or, when that is unreadable across the elevation boundary, its record) a gateway's. A stale
  213. ``gateway.pid`` falls through to ``gateway.lock`` instead of masking it."""
  214. from gateway.status import (
  215. _get_process_start_time,
  216. _live_pid_from_record,
  217. _read_json_file,
  218. _record_matches_live_gateway_pid,
  219. recorded_gateway_home_conflicts,
  220. )
  221. for name in ("gateway.pid", "gateway.lock"):
  222. record = _read_json_file(home / name, bare_pid_ok=True)
  223. pid = _live_pid_from_record(record)
  224. if pid is None or pid == os.getpid():
  225. continue
  226. recorded = record.get("start_time")
  227. if recorded is None or recorded != _get_process_start_time(pid):
  228. continue # unverifiable or recycled: never name it
  229. if recorded_gateway_home_conflicts(record, expected_home=home):
  230. continue
  231. if not _record_matches_live_gateway_pid(record, pid):
  232. continue
  233. return pid
  234. return None
  235. def _refused(home: Path, *, windows: bool = _NATIVE_WINDOWS) -> GatewayDiscovery:
  236. """The owner's control channel refused this same-user client. On Windows (*windows*, the host
  237. fact) name the boundary (an elevated owner, or one in another logon session) with its remedy
  238. instead of the generic ``authorization`` that reads as a credential failure (#139845, #140252)."""
  239. if windows and (pid := _owner_pid(home)) is not None:
  240. from gateway.runtime_bootstrap_windows import probe_refused_owner
  241. try:
  242. named = probe_refused_owner(pid)
  243. except OSError:
  244. named = None
  245. if named is not None:
  246. return GatewayDiscovery("inaccessible", reason_code=named[0], detail=named[1])
  247. return GatewayDiscovery("inaccessible", reason_code="authorization")
  248. def discover_gateway_endpoint(profile_home: str | Path, *, timeout: float = 2.0) -> GatewayDiscovery:
  249. """Read live control identity, preserving uncertainty instead of spawning.
  250. A named profile served by the default multiplexer resolves to the multiplexer's endpoint
  251. with ``profile_id`` = the profile's own home, so clients attach to it with that identity.
  252. """
  253. from hermes_cli.gateway_runtime_discovery import DiscoveryError, missing_owner_state, query_identify
  254. if not math.isfinite(timeout) or timeout <= 0:
  255. raise ValueError("timeout must be finite and positive")
  256. home = _overlay_owner_home(Path(_canonical_home(profile_home)))
  257. try:
  258. return _endpoint(query_identify(home, timeout=timeout), home)
  259. except (FileNotFoundError, ConnectionRefusedError):
  260. served = _served_by_multiplexer(home, timeout=timeout)
  261. if served is not None:
  262. return served
  263. state = missing_owner_state(home)
  264. if state == "inaccessible" and os.name != "nt":
  265. from gateway.runtime_ownership import unreadable_lock_recovery
  266. if detail := unreadable_lock_recovery(home / "gateway.lock"):
  267. return GatewayDiscovery(state, reason_code="foreign_stale_lock", detail=detail)
  268. return GatewayDiscovery(state)
  269. except TimeoutError:
  270. return GatewayDiscovery("inaccessible", reason_code="control_timeout")
  271. except DiscoveryError as exc:
  272. # The code stays bounded; the detail names the local node, its owner or mode, and the fix.
  273. return GatewayDiscovery("inaccessible", reason_code=exc.reason, detail=exc.bounded_detail(500))
  274. except PermissionError as exc:
  275. named = _refused_owner_verdict(exc) # the client proved the boundary from the control pipe (#140706)
  276. return named if named is not None else _refused(home)
  277. except (OSError, ValueError, TypeError):
  278. # Raw peer data, paths and URLs may contain secrets; report bounded codes.
  279. return GatewayDiscovery("inaccessible", reason_code="invalid_control_peer")
  280. def _update_fenced(homes) -> bool:
  281. """True while an update owns the install, judged the way main's launchers and the Desktop
  282. judge ``.hermes-update-in-progress``: a live (or still being written) marker, or a dead or
  283. malformed one while the checkout lock is held (a killed update's tree still runs). A dead
  284. marker over a free lock is a killed update: it never blocks a launch.
  285. Read-only: a client never clears the updater's fence; the launch-time reclaim
  286. (``update_lock.read_live_update``) and the next update remove a dead marker. A marker that
  287. exists but cannot be read is unjudgeable and fences.
  288. """
  289. import time
  290. from hermes_cli import update_lock
  291. for fence_home in homes:
  292. path = fence_home / update_lock.MARKER_NAME
  293. try:
  294. raw = path.read_bytes()
  295. if not raw:
  296. # An exclusive first publish in flight (contract A3), not yet a dead claim.
  297. if time.time() - path.stat().st_mtime < update_lock.EMPTY_MARKER_GRACE_SECONDS:
  298. return True
  299. except FileNotFoundError:
  300. continue
  301. except OSError:
  302. return True
  303. if update_lock.judge_marker(raw)[0] in {"live", "ours"} or update_lock.checkout_lock_held():
  304. return True
  305. return False
  306. def _fenced_verdict(home: Path, timeout: float) -> GatewayDiscovery:
  307. """What ensure answers while an update owns the install. The update's own drain
  308. (``pause-for-update``) is a restart's after-turn wait whose owner may hold a turn parked on
  309. approval/clarify, so it is handed out for clients to answer it (#139041). Discovery only:
  310. nothing is started or replaced while fenced, and any other owner state stays ``update_paused``."""
  311. observed = discover_gateway_endpoint(home, timeout=timeout)
  312. if observed.state == "draining" and attachable(observed):
  313. return observed
  314. return GatewayDiscovery("draining", reason_code="update_paused")
  315. _REPLACE_FAILED_DETAIL = "an older Hermes gateway still owns this profile after a stop; run: hermes gateway stop"
  316. # ``draining`` reasons after which the owner is going away by itself: its own idle exit, or the
  317. # older-Hermes owner this call just stopped. Ensure waits it out and starts the current owner.
  318. _RETIRING = frozenset({"idle_exit", "replaced"})
  319. def _observe_replacing_stale(home: Path, deadline: float, replaced: set[Path]) -> GatewayDiscovery:
  320. """Discovery, except that a gateway an older Hermes left on this home is stopped once (like
  321. ``hermes gateway stop``) and reported as retiring; a refusal keeps ``incompatible`` + its remedy."""
  322. from hermes_cli.gateway_runtime_service import remaining
  323. observed = discover_gateway_endpoint(home, timeout=remaining(deadline))
  324. if observed.state != "incompatible" or observed.reason_code != "runtime_protocol":
  325. return observed
  326. if home in replaced:
  327. return GatewayDiscovery("incompatible", reason_code="runtime_protocol", detail=_REPLACE_FAILED_DETAIL)
  328. from hermes_cli import gateway_runtime_replace as replace
  329. replaced.add(home)
  330. refusal = replace.replace_incompatible_owner(home, deadline=deadline)
  331. if refusal is not None:
  332. return GatewayDiscovery("incompatible", reason_code="runtime_protocol",
  333. detail=replace.REFUSAL_DETAIL.get(refusal, _REPLACE_FAILED_DETAIL))
  334. return GatewayDiscovery("draining", reason_code="replaced")
  335. def ensure_gateway_runtime(profile_home: str | Path, *, timeout: float = DEFAULT_ENSURE_TIMEOUT,
  336. idle_exit: bool = False) -> GatewayDiscovery:
  337. """Ensure once; pending remains pending at deadline.
  338. Never installs anything. The one owner it replaces is a gateway an older Hermes left running on
  339. this install's own home (``incompatible``/``runtime_protocol``, the upgrade case): it is stopped
  340. the way ``hermes gateway stop`` stops it, once, and the current one started like any absent
  341. owner. Another install's or a newer Hermes's gateway stays refused, with the reason in ``detail``
  342. (``hermes_cli.gateway_runtime_replace``).
  343. A successful service command or Popen is not session readiness. After an
  344. owner/start request is observed this invocation never launches another.
  345. ``idle_exit``: an unmanaged daemon this call starts ends itself once idle (client entrypoints:
  346. chat, TUI, cron/kanban/ACP clients). A gateway that is ending that way reports ``draining``
  347. (``idle_exit``); this call waits for it to go and starts the replacement transparently.
  348. """
  349. import time
  350. from hermes_cli.gateway_runtime_service import (
  351. RuntimeStartError, discover_existing_gateway_service, remaining,
  352. start_existing_gateway_service,
  353. )
  354. from hermes_constants import get_default_hermes_root, get_process_hermes_home
  355. if not math.isfinite(timeout) or timeout <= 0:
  356. raise ValueError("timeout must be finite and positive")
  357. deadline = time.monotonic() + timeout
  358. home = _overlay_owner_home(Path(_canonical_home(profile_home)))
  359. requested = False
  360. child = None
  361. child_home, child_offset = home, 0
  362. # A gateway ending through its own idle exit is not a refusal: wait for it to go (its lock is
  363. # still held for a moment after its control socket closes, which reads as "starting"), then
  364. # start the replacement this call owes the client.
  365. retiring = False
  366. replaced: set[Path] = set()
  367. delay = 0.025
  368. try:
  369. while True:
  370. # The install-root marker also covers profiles.
  371. if _update_fenced({home, get_process_hermes_home(), get_default_hermes_root()}):
  372. return _fenced_verdict(home, remaining(deadline))
  373. # Poll BEFORE discovering: an exit observed first, then "absent", cannot be a child that
  374. # published its owner a moment later. A winning racer shows up as starting/ready instead.
  375. status = child.poll() if child is not None else None
  376. observed = _observe_replacing_stale(home, deadline, replaced)
  377. if observed.reason_code == "control_timeout":
  378. return GatewayDiscovery("starting", reason_code="deadline")
  379. if observed.state == "draining" and observed.reason_code in _RETIRING:
  380. retiring, requested, child = True, False, None
  381. time.sleep(min(0.1, remaining(deadline)))
  382. continue
  383. if observed.state not in {"absent", "starting"}:
  384. return observed
  385. if status is not None and observed.state == "absent":
  386. # The one start this invocation requested died without leaving an owner: waiting
  387. # out the deadline cannot change that, and this call never launches a second one.
  388. # Its own (redacted) output is the answer the user needs, not a pointer to a file.
  389. from hermes_cli.gateway_runtime_start import startup_failure_report
  390. return GatewayDiscovery("inaccessible", reason_code="runtime_exited",
  391. detail=startup_failure_report(child_home, child_offset, status, child.pid))
  392. if observed.state == "starting" and not retiring:
  393. requested = True
  394. if observed.state == "absent":
  395. retiring = False
  396. reserved = _starting_multiplexer_root(home) if observed.state == "absent" and not requested else None
  397. if reserved is not None and (reason := _standalone_multiplexer_reason(reserved)):
  398. return GatewayDiscovery("inaccessible", reason_code="profile_parked", detail=reason)
  399. # A default multiplexer that is still starting will serve this named profile; never
  400. # spawn a competing per-profile daemon while its reservation is pending.
  401. if reserved is not None:
  402. requested = True
  403. if observed.state == "absent" and not requested:
  404. # A named profile the default multiplexer serves has no daemon of its own: start
  405. # (or await) the MULTIPLEXER. A per-profile spawn here would become a second owner
  406. # that blocks the multiplexer's next all-or-nothing reserve.
  407. from hermes_cli.gateway_runtime_multiplex import implied_host_root, multiplexer_serves_home
  408. target = multiplexer_serves_home(home) or home
  409. service = discover_existing_gateway_service(target, deadline=deadline)
  410. if service is None and target == home and (host := implied_host_root(home)) is not None:
  411. # Main's guard refuses a named profile a gateway of its own unless it is standalone
  412. # or already installed; with no multiplexer evidence yet, the host is still its owner.
  413. target = host
  414. service = discover_existing_gateway_service(target, deadline=deadline)
  415. # Runtime locks settle races remaining after this second probe.
  416. observed = discover_gateway_endpoint(home, timeout=remaining(deadline))
  417. if observed.state != "absent":
  418. continue
  419. if service is not None:
  420. start_existing_gateway_service(service, deadline=deadline)
  421. else:
  422. from hermes_cli.gateway_runtime_start import spawn_unmanaged_gateway, stdio_log_size
  423. child_home, child_offset = target, stdio_log_size(target)
  424. child = spawn_unmanaged_gateway(target, deadline=deadline, idle_exit=idle_exit)
  425. requested = True
  426. time.sleep(min(delay, remaining(deadline)))
  427. # Capped low: each probe is a lock-file read or one local identify, and READY is noticed
  428. # within ~50 ms instead of up to 250 ms later on every cold start.
  429. delay = min(delay * 1.5, 0.05)
  430. except TimeoutError:
  431. return GatewayDiscovery("starting", reason_code="deadline")
  432. except RuntimeStartError as exc:
  433. return GatewayDiscovery(exc.state, reason_code=exc.reason)
  434. except PermissionError:
  435. return _refused(home)
  436. except OSError:
  437. return GatewayDiscovery("inaccessible", reason_code="runtime_start_failed")