run_profile_reconcile.py 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. """Hot-serve for ``gateway.multiplex_profiles``: keep the served-profile set in step with ``profiles/``
  2. while the multiplexer runs, instead of snapshotting it once at boot.
  3. Three things were start-time snapshots: the secondary adapter set (``_start_secondary_profile_adapters``),
  4. the ``served_profiles`` record in ``gateway_state.json`` (``_record_served_profiles``) and the cron
  5. ticker's ``profile_homes`` list. Everything else (``/p/<profile>/`` prefixes, profile-route eligibility,
  6. handoff/kanban watchers, shared ingress) already reads ``profiles_to_serve()`` / ``_profile_adapters``
  7. live, so reconciling those three is enough for a profile created after boot to be served.
  8. ``reconcile_served_profiles`` runs on the loop under one lock, triggered by the ``rescan-profiles`` control
  9. verb (``hermes_cli/profiles.py`` create/delete fire it through the control socket) and by the supervised
  10. ``_profile_reconcile_watcher`` every ``_PROFILE_RESCAN_INTERVAL_SECS`` as the safety net. A served profile whose ``config.yaml``/``.env``
  11. changed since its adapters were last built is re-scanned too: creators make the profile first and add the
  12. bot token afterwards, and without this an adapter-less profile would stay adapter-less forever.
  13. The same watcher tick (on every gateway, multiplexed or not) re-syncs a home's shell hooks when its
  14. ``shell-hooks-allowlist.json`` (or ``config.yaml``) changed: a non-TTY gateway skips unconsented hooks, so consent recorded
  15. afterwards must register them without a restart (#132993). That re-runs hook registration only, never
  16. an adapter rebuild.
  17. """
  18. from __future__ import annotations
  19. import asyncio
  20. import logging
  21. from functools import partial
  22. from pathlib import Path
  23. from typing import Any, Dict, Optional
  24. from gateway.run_shutdown import _log_suppressed
  25. from hermes_cli.config_backend import ConfigBackendUnavailable
  26. from utils import file_cache_version
  27. logger = logging.getLogger(__name__)
  28. _PROFILE_RESCAN_INTERVAL_SECS = 30.0
  29. _PROFILE_SIGNATURE_FILES = (".env",) # config.yaml: the config backend's version, below
  30. # Bound for one own-gateway liveness probe (``live_gateway_pid_for_home``): the control-socket
  31. # read it can end in has no timeout of its own — on Windows the named pipe stalls there until
  32. # its peer answers — so the await needs one. A healthy identify answers in milliseconds.
  33. _OWN_GATEWAY_PROBE_TIMEOUT_SECS = 5.0
  34. def _stat_signature(path: Path) -> Optional[tuple]:
  35. try:
  36. return file_cache_version(path, always_digest=True) # stored, compared on a later tick
  37. except OSError:
  38. return None
  39. def profile_serve_signature(home: Path) -> tuple:
  40. """Change detector for a served profile's credentials/config: the config backend's signature
  41. of its config layer, then ``file_cache_version`` of ``.env``. Each is hashed every time
  42. (``always_digest``) where it is a file: it is compared long after it was taken, and a racy
  43. digest going empty would read as a change and reconnect the profile's adapters. A remote layer
  44. changes with no local file change, so its signature is the backend's version. A config backend
  45. outage raises ``ConfigBackendUnavailable``; the reconcile cycle contains it and retries."""
  46. from hermes_cli.config_backend import config_version_or_none, supports_file_tooling
  47. cfg = Path(home) / "config.yaml"
  48. sig = [_stat_signature(cfg) if supports_file_tooling() else config_version_or_none(cfg)]
  49. for name in _PROFILE_SIGNATURE_FILES:
  50. try:
  51. sig.append(file_cache_version(Path(home) / name, always_digest=True))
  52. except OSError:
  53. sig.append(None)
  54. return tuple(sig)
  55. def _added_profile_blocker(launch: Path) -> Optional[str]:
  56. """Why a single-profile boot must not start multiplexing now that a second profile exists: the
  57. ``gateway.auto_multiplex_migration: false`` opt-out on the default profile, else boot's preflight."""
  58. from hermes_cli.gateway_migrate_guards import auto_migration_opted_out
  59. from hermes_cli.gateway_multiplex_mode import AUTO_MIGRATION_OPTED_OUT_REASON, implicit_multiplex_blocker
  60. if auto_migration_opted_out(launch):
  61. return AUTO_MIGRATION_OPTED_OUT_REASON
  62. return implicit_multiplex_blocker()
  63. class GatewayProfileReconcileMixin:
  64. """Runtime reconciliation of the multiplexed served-profile set (hot add / unroute / credential-add)."""
  65. _served_profile_homes: Optional[dict[str, Path]] = None
  66. _served_profile_signatures: Optional[dict[str, tuple]] = None
  67. _profile_reconcile_lock: Optional[asyncio.Lock] = None
  68. _profile_own_gateway_warned: Optional[set[str]] = None
  69. _profile_probe_timeout_warned: Optional[set[str]] = None
  70. _hook_consent_signatures: Optional[dict[str, Optional[tuple]]] = None
  71. _single_profile_blocker_logged: Optional[str] = None
  72. # ── state helpers ─────────────────────────────────────────────────────────────────────────────
  73. def _reconcile_lock(self) -> asyncio.Lock:
  74. if self._profile_reconcile_lock is None:
  75. self._profile_reconcile_lock = asyncio.Lock()
  76. return self._profile_reconcile_lock
  77. def served_profile_names(self) -> list:
  78. """Profiles this multiplexer currently serves (active first), from the live bookkeeping."""
  79. homes = self._served_profile_homes or {}
  80. active = getattr(self, "_primary_profile_name", None) or "default"
  81. return ([active] if active in homes or not homes else []) + sorted(n for n in homes if n != active)
  82. def _note_served_profiles(self, profile_homes) -> None:
  83. """Called by ``_record_served_profiles``: remember the served set and each home's signature."""
  84. homes = {str(name): Path(home) for name, home in profile_homes}
  85. self._served_profile_homes = homes
  86. sigs = self._served_profile_signatures if isinstance(self._served_profile_signatures, dict) else {}
  87. self._served_profile_signatures = {name: sigs.get(name) or profile_serve_signature(home)
  88. for name, home in homes.items()}
  89. # ── watcher ───────────────────────────────────────────────────────────────────────────────────
  90. async def _profile_reconcile_watcher(self, interval: float = _PROFILE_RESCAN_INTERVAL_SECS) -> None:
  91. """Supervised safety net, every ``interval`` seconds: re-sync shell hooks whose consent changed
  92. (every gateway), then rescan ``profiles/`` when multiplexing (creators signal the control
  93. socket for an immediate rescan); a gateway standalone only because the host had one profile
  94. starts multiplexing here once a second one exists."""
  95. while self._running:
  96. await asyncio.sleep(interval)
  97. if not self._running:
  98. return
  99. try:
  100. await self._sync_hooks_on_consent_change()
  101. # A standalone gateway never runs a profile reconcile from the watcher, unless it is the
  102. # single-profile boot that starts multiplexing in place for a newly created profile.
  103. if self._multiplex_on() or await self._multiplex_for_added_profile():
  104. await self.reconcile_served_profiles(reason="watcher")
  105. except asyncio.CancelledError:
  106. raise
  107. except (Exception, ConfigBackendUnavailable): # a config outage is a SystemExit by design
  108. logger.warning("Served-profile reconcile failed; retrying next cycle", exc_info=True)
  109. async def _sync_hooks_on_consent_change(self) -> None:
  110. """Re-sync the shell hooks of each home (launch + served) whose consent allowlist or config
  111. changed since the last tick, under that home's scope: hooks consented since start register, revoked ones stop
  112. firing. A home seen for the first time syncs once (consent may have landed after startup
  113. registered). Hook registration only: adapters and sessions are never touched."""
  114. from hermes_constants import get_process_hermes_home
  115. homes = {Path(h) for h in (get_process_hermes_home(), *(self._served_profile_homes or {}).values())}
  116. self._hook_consent_signatures = await asyncio.to_thread(
  117. self._resync_changed_hook_consent, homes, self._hook_consent_signatures or {})
  118. def _resync_changed_hook_consent(self, homes, seen: dict) -> dict:
  119. """Blocking half of ``_sync_hooks_on_consent_change``; returns the signatures it applied. The
  120. key covers ``config.yaml`` too: a sync that read an empty/unreadable config applied nothing,
  121. so its recovery must re-run it. A failed sync is left unrecorded and retried next tick."""
  122. from agent.shell_hooks import ALLOWLIST_FILENAME, sync_from_config
  123. from gateway.run import _load_gateway_config, _profile_runtime_scope
  124. from gateway.run_turn import GatewayTurnMixin
  125. current = {}
  126. for home in (h.resolve() for h in homes):
  127. sig = tuple(_stat_signature(home / name) for name in (ALLOWLIST_FILENAME, "config.yaml"))
  128. if seen.get(str(home), ()) == sig:
  129. current[str(home)] = sig
  130. continue
  131. load = partial(_load_gateway_config, home / "config.yaml")
  132. scope = (_profile_runtime_scope(home) if self._multiplex_on()
  133. else GatewayTurnMixin._standalone_launch_scope())
  134. try:
  135. with scope:
  136. sync_from_config(load)
  137. except Exception: # health: allow BLE001 -- one home's failure never blocks the others
  138. logger.warning("shell-hook re-sync failed for %s; retrying next cycle", home, exc_info=True)
  139. continue
  140. current[str(home)] = sig
  141. return current
  142. # ── reconcile ─────────────────────────────────────────────────────────────────────────────────
  143. async def reconcile_served_profiles(self, *, reason: str = "request") -> dict[str, Any]:
  144. """Diff ``profiles/`` (what exists now) against the served set (the process reservation):
  145. reserve + build the runtime and start adapters for new profiles, tear down, unroute and
  146. unreserve deleted ones, (re)build adapters for served profiles whose config/.env changed.
  147. Other profiles' adapters are never touched. A new profile whose home another gateway owns or
  148. whose store is unusable is parked (logged, not served) and left for an explicit rescan.
  149. Returns ``{"added", "removed", "rescanned", "parked", "served_profiles"}``."""
  150. from gateway.run import MultiplexConfigError
  151. from hermes_cli.profiles import profiles_to_serve
  152. result: dict[str, Any] = {"added": [], "removed": [], "rescanned": [], "parked": [], "reason": reason}
  153. if not self._multiplex_on() and not await self._multiplex_for_added_profile():
  154. return {**result, "multiplex": False, "served_profiles": self.served_profile_names()}
  155. if not self._running or self._served_profile_homes is None:
  156. # Startup enumerates profiles/ itself; a rescan before it finishes has nothing to diff against.
  157. return {**result, "pending": True, "served_profiles": self.served_profile_names()}
  158. async with self._reconcile_lock():
  159. active = getattr(self, "_primary_profile_name", None) or "default"
  160. live = {str(name): Path(home) for name, home in profiles_to_serve(multiplex=True)}
  161. listed = set(live) # a probe skip below is not a deletion: it never clears a parked verdict
  162. known = dict(self._served_profile_homes or {})
  163. from gateway.status import live_gateway_pid_for_home
  164. blocked = set()
  165. timed_out = set()
  166. warned = self._profile_own_gateway_warned or set()
  167. timeout_warned = self._profile_probe_timeout_warned or set()
  168. for name in list(live):
  169. if name == active or name in known:
  170. continue
  171. # The probe can end in a control-socket read that has no timeout of its own
  172. # (a Windows named pipe stalls there until its peer answers). Inline on the
  173. # loop thread it parked shutdown_watchdog liveness probes and the multiplexer
  174. # was hard-killed with exit 75 (#132547). Probe off the loop, bounded: a
  175. # stalled probe only wedges one bounded housekeeping worker for this cycle
  176. # (wait_for cancels the still-queued await), never the event loop.
  177. try:
  178. pid = await asyncio.wait_for(
  179. self._run_housekeeping_in_executor(live_gateway_pid_for_home, live[name]),
  180. timeout=_OWN_GATEWAY_PROBE_TIMEOUT_SECS)
  181. except TimeoutError:
  182. # Unprovable is not "own gateway running": skip it this cycle without
  183. # warning about (or remembering) a gateway that may not exist. Warn once
  184. # per stall; a peer that stays wedged repeats at DEBUG every cycle.
  185. timed_out.add(name)
  186. log = logger.debug if name in timeout_warned else logger.warning
  187. log("[MULTIPLEX] Own-gateway probe for profile '%s' timed out; "
  188. "not serving it this cycle", name)
  189. del live[name]
  190. continue
  191. if pid is not None:
  192. blocked.add(name)
  193. if name not in warned:
  194. logger.warning("[MULTIPLEX] Profile '%s' still runs its own gateway; "
  195. "stop it before the host can serve this profile", name)
  196. del live[name]
  197. self._profile_own_gateway_warned = blocked
  198. self._profile_probe_timeout_warned = timed_out
  199. sigs = self._served_profile_signatures or {}
  200. from gateway.run_runtime import unpark_profile
  201. for name in [n for n in self._parked_profile_names() if n not in listed]:
  202. unpark_profile(self, name)
  203. parked = self._parked_profile_names()
  204. # The watcher never re-parks the same profile every cycle; a creator's explicit signal does.
  205. retry_parked = reason == "control-socket"
  206. added = [n for n in live if n not in known and n != active and (retry_parked or n not in parked)]
  207. removed = [n for n in known if n not in live and n != active]
  208. changed = [n for n in live if n in known and n != active and n not in added
  209. and profile_serve_signature(live[n]) != sigs.get(n)]
  210. current = {n: h for n, h in live.items() if n in known or n == active}
  211. return await self._apply_profile_changes(
  212. current, added, removed, changed, reason=reason, live=live, result=result)
  213. async def _apply_profile_changes(self, current, added, removed, changed, *, reason,
  214. live=None, result=None):
  215. """Apply a selected diff under the reconcile lock, shared by the watcher and the
  216. serve/unserve control verbs. New profiles first get a runtime (reservation + store); one
  217. whose home another gateway owns or whose store is unusable is parked, not served."""
  218. from gateway.run import MultiplexConfigError
  219. from gateway.run_runtime import park_profile, unpark_profile
  220. from hermes_cli.profiles import profiles_to_serve
  221. active = getattr(self, "_primary_profile_name", None) or "default"
  222. result = dict(result or {"added": [], "removed": [], "rescanned": [], "parked": [], "reason": reason})
  223. result.setdefault("parked", [])
  224. live = live if live is not None else dict(current)
  225. known = dict(self._served_profile_homes or {})
  226. sigs = self._served_profile_signatures or {}
  227. if not (added or removed or changed):
  228. return {**result, "served_profiles": self.served_profile_names()}
  229. for name in removed:
  230. await self._unserve_profile(name, known[name])
  231. result["removed"].append(name)
  232. for name in list(added):
  233. reason_ = await self._serve_profile_runtime(name, live[name])
  234. if reason_ is not None:
  235. added.remove(name)
  236. park_profile(self, name, reason_)
  237. result["parked"].append(name)
  238. continue
  239. unpark_profile(self, name)
  240. current[name] = live[name]
  241. claimed = self._live_resource_claims(active)
  242. transient_failed = set()
  243. config_parked = set()
  244. for name in added + changed:
  245. # Only acknowledge the configuration observed before connecting;
  246. # a setup save during an awaited handshake needs another scan.
  247. scan_signature = profile_serve_signature(current[name])
  248. if name in changed:
  249. self._drop_stale_platform_verdicts(name)
  250. try:
  251. connected = await self._start_one_profile_adapters(name, current[name], claimed)
  252. except MultiplexConfigError as exc:
  253. # Runtime config refusal is a real park, not an adapter-less served profile:
  254. # withdraw the authority/tickets, release the reservation and publish the
  255. # terminal parked verdict. A control-socket rescan can explicitly retry it.
  256. logger.error("[MULTIPLEX] Profile '%s' not served: %s", name, exc)
  257. home = current.pop(name)
  258. await self._unserve_profile(name, home)
  259. park_profile(self, name, str(exc))
  260. result["parked"].append(name)
  261. config_parked.add(name)
  262. continue
  263. except Exception:
  264. logger.error("[MULTIPLEX] Failed to start adapters for profile '%s'", name, exc_info=True)
  265. connected = 0
  266. # A transient failure is not the deliberate park above: leave the signature
  267. # unacknowledged so the next reconcile retries the connect.
  268. transient_failed.add(name)
  269. else:
  270. sigs[name] = scan_signature
  271. if name in added:
  272. logger.info("[MULTIPLEX] Now serving profile '%s' (%s adapter(s) connected; %s)", name, connected, reason)
  273. result["added"].append(name)
  274. else:
  275. logger.info("[MULTIPLEX] Re-scanned profile '%s' after config/.env change (%s adapter(s) connected)", name, connected)
  276. result["rescanned"].append(name)
  277. if config_parked:
  278. added = [name for name in added if name not in config_parked]
  279. self._served_profile_signatures = sigs
  280. # A profile deleted while an adapter above was still connecting must not be recorded back
  281. # (the deleter's signal timed out against this lock and rmtree already ran).
  282. live_now = {str(name) for name, _home in profiles_to_serve(multiplex=True)}
  283. for name in [n for n in current if n not in live_now and n != active]:
  284. await self._unserve_profile(name, current.pop(name))
  285. result["removed"].append(name)
  286. added = [n for n in added if n != name]
  287. self._record_served_profiles(active, list(current.items()))
  288. # ``_note_served_profiles`` fills a missing signature with the current one; that refill
  289. # would park a transiently-failed profile exactly like the config-error case above.
  290. for name in transient_failed:
  291. if isinstance(self._served_profile_signatures, dict):
  292. self._served_profile_signatures.pop(name, None)
  293. # A cached config with no live adapters is owed a home-channel notice nothing can
  294. # deliver, and the planned-restart marker then never clears.
  295. configs = getattr(self, "_profile_configs", None)
  296. if isinstance(configs, dict):
  297. configs.pop(name, None)
  298. if added:
  299. from gateway.run_runtime import recover_served_native_rows
  300. await recover_served_native_rows(self)
  301. await self._after_profiles_added([(n, current[n]) for n in added])
  302. result["served_profiles"] = self.served_profile_names()
  303. return result
  304. def _drop_stale_platform_verdicts(self, name: str) -> None:
  305. """Before a changed profile's rescan, drop its ``<name>:<platform>`` status entries for
  306. platforms with no live or queued adapter: the rescan re-publishes every verdict that still
  307. holds, so a fatal/disabled entry about a credential the user just removed can't outlive it."""
  308. from gateway.run import _write_runtime_status_quiet
  309. live = set((getattr(self, "_profile_adapters", None) or {}).get(name) or {})
  310. live |= set((getattr(self, "_profile_failed_platforms", None) or {}).get(name) or {})
  311. noted = getattr(self, "_unserved_secondary_platforms", None)
  312. if isinstance(noted, set):
  313. noted -= {entry for entry in noted if entry[0] == name}
  314. _write_runtime_status_quiet(drop_profile_platforms=name,
  315. keep_platforms={f"{name}:{getattr(p, 'value', p)}" for p in live})
  316. def _parked_profile_names(self) -> list:
  317. """Profiles that exist but could not be served (unusable store, home owned elsewhere); boot's
  318. ``initialize_gateway_runtime`` parks into the same published ``name -> reason`` map."""
  319. from gateway.run_runtime import parked_profile_map
  320. return list(parked_profile_map(self))
  321. async def _multiplex_for_added_profile(self) -> bool:
  322. """Start multiplexing in place when this gateway booted standalone ONLY because the host had
  323. one profile and a second now exists (#141623). Boot settles the mode once, so without this a
  324. profile created afterwards (Desktop's New Agent, ``hermes profile create``) was never served:
  325. every ticket, session and delete for it was refused until a manual restart.
  326. No restart and no drain: the launch env is frozen and unscoped secret reads fail closed
  327. (``activate_multi_profile_hosting``, the hosted-room precedent), the launch home becomes the
  328. first entry of the reservation, and the caller's ordinary hot-serve reconcile serves the new
  329. profile. Turns already running keep their bound scope; the launch profile's later work binds
  330. its own scope through ``_standalone_launch_scope`` and the re-wired primary handlers. Any
  331. other standalone verdict, or a preflight blocker now, leaves the gateway as it is; nothing is
  332. latched, so the next rescan or watcher tick retries once the blocker clears. The switch is an
  333. automatic migration, so ``gateway.auto_multiplex_migration: false`` holds it back the same way
  334. (re-read on every rescan: setting it back to true converges without a restart). True when armed."""
  335. from hermes_cli.gateway_multiplex_mode import (
  336. AUTO_MIGRATION_OPTED_OUT_REASON, SINGLE_PROFILE_REASON, MultiplexDecision, log_multiplex_decision,
  337. record_multiplex_decision, recorded_standalone_reason)
  338. from hermes_cli.profiles import profiles_to_serve
  339. from hermes_constants import get_default_hermes_root, get_process_hermes_home
  340. # Serialized with reconcile: a second rescan (watcher + creator) resuming mid-activation
  341. # reset the reservation the first one's reconcile had already grown.
  342. async with self._reconcile_lock():
  343. if self._multiplex_on():
  344. return True # a concurrent rescan activated while this one waited
  345. if not self._running or getattr(self, "_draining", False):
  346. return False # shutdown began: never arm fail-closed secrets under drain-time work
  347. if recorded_standalone_reason() not in (SINGLE_PROFILE_REASON, AUTO_MIGRATION_OPTED_OUT_REASON):
  348. return False
  349. launch = get_process_hermes_home().resolve()
  350. if launch != get_default_hermes_root().resolve():
  351. return False # only the default home runs the multiplexer (``_launch_home_may_multiplex``)
  352. if len(profiles_to_serve(multiplex=True, include_parked=True)) < 2:
  353. return False
  354. blocker = await self._run_housekeeping_in_executor(_added_profile_blocker, launch)
  355. if blocker is not None:
  356. if blocker != self._single_profile_blocker_logged:
  357. logger.warning("[MULTIPLEX] A new profile exists but this gateway stays standalone: %s", blocker)
  358. self._single_profile_blocker_logged = blocker
  359. if blocker == AUTO_MIGRATION_OPTED_OUT_REASON:
  360. # Published so `gateway ensure` refuses the new profile naming the opt-out (Desktop shows
  361. # that detail) instead of "only one profile exists".
  362. record_multiplex_decision(MultiplexDecision(False, "guard", blocker))
  363. return False
  364. if not self._running or getattr(self, "_draining", False):
  365. return False # shutdown began during the preflight await
  366. from hermes_constants import hermes_home_key
  367. from tools.mcp_tool_scope import rekey_unscoped_connections
  368. from tui_gateway.launch_profile_policy import activate_multi_profile_hosting
  369. activate_multi_profile_hosting()
  370. rekey_unscoped_connections(hermes_home_key(launch))
  371. self.config.multiplex_profiles = True
  372. self.config._runtime_profile_homes = (("default", launch),)
  373. registry = getattr(self, "session_authorities", None)
  374. if registry is not None:
  375. registry.multiplexed = True
  376. for adapter in list(self.adapters.values()):
  377. adapter.set_message_handler(self._primary_message_handler())
  378. adapter.set_busy_session_handler(self._primary_busy_session_handler())
  379. adapter.set_platform_event_handler(self._primary_platform_event_handler())
  380. self._record_served_profiles("default", [("default", launch)])
  381. log_multiplex_decision(MultiplexDecision(True, "default", "a second profile was created"))
  382. logger.info("[MULTIPLEX] A second profile exists; serving every profile from this process (no restart)")
  383. return True
  384. async def _serve_profile_runtime(self, name: str, home: Path) -> Optional[str]:
  385. """Grow the reservation by *home* and build its session authority (boot's per-secondary
  386. steps). Returns the park reason — reservation released — when another gateway owns the home
  387. (a stray per-profile daemon) or its store cannot be opened; None when served. An
  388. adapters-only runner (no authority registry) grows the reservation alone."""
  389. from gateway.run_runtime import release_profile_home, reserve_profile_home_in_worker, serve_profile_runtime
  390. from gateway.runtime_ownership import OwnershipConflict
  391. try:
  392. await reserve_profile_home_in_worker(self, name, home)
  393. except (OwnershipConflict, OSError) as exc:
  394. logger.error("[MULTIPLEX] Profile '%s' not served: %s", name, exc)
  395. return f"home owned by another gateway: {exc}"
  396. if getattr(self, "session_authorities", None) is None:
  397. return None
  398. try:
  399. await serve_profile_runtime(self, name, home)
  400. except (Exception, ConfigBackendUnavailable) as exc: # unservable now: parked, retried by rescan
  401. if self.session_authorities.for_home(home) is not None:
  402. # Retirement of the failed attempt missed its deadline: its writer may still use
  403. # this home, so the reservation and authority stay; the profile is parked (no
  404. # adapters) and the batch goes on for every other profile.
  405. logger.error("[MULTIPLEX] Profile '%s' not served (%s); a turn outlived its Stop, "
  406. "ownership retained", name, exc, exc_info=True)
  407. return f"stopping after a failed serve; ownership retained: {exc}"
  408. logger.error("[MULTIPLEX] Profile '%s' not served: its session store is unusable (%s): %s",
  409. name, home, exc, exc_info=True)
  410. release_profile_home(self, home)
  411. return f"session store unusable: {exc}"
  412. return None
  413. def _live_resource_claims(self, active: str) -> dict[tuple, str]:
  414. """Startup's ``claimed`` map rebuilt from what is live now: primary claims plus every connected
  415. secondary's credential/listener, so a hot-added profile reusing a token is parked, never a
  416. second poller."""
  417. claimed = self._primary_resource_claims(active)
  418. for profile_name, adapters in (getattr(self, "_profile_adapters", None) or {}).items():
  419. for platform, adapter in list(adapters.items()):
  420. for claim in self._adapter_claims(platform, adapter):
  421. if claim is not None:
  422. claimed[claim] = profile_name
  423. return claimed
  424. async def _after_profiles_added(self, profile_homes) -> None:
  425. """Per-profile startup side effects for hot-added profiles: log routing + scoped MCP discovery."""
  426. from gateway.run import _enable_multiplex_log_routing, _profile_runtime_scope
  427. from contextvars import copy_context
  428. with _log_suppressed(logging.DEBUG, "log routing refresh failed", exc_info=True):
  429. _enable_multiplex_log_routing(self.config)
  430. from tools.mcp_oauth import suppress_interactive_oauth
  431. loop = asyncio.get_running_loop()
  432. for profile_name, profile_home in profile_homes:
  433. try:
  434. from tools.mcp_tool_discovery import discover_mcp_tools
  435. with _profile_runtime_scope(Path(profile_home)), suppress_interactive_oauth():
  436. await loop.run_in_executor(None, copy_context().run, discover_mcp_tools)
  437. except Exception:
  438. logger.warning("MCP tool discovery failed for profile '%s'", profile_name, exc_info=True)
  439. async def _unserve_profile(self, name: str, home: Path) -> None:
  440. """Stop and unroute one profile: cancel its reconnects, tear down its adapters, drop its
  441. bookkeeping and release this process's handles into its home so the deleter's rmtree succeeds.
  442. The releasing process is not always the deleter (#130244): under multiplexing THIS gateway
  443. process routes the profile's logs and owns its scoped MCP servers, so both must be released
  444. here too or the deleter's rmtree still fails on ``logs/.__agent.lock`` / ``mcp-stderr.log``.
  445. The whole teardown runs inside the removed profile's own runtime scope: adapter disconnect
  446. hooks, the agent-cache eviction (provider/memory shutdown) and the state/memory handle
  447. release all read config and credentials at call time, and this coroutine runs on the
  448. reconcile task with no profile bound — unscoped they resolved against the LAUNCH home, so a
  449. teardown hook needing this profile's credential failed closed (or, worse, borrowed the launch
  450. profile's). Secrets are not re-hydrated: teardown must not block the loop on a source fetch.
  451. """
  452. from gateway.run import _profile_runtime_scope, _write_runtime_status_quiet
  453. from gateway.run_runtime import release_profile_home, unserve_profile_runtime
  454. # Retire before teardown; a False verdict (writer outlived its Stop) is honoured below.
  455. retired = getattr(self, 'session_authorities', None) is None or await unserve_profile_runtime(self, home)
  456. pending = (getattr(self, "_profile_failed_platforms", None) or {}).pop(name, None) or {}
  457. tasks = [t for t in pending.values() if isinstance(t, asyncio.Task) and not t.done()]
  458. # Its adapters stay in the map (live owners of their bots) while the reconnects cancel; the retiring
  459. # mark keeps anything finishing meanwhile (a fatal teardown, a startup handoff) from queuing another.
  460. retiring = getattr(self, "_retiring_profiles", None)
  461. if retiring is None:
  462. retiring = self._retiring_profiles = set()
  463. retiring.add(name)
  464. try:
  465. for task in tasks:
  466. task.cancel()
  467. if tasks:
  468. await asyncio.wait(tasks, timeout=self._adapter_disconnect_timeout_secs())
  469. finally:
  470. retiring.discard(name)
  471. with _profile_runtime_scope(Path(home), hydrate_secrets=False):
  472. adapters = (getattr(self, "_profile_adapters", None) or {}).pop(name, None) or {}
  473. for platform, adapter in adapters.items(): # no await since the pop: each holds its bot until torn down
  474. self._hold_secondary_claims(name, platform, adapter, self._adapter_claims(platform, adapter))
  475. try:
  476. for platform, adapter in list(adapters.items()):
  477. await self._bounded_adapter_teardown(adapter, platform, profile=name)
  478. self._release_secondary_claims(name, platform, adapter) # its bot is free once it is down
  479. finally:
  480. # Every hold of the profile goes, a fatal teardown's still disconnecting included (the profile
  481. # is gone, so its handler queues nothing): none may block a re-serve.
  482. held = getattr(self, "_secondary_held_claims", None) or {}
  483. for key in [key for key in held if key[0] == name]:
  484. del held[key]
  485. # Its ``<name>:<platform>`` runtime entries describe a profile that no longer exists.
  486. _write_runtime_status_quiet(drop_profile_platforms=name)
  487. for attr in ("pairing_stores", "_busy_text_modes_by_profile", "_busy_input_modes_by_profile",
  488. "_busy_text_timing_by_profile", "_human_delay_by_profile", "_profile_configs",
  489. "_profile_bot_platforms"):
  490. store = getattr(self, attr, None)
  491. if isinstance(store, dict):
  492. store.pop(name, None)
  493. if isinstance(self._served_profile_homes, dict):
  494. self._served_profile_homes.pop(name, None)
  495. if isinstance(self._served_profile_signatures, dict):
  496. self._served_profile_signatures.pop(name, None)
  497. from gateway.session import _session_key_namespace
  498. prefix = _session_key_namespace(name) + ":"
  499. cache = getattr(self, "_agent_cache", None)
  500. for key in [k for k in list(cache or {}) if str(k).startswith(prefix)]:
  501. with _log_suppressed(logging.DEBUG, "agent eviction failed for %s", key, exc_info=True):
  502. self._evict_cached_agent(key)
  503. if not retired:
  504. # A writer outlived its Stop and may still write this home: the profile leaves the
  505. # served set (ingress is gone above) but keeps its reservation and store handles until
  506. # this process exits. Raising here aborted the reconcile for every other profile.
  507. logger.error("[MULTIPLEX] Profile '%s' unrouted, but a turn outlived its Stop; ownership retained", name)
  508. return
  509. # The reservation (gateway.lock) is released LAST: a deleter waiting on it may rmtree the
  510. # home as soon as it goes, so every handle this process holds there must close first.
  511. try:
  512. with _log_suppressed(logging.DEBUG, "profile handle release failed", exc_info=True):
  513. from hermes_state_registry import close_all_under
  514. close_all_under(home)
  515. with _log_suppressed(logging.DEBUG, "memory-store release failed", exc_info=True):
  516. from plugins.memory import import_provider_module # catalog plugin since it left core
  517. import_provider_module("holographic", "store").MemoryStore.release_all_under(home)
  518. # The gateway process is a second holder the CLI-side delete cannot reach: its routed
  519. # log files (logs/.__agent.lock, logs/.__errors.lock) and its scoped MCP servers'
  520. # mcp-stderr.log handle live HERE, so unserve must release them in this process too
  521. # (#130244) — the same two calls ``delete_profile`` makes for the deleting process.
  522. # Both calls BLOCK (the log release stops/joins the QueueListener, which can be inside
  523. # a ConcurrentRotatingFileHandler emit waiting on another process's rotation lock; the
  524. # MCP shutdown waits on future.result for the MCP loop), so they run off the loop in a
  525. # worker thread — otherwise every other served profile's events stall behind them.
  526. # ``asyncio.to_thread`` copies the context, so the surrounding
  527. # ``_profile_runtime_scope`` home/secret override stays visible in the worker.
  528. with _log_suppressed(logging.DEBUG, "scoped MCP shutdown failed", exc_info=True):
  529. from hermes_constants import hermes_home_key
  530. from tools.mcp_tool_lifecycle import shutdown_mcp_servers
  531. await asyncio.to_thread(shutdown_mcp_servers, scope=hermes_home_key(home))
  532. with _log_suppressed(logging.DEBUG, "profile log handler release failed", exc_info=True):
  533. from hermes_logging import release_profile_log_handlers
  534. await asyncio.to_thread(release_profile_log_handlers, home)
  535. finally:
  536. release_profile_home(self, home)
  537. logger.info("[MULTIPLEX] Profile '%s' unserved — %d adapter(s) stopped and unrouted", name, len(adapters))
  538. def _profile_lifecycle_verb(runner, *, serve: bool):
  539. """Build on the owning loop; socket handlers themselves run on executor threads."""
  540. loop = asyncio.get_running_loop()
  541. async def apply(name):
  542. from hermes_cli.profiles import profiles_to_serve, profile_is_parked
  543. if not runner._multiplex_on() or not runner._running or runner._served_profile_homes is None:
  544. return {"error": "host multiplexer is not ready"}
  545. async with runner._reconcile_lock():
  546. active = getattr(runner, "_primary_profile_name", None) or "default"
  547. if not isinstance(name, str) or not name or name == active:
  548. return {"error": "a non-launch profile name is required"}
  549. known = dict(runner._served_profile_homes)
  550. if not serve:
  551. if name not in known:
  552. return {"error": f"profile '{name}' is not served"}
  553. await runner._unserve_profile(name, known.pop(name))
  554. runner._record_served_profiles(active, list(known.items()))
  555. return {"unserved": name, "served_profiles": runner.served_profile_names()}
  556. installed = dict(profiles_to_serve(True, include_parked=True))
  557. if name not in installed:
  558. return {"error": f"unknown profile '{name}'"}
  559. if name != "default" and profile_is_parked(installed[name]):
  560. return {"error": f"profile '{name}' is parked (gateway.parked)"}
  561. if name in known:
  562. return {"error": f"profile '{name}' is already served"}
  563. known[name] = installed[name]
  564. result = await runner._apply_profile_changes(known, [name], [], [], reason="control-socket")
  565. if name not in result["served_profiles"]:
  566. return {"error": f"profile '{name}' was removed or parked during startup"}
  567. return {"served": name, "served_profiles": result["served_profiles"]}
  568. def handler(params):
  569. future = asyncio.run_coroutine_threadsafe(apply(params.get("name")), loop)
  570. try:
  571. return future.result(timeout=5.0)
  572. except TimeoutError:
  573. # Keep running: the client must not interpret an incomplete teardown as stopped.
  574. return {"pending": True, "served_profiles": runner.served_profile_names()}
  575. except Exception as exc:
  576. logger.warning("Profile lifecycle request failed", exc_info=True)
  577. return {"error": f"{type(exc).__name__}: {exc}"}
  578. return handler
  579. def unserve_profile_verb(runner):
  580. return _profile_lifecycle_verb(runner, serve=False)
  581. def serve_profile_verb(runner):
  582. return _profile_lifecycle_verb(runner, serve=True)
  583. def _mcp_config_reconciler(runner=None):
  584. """Housekeeping chore keeping live MCP servers in step with ``mcp_servers`` on disk, every tick
  585. after the first (startup discovery owns that one). Reconciling on DRIFT rather than only on a
  586. config EDIT is what brings back a server whose FIRST connect failed (#112445): it never reached
  587. ``_servers``, so the parked self-probe — a property of a task that connected once — cannot revive
  588. it, and its config never changes. The reconcile is a cached config read plus set compares when
  589. nothing moved; a server dropped from config is torn down (a parked one otherwise self-probes
  590. every ``_PARKED_RETRY_INTERVAL`` for the life of the process) and a missing one is reconnected
  591. only once its per-server connect cooldown (30s→600s backoff) has lapsed, so a chronically failing
  592. server is retried on that schedule, not every tick. Interactive OAuth is suppressed — this runs
  593. on a housekeeping thread nobody is watching."""
  594. primed: set = set()
  595. def _reconcile_current(label: str) -> None:
  596. from tools.mcp_oauth import suppress_interactive_oauth
  597. from tools.mcp_tool_discovery import reconcile_mcp_servers_with_config
  598. if label not in primed:
  599. primed.add(label)
  600. return # first tick: startup discovery already reflects this config (or is still running)
  601. with suppress_interactive_oauth():
  602. result = reconcile_mcp_servers_with_config()
  603. if result["removed"] or result["added"]:
  604. logger.info("MCP servers reconciled with config (%s): removed=%s added=%s",
  605. label, result["removed"], result["added"])
  606. return lambda: _for_each_served_profile(runner, _reconcile_current)
  607. def _for_each_served_profile(runner, body) -> None:
  608. """Run ``body(profile_label)`` once per served profile, inside that profile's runtime scope.
  609. Housekeeping runs on a bare thread with no turn on the stack, so nothing binds a profile for it:
  610. ``get_hermes_home()`` and ``get_secret()`` see the LAUNCH profile's values, and under
  611. ``gateway.multiplex_profiles`` a fail-closed credential read logs ``no profile secret scope on a
  612. multiplexed call`` on every tick (the skills-sync pulls resolved Nous credentials this way, four
  613. WARNINGs per hourly tick per chore). A single-profile gateway runs ``body`` once, unscoped:
  614. there the process env IS the profile's own value — unless a hosted room already flipped the
  615. process-wide guard (#112878), in which case the launch profile's OWN scope is bound, as
  616. ``run_turn.py::_standalone_launch_scope`` does for turns."""
  617. from gateway.run import _multiplex_profile_homes, _profile_runtime_scope
  618. config = getattr(runner, "config", None)
  619. if not getattr(config, "multiplex_profiles", False):
  620. from gateway.run_turn import GatewayTurnMixin
  621. with GatewayTurnMixin._standalone_launch_scope():
  622. body("default")
  623. return
  624. for profile_name, profile_home in _multiplex_profile_homes(config):
  625. # One boundary per profile: callers (``_housekeeping_chore``) catch only at the tick level,
  626. # so one profile's unreadable store or broken .env abandoned every profile after it, on
  627. # every tick. The launch store failing is reachable (``_init_session_db`` tolerates it and
  628. # keeps running), and serve defers every served profile's sweep to this loop.
  629. try:
  630. with _profile_runtime_scope(Path(profile_home)):
  631. body(str(profile_name))
  632. except Exception as exc:
  633. logger.debug("Housekeeping for profile %s skipped: %s", profile_name, exc)
  634. def profile_scoped_chore(runner, chore):
  635. """Wrap a zero-arg housekeeping chore that reads the profile's home, config or credentials so it
  636. runs once per served profile under that profile's scope (see ``_for_each_served_profile``)."""
  637. return lambda: _for_each_served_profile(runner, lambda _label: chore())
  638. def migrate_profile_identity_verb(runner):
  639. """Build the ``migrate-profile-identity`` control-verb handler for ``hermes profile rename``
  640. (#111926). The live multiplexer owns the routing index in memory and writes it back
  641. periodically, so a CLI-side rewrite of ``agent:<old>:*`` would be clobbered on the next save;
  642. the CLI therefore asks this process to rekey both durable stores AND ``SessionStore._entries``.
  643. Runs on the control-socket executor thread; ``rekey_profile_routing`` takes the store lock."""
  644. def _handler(params: dict) -> dict:
  645. old, new = str(params.get("old") or "").strip(), str(params.get("new") or "").strip()
  646. if not old or not new or old == new:
  647. return {"ok": False, "error": "old/new required and must differ"}
  648. store = getattr(runner, "session_store", None)
  649. if store is None:
  650. return {"ok": False, "error": "live gateway has no session store"}
  651. acquired = []
  652. try:
  653. from hermes_state_registry import acquire, release_or_close
  654. db_counts: dict[str, dict[str, int]] = {}
  655. routing_db = getattr(store, "_routing_db", None)
  656. if routing_db is not None and hasattr(routing_db, "rekey_profile_state"):
  657. db_counts["routing"] = routing_db.rekey_profile_state(old, new)
  658. routing_home = getattr(store, "_routing_home", None)
  659. profile_path = Path(routing_home) / "profiles" / new / "state.db" if routing_home else None
  660. if profile_path is not None and profile_path.exists():
  661. profile_db = acquire(profile_path)
  662. acquired.append(profile_db)
  663. db_counts["profile"] = profile_db.rekey_profile_state(old, new)
  664. rekeyed = store.rekey_profile_routing(old, new)
  665. return {"ok": True, "rekeyed": rekeyed, "db": db_counts}
  666. except Exception as exc:
  667. logger.warning("Profile identity migration failed for %r->%r: %s", old, new, exc)
  668. return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
  669. finally:
  670. for db in acquired:
  671. try:
  672. release_or_close(db)
  673. except Exception:
  674. logger.debug("Failed to release renamed profile state DB", exc_info=True)
  675. return _handler
  676. def purge_profile_identity_verb(runner):
  677. """Build the ``purge-profile-identity`` control-verb handler for ``hermes profile delete``
  678. (#111926, delete side). The live multiplexer owns the routing index in memory and writes it back
  679. periodically, so a CLI-side DELETE of ``agent:<name>:*`` rows would be undone by its next save;
  680. the CLI therefore asks this process to drop the durable rows AND ``SessionStore._entries``.
  681. Deliberately NOT part of ``_unserve_profile()``: that path also unserves names that are still
  682. alive elsewhere in the identity story — a rename's old name leaves the served set exactly like a
  683. delete does (its directory is gone either way) — and purging there would race the rekey it is
  684. supposed to leave intact. Only the delete path invokes this verb. Runs on the control-socket
  685. executor thread; ``purge_profile_routing`` takes the store lock."""
  686. def _handler(params: dict) -> dict:
  687. name = str(params.get("name") or "").strip()
  688. if not name:
  689. return {"ok": False, "error": "name required"}
  690. store = getattr(runner, "session_store", None)
  691. if store is None:
  692. return {"ok": False, "error": "live gateway has no session store"}
  693. try:
  694. db_counts: dict[str, dict[str, int]] = {}
  695. routing_db = getattr(store, "_routing_db", None)
  696. if routing_db is not None and hasattr(routing_db, "purge_profile_state"):
  697. db_counts["routing"] = routing_db.purge_profile_state(name)
  698. dropped = store.purge_profile_routing(name)
  699. return {"ok": True, "dropped": dropped, "db": db_counts}
  700. except Exception as exc:
  701. logger.warning("Profile identity purge failed for %r: %s", name, exc)
  702. return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
  703. return _handler