web_server_gateway.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776
  1. """Gateway/process helpers for the dashboard: per-profile gateway topology (+cache), action
  2. subprocess spawning, gateway restart plumbing, system platform display.
  3. """
  4. import logging
  5. import json
  6. import os
  7. import re
  8. import subprocess
  9. import sys
  10. import threading
  11. import time
  12. import urllib.request
  13. from pathlib import Path
  14. from typing import Any, Dict, List, Optional, Tuple
  15. from hermes_cli._subprocess_compat import windows_detach_flags, windows_detach_flags_without_breakaway
  16. from hermes_cli.config import get_hermes_home
  17. # Same logger the code used before extraction (record parity).
  18. _log = logging.getLogger("hermes_cli.web_server")
  19. # Defense against a hostile/compromised upstream serving an unbounded body to
  20. # the dashboard's JSON readers: refuse to buffer past this size.
  21. _DASHBOARD_JSON_RESPONSE_BODY_MAX_BYTES = 1024 * 1024
  22. def _read_dashboard_json_response(response) -> Any:
  23. """Read a JSON HTTP response body with a hard size cap (issue #54808)."""
  24. raw = response.read(_DASHBOARD_JSON_RESPONSE_BODY_MAX_BYTES + 1)
  25. if len(raw) > _DASHBOARD_JSON_RESPONSE_BODY_MAX_BYTES:
  26. raise ValueError(
  27. "dashboard HTTP JSON response exceeded "
  28. f"{_DASHBOARD_JSON_RESPONSE_BODY_MAX_BYTES} bytes"
  29. )
  30. return json.loads(raw.decode("utf-8"))
  31. def _probe_gateway_health() -> tuple[bool, dict | None]:
  32. """Probe the gateway's HTTP health endpoint (cross-container). Blocking — run in an executor.
  33. DEPRECATED: driven by the ``GATEWAY_HEALTH_URL`` / ``GATEWAY_HEALTH_TIMEOUT`` env vars,
  34. to be replaced by a dashboard config key; do not add callers. Accepts a base URL or an
  35. explicit ``/health`` / ``/health/detailed`` path; tries ``/health/detailed`` first.
  36. """
  37. from hermes_cli.web_server import _GATEWAY_HEALTH_TIMEOUT, _GATEWAY_HEALTH_URL
  38. if not _GATEWAY_HEALTH_URL:
  39. return False, None
  40. base = re.sub(r"/health(/detailed)?$", "", _GATEWAY_HEALTH_URL.rstrip("/"))
  41. # Bearer credential for an auth-protected gateway (issue #76051): sent to the
  42. # authenticated /health/detailed probe ONLY — never to the public /health fallback,
  43. # which is unauthenticated by design.
  44. api_key = os.environ.get("API_SERVER_KEY", "").strip()
  45. for path in (f"{base}/health/detailed", f"{base}/health"):
  46. try:
  47. req = urllib.request.Request(path, method="GET")
  48. if api_key and path.endswith("/health/detailed"):
  49. req.add_header("Authorization", f"Bearer {api_key}")
  50. with urllib.request.urlopen(req, timeout=_GATEWAY_HEALTH_TIMEOUT) as resp:
  51. if resp.status == 200:
  52. return True, _read_dashboard_json_response(resp)
  53. except Exception:
  54. continue
  55. return False, None
  56. # ``platform-name -> (config port key, adapter default)`` for port-binding gateway platforms.
  57. # Mirrors PORT_BINDING_PLATFORM_VALUES (gateway/config.py) and each adapter's DEFAULT_PORT /
  58. # DEFAULT_WEBHOOK_PORT. Display-only data for the topology readout, not a bind source.
  59. _PORT_BINDING_PLATFORM_PORTS: dict[str, tuple[str, int]] = {
  60. "webhook": ("port", 8644), "api_server": ("port", 8642), "msgraph_webhook": ("port", 8646),
  61. "feishu": ("webhook_port", 8765), "wecom_callback": ("port", 8645), "bluebubbles": ("webhook_port", 8645),
  62. "sms": ("webhook_port", 8080), "whatsapp_cloud": ("webhook_port", 8090), "line": ("port", 8646),
  63. "teams": ("port", 3978),
  64. }
  65. # Platform states that mean the adapter is NOT serving its port right now.
  66. _PLATFORM_DEAD_STATES = frozenset({"fatal", "disconnected", "stopped"})
  67. def _profile_platform_ports(profile_home: Path, runtime: Optional[dict]) -> dict[str, int]:
  68. """Best-effort ``platform -> host TCP port`` for one profile's live gateway.
  69. Ports come from the profile's own config.yaml (``gateway.platforms`` then top-level
  70. ``platforms`` — later wins, matching load_gateway_config precedence), falling back to the
  71. adapter default. Env-var overrides (e.g. WEBHOOK_PORT in that profile's .env) are not resolved.
  72. """
  73. platforms = (runtime or {}).get("platforms") or {}
  74. active = [
  75. name for name, state in platforms.items()
  76. if name in _PORT_BINDING_PLATFORM_PORTS
  77. and isinstance(state, dict)
  78. and state.get("state") not in _PLATFORM_DEAD_STATES]
  79. if not active:
  80. return {}
  81. blocks: dict[str, dict] = {}
  82. try:
  83. # load_config() targets the ACTIVE profile's home; read the probed profile's file raw.
  84. from hermes_cli.config import read_user_config_raw
  85. cfg = read_user_config_raw(profile_home / "config.yaml")
  86. gateway_cfg = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {}
  87. for src in ((gateway_cfg or {}).get("platforms"), cfg.get("platforms")):
  88. if not isinstance(src, dict):
  89. continue
  90. for plat_name, plat_block in src.items():
  91. if isinstance(plat_block, dict):
  92. blocks.setdefault(plat_name, {}).update(plat_block)
  93. except Exception:
  94. blocks = {}
  95. ports: dict[str, int] = {}
  96. for name in active:
  97. port_key, default_port = _PORT_BINDING_PLATFORM_PORTS[name]
  98. block = blocks.get(name) or {}
  99. extra = block.get("extra") if isinstance(block.get("extra"), dict) else {}
  100. raw = block.get(port_key, (extra or {}).get(port_key, default_port))
  101. try:
  102. ports[name] = int(raw)
  103. except (TypeError, ValueError):
  104. ports[name] = default_port
  105. return ports
  106. def _profile_gateway_writer_identity(profile_home: Path, runtime: Optional[dict]) -> Optional[tuple]:
  107. """``(pid, start_time)`` of the profile's LIVE gateway, or None.
  108. Uses the same validated-liveness helper and the same ``_get_process_start_time`` that stamped
  109. the record, so equality is exact (no unit/clock-source mismatch).
  110. """
  111. try:
  112. from gateway.status import _get_process_start_time, get_runtime_status_running_pid
  113. pid = get_runtime_status_running_pid(runtime, expected_home=profile_home)
  114. if pid is None:
  115. return None
  116. start_time = _get_process_start_time(pid)
  117. return None if start_time is None else (pid, start_time)
  118. except Exception:
  119. return None
  120. def _owned_profile_platforms(writer_identity: Optional[tuple], platforms: dict) -> dict:
  121. """Keep only platform entries stamped by the profile's CURRENT gateway process.
  122. Gateway startup preserves plain platform entries in gateway_state.json across restarts, so the
  123. raw map can carry fatal state for platforms since disabled/removed. Cross-profile aggregation
  124. has no config context to filter against, so it demands exact ``(pid, start_time)`` writer
  125. identity instead. Fail closed: legacy entries without identity, or no live process, yield {} —
  126. a false "degraded forever" is the worse failure mode.
  127. """
  128. if writer_identity is None:
  129. return {}
  130. live_pid, live_start = writer_identity
  131. return {
  132. key: value for key, value in platforms.items()
  133. if isinstance(value, dict)
  134. and value.get("writer_pid") == live_pid
  135. and value.get("writer_start_time") == live_start}
  136. def _collect_profile_gateway_topology() -> dict[str, Any]:
  137. """Enumerate profiles and the gateways serving them for ``/api/status``.
  138. Returns ``profiles`` (all profile names via the cheap ``profiles_to_serve(True)`` chokepoint),
  139. ``gateways`` (one ``{"profile", "ports", "served_profiles"?}`` per LIVE gateway; liveness via
  140. ``_check_gateway_running`` so it agrees with the sidebar), ``gateway_mode``
  141. (multiplex / single / multiple / none) and ``profile_platforms`` — ownership-filtered runtime
  142. platform maps per live gateway, an internal aggregation input never exposed directly.
  143. """
  144. try:
  145. from hermes_cli.profiles import _check_gateway_running, profiles_to_serve, profile_is_parked
  146. from gateway.status import read_runtime_status
  147. homes = profiles_to_serve(True, include_standalone=True, include_parked=True)
  148. except Exception:
  149. _log.debug("profile/gateway topology enumeration failed", exc_info=True)
  150. return {"profiles": [], "gateway_mode": "unknown", "gateways": [], "profile_platforms": {}}
  151. gateways: list[dict[str, Any]] = []
  152. profile_platforms: dict[str, dict] = {}
  153. multiplex = False
  154. standalone_reason: Optional[str] = None
  155. for name, home in homes:
  156. try:
  157. # A served profile's liveness is the multiplexer's: listing it here showed one phantom
  158. # gateway per served profile beside the host.
  159. if not (_check_gateway_running(home) if name == "default" else _has_own_gateway(home)):
  160. continue
  161. except Exception:
  162. continue
  163. try:
  164. runtime = read_runtime_status(home / "gateway_state.json")
  165. except Exception:
  166. runtime = None
  167. served = [str(p) for p in ((runtime or {}).get("served_profiles") or [])]
  168. if name == "default" and len(served) > 1:
  169. multiplex = True
  170. if (runtime or {}).get("multiplex_standalone_reason"):
  171. standalone_reason = str(runtime["multiplex_standalone_reason"])
  172. plats = (runtime or {}).get("platforms")
  173. owned: dict = {}
  174. if isinstance(plats, dict) and plats:
  175. owned = _owned_profile_platforms(_profile_gateway_writer_identity(home, runtime), plats)
  176. if owned:
  177. profile_platforms[name] = owned
  178. # Ports from the OWNED entries too: a platform entry a previous process left "connected"
  179. # reported a port the live gateway does not bind.
  180. entry: dict[str, Any] = {"profile": name, "ports": _profile_platform_ports(home, {"platforms": owned})}
  181. if served:
  182. entry["served_profiles"] = served
  183. gateways.append(entry)
  184. if multiplex:
  185. mode = "multiplex"
  186. else:
  187. mode = {0: "none", 1: "single"}.get(len(gateways), "multiple")
  188. # A guard refusal on a multi-profile host is what the dashboard banner shows; a single-profile
  189. # install has nothing unserved and gets no banner.
  190. from hermes_cli.gateway_multiplex_mode import SINGLE_PROFILE_REASON
  191. if standalone_reason == SINGLE_PROFILE_REASON or len(homes) < 2:
  192. standalone_reason = None
  193. return {
  194. "profiles": [name for name, _home in homes],
  195. "parked_profiles": [name for name, home in homes if name != "default" and profile_is_parked(home)],
  196. "gateway_mode": mode,
  197. "multiplex_standalone_reason": standalone_reason,
  198. "gateways": gateways,
  199. "profile_platforms": profile_platforms}
  200. # /api/status is polled ~1/s by the desktop app while it waits for the backend. Each uncached
  201. # collect walks 7+ profile homes (pure-Python yaml + psutil + realpath) in the default executor;
  202. # concurrent polls pile up and hold the GIL for 14-16s, starving the loop so the desktop WS never
  203. # gets gateway.ready. A short TTL cache with a collapse lock keeps the scan to one per window.
  204. # The cache remembers which collector produced the entry: tests monkeypatch
  205. # _collect_profile_gateway_topology per case, and a swapped collector is a miss (no reset hook).
  206. _TOPOLOGY_CACHE: dict[str, Any] = {"ts": 0.0, "data": None, "fn": None}
  207. _TOPOLOGY_CACHE_LOCK = threading.Lock()
  208. _TOPOLOGY_CACHE_TTL = 10.0
  209. def _topology_cache_get(fn: Any) -> Optional[dict[str, Any]]:
  210. c = _TOPOLOGY_CACHE
  211. fresh = c["fn"] is fn and time.monotonic() - c["ts"] < _TOPOLOGY_CACHE_TTL
  212. return c["data"] if fresh and c["data"] is not None else None
  213. def _collect_profile_gateway_topology_cached() -> dict[str, Any]:
  214. fn = _collect_profile_gateway_topology
  215. cached = _topology_cache_get(fn)
  216. if cached is not None:
  217. return cached
  218. with _TOPOLOGY_CACHE_LOCK:
  219. cached = _topology_cache_get(fn)
  220. if cached is not None:
  221. return cached
  222. data = fn()
  223. _TOPOLOGY_CACHE.update(data=data, fn=fn, ts=time.monotonic())
  224. return data
  225. def _load_configured_gateway_platforms() -> set[str]:
  226. """Connected platform names; synchronous by design — the first ``load_gateway_config()`` does
  227. platform discovery and can outlast Desktop's WS connect timeout on Windows, so ``get_status``
  228. runs this in Starlette's worker pool."""
  229. from gateway.config import load_gateway_config
  230. return {platform.value for platform in load_gateway_config().get_connected_platforms()}
  231. _WINDOWS_11_MIN_BUILD = 22000
  232. def _windows_build_number(version: str, platform_label: str) -> Optional[int]:
  233. """Extract the Windows NT build number from stdlib platform strings."""
  234. for value in (version or "", platform_label or ""):
  235. match = re.search(r"(?:^|[^\d])10\.0\.(\d{5,})(?:[^\d]|$)", value)
  236. if match:
  237. return int(match.group(1))
  238. return None
  239. def _display_system_platform(*, system: str, release: str, version: str, platform_label: str) -> dict[str, str]:
  240. """Host OS fields for display; Windows 10 builds >= 22000 are relabelled Windows 11."""
  241. if system == "Windows" and release == "10":
  242. build = _windows_build_number(version, platform_label)
  243. if build is not None and build >= _WINDOWS_11_MIN_BUILD:
  244. platform_label = re.sub(r"^Windows-10(?=-)", "Windows-11", platform_label, count=1)
  245. release = "11"
  246. return {"os": system, "os_release": release, "os_version": version, "platform": platform_label}
  247. # Gateway + update actions (invoked from the Status page). Spawned detached so the request
  248. # returns immediately; stdin is DEVNULL so stray input() fails fast; stdout/stderr stream to
  249. # ~/.hermes/logs/<action>.log which the dashboard tails.
  250. _ACTION_LOG_DIR: Path = get_hermes_home() / "logs"
  251. # Short ``name`` (from the URL) → log file name under _ACTION_LOG_DIR.
  252. _ACTION_LOG_FILES: dict[str, str] = {
  253. "gateway-restart": "gateway-restart.log",
  254. "gateway-start": "gateway-start.log",
  255. "gateway-stop": "gateway-stop.log",
  256. "gateway-migrate": "gateway-migrate.log",
  257. "hermes-update": "hermes-update.log",
  258. **{name: f"action-{name}.log" for name in (
  259. "doctor", "security-audit", "backup", "import", "checkpoints-prune", "skills-install",
  260. "skills-uninstall", "skills-update", "curator-run", "prompt-size", "dump", "config-migrate",
  261. "tools-post-setup",
  262. )},
  263. }
  264. # ``name`` → most recent Popen handle / argv / action id, so ``status`` needs no ``ps``.
  265. _ACTION_PROCS: dict[str, subprocess.Popen] = {}
  266. _ACTION_COMMANDS: dict[str, tuple[str, ...]] = {}
  267. _ACTION_IDS: dict[str, str] = {}
  268. # ``name`` → synthetic result for actions handled without a subprocess (e.g. unsupported Docker updates).
  269. _ACTION_RESULTS: dict[str, dict[str, Any]] = {}
  270. # ``name`` → (pid, byte offset of that child's section in its log): a later spawn that fails after
  271. # writing its own header must not lend its text to the child that is still on record.
  272. _ACTION_LOG_STARTS: dict[str, tuple[int, int]] = {}
  273. def _terminate_desktop_managed_gateway() -> None:
  274. """Stop a live gateway restart child when its Desktop backend shuts down."""
  275. proc = _ACTION_PROCS.get("gateway-restart")
  276. if proc is None:
  277. return
  278. try:
  279. if proc.poll() is None:
  280. proc.terminate()
  281. except OSError:
  282. pass # exited between poll() and terminate()
  283. def _named_profile_from_action(subcommand: list[str]) -> Optional[str]:
  284. """Return the named-profile selector that :func:`_profile_cli_args` puts in front of an action.
  285. Deliberately inspects only the leading selector: values after the real subcommand may
  286. legitimately contain ``-p`` / ``--profile`` for a nested process (``mcp add --args ...``).
  287. """
  288. if len(subcommand) >= 2 and subcommand[0] in {"-p", "--profile"}:
  289. return str(subcommand[1]).strip() or None
  290. if subcommand and str(subcommand[0]).startswith("--profile="):
  291. return str(subcommand[0]).split("=", 1)[1].strip() or None
  292. return None
  293. def _is_host_gateway_spawn(subcommand: list[str]) -> bool:
  294. """True when *subcommand* starts the host multiplexer, not a named profile's own gateway.
  295. ``hermes gateway restart`` and ``hermes -p default gateway restart`` are the host.
  296. ``hermes -p coder gateway stop`` is not.
  297. """
  298. profile = _named_profile_from_action(subcommand)
  299. if profile not in (None, "default"):
  300. return False
  301. args = list(subcommand)
  302. if profile is not None:
  303. if args and args[0] in {"-p", "--profile"}:
  304. args = args[2:]
  305. elif args and str(args[0]).startswith("--profile="):
  306. args = args[1:]
  307. return bool(args) and args[0] == "gateway"
  308. def _profile_action_environment(
  309. subcommand: list[str], env_overrides: Optional[dict[str, str]] = None,
  310. ) -> dict[str, str]:
  311. """Environment for a detached ``hermes <subcommand>`` action.
  312. The dashboard loads its own profile's ``.env`` into process-global ``os.environ``. Copying
  313. that mapping verbatim into ``hermes -p <other> ...`` lets the named child see the dashboard
  314. profile's platform credentials and ports *before* its own dotenv loads (``load_hermes_dotenv``
  315. does not override keys already present): a supposedly A2A-only profile then claims the default
  316. Discord token and binds the default API/BlueBubbles ports.
  317. Named-profile actions therefore start from Hermes' standard scrubbed subprocess env, then drop
  318. the profile-managed keys plus every key declared by the dashboard/default profile dotenv files
  319. and their hydrated secret sources, and pin ``HERMES_HOME`` to the target profile. The child's
  320. normal startup then loads that profile's own ``.env``. A host-gateway verb (bare ``gateway``
  321. or ``-p default gateway``) starts from ``host_gateway_child_env`` so a named-profile dashboard
  322. cannot donate its dotenv to the multiplexer. Other actions without a profile selector keep
  323. the historical environment exactly.
  324. """
  325. profile = _named_profile_from_action(subcommand)
  326. if _is_host_gateway_spawn(subcommand):
  327. from tools.environments.local import host_gateway_child_env
  328. action_env = host_gateway_child_env()
  329. elif profile is None:
  330. action_env = dict(os.environ)
  331. else:
  332. from hermes_cli.env_loader import (
  333. _PROFILE_MANAGED_ENV_KEYS, _env_keys_defined_in_dotenv, get_secret_source_values,
  334. )
  335. from hermes_cli.web_server_profiles import _resolve_profile_dir
  336. from hermes_constants import apply_subprocess_home_env, get_default_hermes_root
  337. from tools.environments.local import build_subprocess_env, strip_launch_profile_env
  338. target_home = _resolve_profile_dir(profile)
  339. action_env = build_subprocess_env(base=os.environ, scrub_secrets=True)
  340. profile_keys = set(_PROFILE_MANAGED_ENV_KEYS)
  341. try:
  342. source_homes = {str(get_default_hermes_root()), str(get_hermes_home())}
  343. except Exception:
  344. source_homes = set()
  345. for source_home in source_homes:
  346. profile_keys.update(_env_keys_defined_in_dotenv(Path(source_home) / ".env"))
  347. # Secret managers contribute locally named credentials that never appear in .env;
  348. # the dashboard already hydrated its own sources, so their key names are a boundary too.
  349. profile_keys.update(get_secret_source_values(source_home).keys())
  350. for key in profile_keys:
  351. action_env.pop(key, None)
  352. # Authorization gates that reached this process outside any dotenv (unit-file
  353. # ``Environment=``, an operator export) are not in ``profile_keys``; the target
  354. # profile's ``.env`` rarely defines them, so they would survive into the child (#113270).
  355. strip_launch_profile_env(action_env, target_home)
  356. # Pin the child before import-time startup runs; the explicit -p flag stays authoritative
  357. # and resolves to the same validated directory.
  358. action_env["HERMES_HOME"] = str(target_home)
  359. apply_subprocess_home_env(action_env)
  360. action_env["HERMES_NONINTERACTIVE"] = "1"
  361. # A config.yaml allow_all_users grant bridged into os.environ must not outlive the config that
  362. # produced it: drop it so the restarted child re-derives the posture from its own config.yaml.
  363. from gateway.config_loader import drop_bridged_env
  364. drop_bridged_env(action_env)
  365. # The dashboard runs inside the gateway process, so os.environ carries _HERMES_GATEWAY=1;
  366. # inheriting it trips the child's in-process restart-loop guard (exit 1). Drop it, like
  367. # the gateway's own restart watcher does (gateway/run.py, #52470).
  368. action_env.pop("_HERMES_GATEWAY", None)
  369. if env_overrides:
  370. action_env.update(env_overrides)
  371. return action_env
  372. # Gateway lifecycle verbs the CLI refuses below root on a system-scope install
  373. # (``gateway.py::_require_root_for_system_service``).
  374. _ROOT_REQUIRING_GATEWAY_VERBS = frozenset({"restart", "start", "stop"})
  375. def _action_targets_system_gateway(subcommand: list[str]) -> bool:
  376. """True when *subcommand* is a gateway lifecycle verb that resolves to the SYSTEM unit.
  377. Scope is decided by the CLI's own picker (``_select_systemd_scope``) evaluated for the profile
  378. the action addresses, not by "a system unit exists": a host carrying both units resolves to the
  379. user unit, which the dashboard user operates unelevated. Same root/sudo posture as the
  380. ``hermes update`` fleet restart (``update_cmd_fleet._needs_sudo`` / ``_sudo_noninteractive_ok``).
  381. """
  382. from hermes_cli.update_cmd_fleet import _needs_sudo
  383. if not _needs_sudo("system"):
  384. return False
  385. try:
  386. verb = subcommand[subcommand.index("gateway") + 1]
  387. except (ValueError, IndexError):
  388. return False
  389. if verb not in _ROOT_REQUIRING_GATEWAY_VERBS:
  390. return False
  391. from hermes_cli.gateway import _select_systemd_scope
  392. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  393. profile = _named_profile_from_action(subcommand)
  394. if profile is None:
  395. return _select_systemd_scope(False)
  396. # Unit names are derived from HERMES_HOME, so a selector-bearing action must be resolved
  397. # against the TARGET profile's home (``-p default gateway restart`` from a pooled named
  398. # dashboard asks about the default unit, not about its own).
  399. from hermes_cli.web_server_profiles import _resolve_profile_dir
  400. token = set_hermes_home_override(_resolve_profile_dir(profile))
  401. try:
  402. return _select_systemd_scope(False)
  403. finally:
  404. reset_hermes_home_override(token)
  405. def _spawn_hermes_action(
  406. subcommand: list[str], name: str, *, env_overrides: Optional[dict[str, str]] = None
  407. ) -> subprocess.Popen:
  408. """Spawn ``hermes <subcommand>`` detached (via ``hermes_cli.main``) and record the handle."""
  409. from hermes_cli.web_server import PROJECT_ROOT
  410. _ACTION_LOG_DIR.mkdir(parents=True, exist_ok=True)
  411. log_file = open(_ACTION_LOG_DIR / _ACTION_LOG_FILES[name], "ab", buffering=0)
  412. section_start = os.fstat(log_file.fileno()).st_size
  413. # The header carries the action id durably: the update's completion marker lands in the ROOT
  414. # update.log every profile shares, and only this id ties one to the action THIS log started.
  415. action_id = (env_overrides or {}).get("HERMES_ACTION_ID")
  416. stamp = time.strftime('%Y-%m-%d %H:%M:%S') + (f" {action_id}" if action_id else "")
  417. log_file.write(f"\n=== {name} started {stamp} ===\n".encode())
  418. if action_id:
  419. # ...and a sidecar keeps it once build output pushes the header past the status route's
  420. # bounded tail, or the log rotates: a restarted dashboard still knows which action it ran.
  421. from hermes_cli.runtime_state import _atomic_bytes
  422. _atomic_bytes(_ACTION_LOG_DIR / f"{name}.action_id", action_id.encode("ascii"))
  423. from hermes_cli._launchers import runtime_command
  424. cmd = runtime_command(PROJECT_ROOT, subcommand)
  425. if _action_targets_system_gateway(subcommand):
  426. # A system-scope lifecycle verb spawned as the dashboard's own user can only ever write
  427. # "System gateway <verb> requires root" into this log, so the button never worked on a
  428. # system install (#110820). Elevate — the CLI is sudo-aware: it adopts the unit's
  429. # HERMES_HOME past sudo's env_reset and reads SUDO_USER for the service identity.
  430. # ``-n`` never prompts (stdin is DEVNULL anyway); without a passwordless path the
  431. # REQUEST fails instead of reporting a started action whose child refuses. Same
  432. # two-step gate as the ``hermes update`` fleet restart: a refused blanket probe falls
  433. # back to ``sudo -l`` on the exact argv, so a command-scoped NOPASSWD entry qualifies.
  434. from hermes_cli.update_cmd_fleet import _sudo_noninteractive_ok
  435. if not _sudo_noninteractive_ok(["-l", "--", *cmd]):
  436. message = (
  437. f"{name} targets the system-scope gateway service, which requires root, and "
  438. "passwordless sudo is unavailable for the dashboard user. Run "
  439. f"'sudo hermes {' '.join(subcommand)}' on the host, or grant that user NOPASSWD sudo."
  440. )
  441. log_file.write(f"{message}\n".encode())
  442. log_file.close()
  443. raise RuntimeError(message)
  444. cmd = ["sudo", "-n", *cmd]
  445. # Named-profile actions get a scrubbed, pinned environment so the child cannot inherit the
  446. # dashboard profile's credentials; see _profile_action_environment (also drops _HERMES_GATEWAY).
  447. action_env = _profile_action_environment(subcommand, env_overrides)
  448. def _popen(**detach: Any) -> subprocess.Popen:
  449. return subprocess.Popen(
  450. cmd, cwd=str(PROJECT_ROOT), stdin=subprocess.DEVNULL, stdout=log_file, stderr=subprocess.STDOUT,
  451. env=action_env, **detach,
  452. )
  453. try:
  454. if sys.platform == "win32":
  455. try:
  456. proc = _popen(creationflags=windows_detach_flags())
  457. except OSError as exc:
  458. # CREATE_BREAKAWAY_FROM_JOB is refused ("Access is denied") when the parent's job
  459. # object forbids breakaway — an Electron/Tauri-launched dashboard, some Windows
  460. # Terminal configs. The flag is best-effort by contract (windows_detach_flags);
  461. # retry without it so the action still spawns, and record why for the status route.
  462. if getattr(exc, "winerror", None) != 5:
  463. raise
  464. log_file.write(
  465. f"{name}: breakaway spawn refused ({exc}); retrying without CREATE_BREAKAWAY_FROM_JOB\n".encode()
  466. )
  467. proc = _popen(creationflags=windows_detach_flags_without_breakaway())
  468. else:
  469. # detached-spawn: ok — dashboard gateway start/restart actions, which must outlive the dashboard
  470. proc = _popen(start_new_session=True)
  471. except OSError as exc:
  472. # Without this the action log would hold only the started-header for a spawn that never
  473. # happened; close the handle the child would have owned and let the endpoint report it.
  474. log_file.write(f"{name}: spawn failed: {exc}\n".encode())
  475. log_file.close()
  476. raise
  477. log_file.close() # child holds its own dup'd fd; keeping ours leaks one per action
  478. _ACTION_RESULTS.pop(name, None)
  479. _ACTION_COMMANDS[name] = tuple(subcommand)
  480. _ACTION_PROCS[name] = proc
  481. _ACTION_LOG_STARTS[name] = (proc.pid, section_start)
  482. if action_id:
  483. _ACTION_IDS[name] = action_id
  484. else:
  485. _ACTION_IDS.pop(name, None)
  486. return proc
  487. def _own_profile_selector(profile: Optional[str]) -> Optional[str]:
  488. """The profile a lifecycle verb addresses: the explicit selector, else the process's own
  489. profile (a pooled Desktop ``hermes --profile X serve`` answers ``/api/gateway/*`` without
  490. ``?profile=``; an unscoped verb there is about X). The default home resolves to the literal
  491. ``"default"`` selector, never to a bare argv: a selector-less child re-reads the sticky
  492. ``active_profile`` and would act on another profile's gateway (and skips the named-target
  493. env scrub). ``None`` is reserved for a home that is not a profile home at all."""
  494. requested = (profile or "").strip()
  495. if requested:
  496. return requested
  497. from hermes_constants import get_process_hermes_home, profile_name_for_home
  498. return profile_name_for_home(get_process_hermes_home()) or None
  499. def _gateway_subcommand(profile: Optional[str], verb: str) -> list[str]:
  500. """``hermes [-p X] gateway <verb>`` argv for a dashboard lifecycle action. A profile served by the
  501. live default multiplexer has no gateway of its own: ``restart`` targets the multiplexer (the process
  502. that actually serves X — a ``-p X gateway restart`` child only exits 78 into the action log while the
  503. UI reports "restarted"); ``start``/``stop`` are refused by the caller (``multiplexed_profile_refusal``).
  504. The multiplexer is addressed as ``-p default`` explicitly: a bare ``gateway restart`` spawned from a
  505. pooled ``--profile X serve`` would inherit X's ``HERMES_HOME`` and hit the same exit-78 refusal. The
  506. selector is explicit for the default home too: a bare child re-reads the sticky ``active_profile``
  507. and would restart another profile's gateway."""
  508. from hermes_cli.web_server_profiles import _profile_cli_args
  509. profile = _own_profile_selector(profile)
  510. args = _profile_cli_args(profile)
  511. if profile and verb == "restart" and multiplexed_profile_refusal(profile, verb) is not None:
  512. # A served profile's restart targets the multiplexer, addressed as ``-p default``.
  513. args = ["-p", "default"]
  514. return args + ["gateway", verb]
  515. def _profile_is_multiplexed(profile: str) -> bool:
  516. from hermes_cli.gateway import named_profile_served_by_running_multiplexer
  517. return named_profile_served_by_running_multiplexer(profile)
  518. def _has_own_gateway(profile_dir: Path) -> bool:
  519. """A live gateway of the profile's OWN (a ``--force``-started separate one), not the multiplexer that
  520. serves it. Gateway liveness reports a served profile as running on the multiplexer's PID (#97120),
  521. so reading liveness alone made every served profile look self-hosted and the refusal below never
  522. fired while a multiplexer was live, which is the only time it is needed."""
  523. from gateway.status import get_running_pid, multiplexer_liveness_for_profile, resolve_gateway_liveness
  524. from hermes_cli.profiles import _check_gateway_running
  525. if not _check_gateway_running(profile_dir):
  526. return False
  527. served = multiplexer_liveness_for_profile(profile_dir)
  528. if served is None:
  529. return True
  530. liveness = resolve_gateway_liveness(
  531. profile_dir=profile_dir, use_cache=False,
  532. pid_probe=lambda path: get_running_pid(path, cleanup_stale=False))
  533. return liveness.running and liveness.pid != served[0]
  534. def multiplexed_profile_refusal(profile: Optional[str], verb: str) -> Optional[str]:
  535. """Refusal text for ``gateway start``/``stop`` on a named profile with no gateway of its own (a
  536. ``--force``-started separate one is managed normally), else None. A profile the live host
  537. multiplexer serves is parked by ``stop`` and a parked one is unparked by ``start`` (the spawned
  538. ``hermes -p X gateway <verb>`` runs ``gateway_profile_lifecycle``), so neither is refused;
  539. ``start`` on an unparked named profile is — one host gateway serves every profile, so a new
  540. per-profile gateway is never the answer (the CLI twin ``_named_profile_refused_under_multiplexer``
  541. exits 78 into an action log nobody reads while the UI shows the verb as done)."""
  542. requested = _own_profile_selector(profile) or ""
  543. if not requested or requested.lower() in {"current", "default"}:
  544. return None
  545. served = _profile_is_multiplexed(requested)
  546. from hermes_cli.profiles import profile_is_parked, profile_is_standalone
  547. from hermes_cli.web_server_profiles import _resolve_profile_dir
  548. profile_dir = _resolve_profile_dir(requested)
  549. standalone = profile_is_standalone(profile_dir)
  550. if standalone:
  551. # The profile opted out of the host multiplexer, so its own gateway is the answer now: only a
  552. # host record that still lists it (the host started before the key was set) is refused.
  553. if not served:
  554. return None
  555. from gateway.host_attach import standalone_rescan_message
  556. return standalone_rescan_message(requested)
  557. if verb == "start" and profile_is_parked(profile_dir):
  558. from gateway.host_attach import host_gateway
  559. if host_gateway() is not None:
  560. return None # a live host unparks it; with no host the refusal below still applies
  561. if not served and verb != "start":
  562. return None
  563. if _has_own_gateway(profile_dir):
  564. return None
  565. if served:
  566. if verb == "stop":
  567. return None # parks the profile inside the host
  568. return (f"The default gateway already serves profile '{requested}' as a multiplexer; "
  569. f"{verb} it from the default profile instead of a separate gateway for this profile.")
  570. from hermes_cli.gateway_migrate import _installed_services
  571. if _installed_services(profile_dir):
  572. return None # a --force-installed fleet member is not NEW; its own service is started normally
  573. return (f"Profile '{requested}' does not get a gateway of its own: one host gateway serves every "
  574. f"profile. Install or start it from the default profile (hermes gateway install), or fold an "
  575. f"existing per-profile fleet with `hermes gateway migrate --multiplex`; "
  576. f"`hermes -p {requested} gateway install --force` starts a separate one anyway.")
  577. def live_session_refusal(profile: Optional[str], verb: str) -> Optional[str]:
  578. """Refusal text when the gateway ``verb`` would act on has live sessions, else None. Asked before
  579. spawning: the child refuses the same way, but into an action log the UI never reads."""
  580. from hermes_cli.gateway_lifecycle_guards import refusal_lines
  581. from hermes_cli.web_server_profiles import _resolve_profile_dir
  582. homes = [_resolve_profile_dir(_own_profile_selector(profile) or "default")]
  583. from gateway.host_attach import host_gateway
  584. owner = host_gateway()
  585. if owner is not None:
  586. homes.append(Path(owner.home))
  587. lines = refusal_lines(verb, homes, flag_hint=f"POST /api/gateway/{verb}?interrupt=true")
  588. return "\n".join(lines) or None
  589. def _restart_gateway_after(profile: Optional[str], *, what: str, label: str) -> dict[str, Any]:
  590. """Best-effort gateway restart after a config change. The save stays authoritative: a failed
  591. spawn is reported (``restart_started: False`` + ``restart_error``) so the UI can fall back to
  592. its manual restart banner instead of failing the request. Live sessions refuse it the same way,
  593. with ``restart_busy`` set so a UI can say so in its own words."""
  594. from hermes_cli.web_server import _spawn_gateway_restart
  595. refusal = live_session_refusal(profile, "restart")
  596. if refusal:
  597. return {"restart_started": False, "restart_error": refusal, "restart_busy": True}
  598. try:
  599. proc, reused = _spawn_gateway_restart(profile)
  600. except Exception as exc:
  601. _log.exception("Failed to auto-restart gateway after %s", what)
  602. return {"restart_started": False, "restart_error": str(exc)}
  603. if reused:
  604. _log.info("%s: reusing in-flight gateway restart (pid %s)", label, proc.pid)
  605. return {"restart_started": True, "restart_action": "gateway-restart", "restart_pid": proc.pid}
  606. _ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
  607. _RESTART_REASON_MAX = 300
  608. def _restart_log_last_line(pid: int) -> str:
  609. """The last line the ``gateway-restart`` child *pid* wrote to its log, or ``""``: its section
  610. runs from the offset ``_spawn_hermes_action`` recorded to the next spawn's header."""
  611. start = _ACTION_LOG_STARTS.get("gateway-restart")
  612. if start is None or start[0] != pid:
  613. return ""
  614. log_path = _ACTION_LOG_DIR / _ACTION_LOG_FILES["gateway-restart"]
  615. try:
  616. with log_path.open("rb") as log:
  617. log.seek(start[1])
  618. section = log.read(256 * 1024).decode("utf-8", errors="replace")
  619. except OSError:
  620. return ""
  621. header, _, body = section.lstrip("\n").partition("\n")
  622. if not header.startswith("=== gateway-restart started"):
  623. return "" # the log was rotated or truncated since
  624. body = body.split("\n=== gateway-restart started", 1)[0]
  625. lines = [_ANSI_ESCAPE.sub("", line).strip() for line in body.splitlines()]
  626. last = next((line for line in reversed(lines) if line), "")
  627. return last[:_RESTART_REASON_MAX]
  628. def gateway_restart_failure(profile: Optional[str]) -> Optional[dict[str, Any]]:
  629. """Why the newest restart of the *profile* gateway failed, or ``None`` while it runs, after it
  630. succeeded, or when this process started none. A child that exits nonzero leaves the old
  631. gateway serving the old config, so a UI waiting for the new one learns here that none is coming.
  632. ``pid`` is the ``restart_pid`` the restart answered, so the UI ignores an older restart."""
  633. from hermes_cli import web_server
  634. recent = web_server._LAST_GATEWAY_RESTART
  635. if recent is None:
  636. return None
  637. _spawned_at, proc, command = recent
  638. if tuple(arg for arg in command if arg != "--interrupt") != tuple(_gateway_subcommand(profile, "restart")):
  639. return None
  640. exit_code = proc.poll()
  641. if not exit_code:
  642. return None
  643. return {"pid": proc.pid, "exit_code": exit_code, "reason": _restart_log_last_line(proc.pid) or f"exit code {exit_code}"}
  644. def _split_text_for_speak_stream(text: str, cap: int) -> list:
  645. """Split *text* into provider-cap-sized pieces on sentence boundaries.
  646. Deliberately NOT unified with gateway.platforms.helpers' split_text_fence_aware: this
  647. reflows whitespace (sentences re-joined with single spaces) and has no fence semantics.
  648. """
  649. from tools.tts_streaming import SENTENCE_BOUNDARY_RE as _SENTENCE_BOUNDARY_RE
  650. cap = cap if cap and cap > 0 else 4000
  651. pieces, buf = [], ""
  652. for sentence in filter(str.strip, _SENTENCE_BOUNDARY_RE.split(text)):
  653. while len(sentence) > cap:
  654. pieces.append(sentence[:cap])
  655. sentence = sentence[cap:]
  656. if buf and len(buf) + len(sentence) + 1 > cap:
  657. pieces.append(buf)
  658. buf = sentence
  659. else:
  660. buf = f"{buf} {sentence}" if buf else sentence
  661. if buf:
  662. pieces.append(buf)
  663. return pieces
  664. # Per-row fields no session LIST consumer reads but that dominate the payload (``system_prompt``
  665. # is the fully rendered prompt, tens of KB per row — 96% of a 528KB /api/sessions response).
  666. # Detail reads stay complete; list callers that need full rows pass ``?full=1``.
  667. _SESSION_LIST_HEAVY_FIELDS = ("system_prompt", "model_config")
  668. def _strip_session_list_rows(sessions: list[dict[str, Any]]) -> list[dict[str, Any]]:
  669. for s in sessions:
  670. for key in _SESSION_LIST_HEAVY_FIELDS:
  671. s.pop(key, None)
  672. return sessions