main_dashboard.py 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126
  1. """Dashboard/serve support: managed-service restart (systemd/respawn), status/listening probes, SSH session token file, named-profile routing, web-dist resolution, update stdio hangup protection.
  2. Split out of ``hermes_cli/main.py``. Names that still live in main (``PROJECT_ROOT``, ...)
  3. are imported lazily inside the functions that use them (avoids an import cycle).
  4. """
  5. import contextlib
  6. import os
  7. import re
  8. import shlex
  9. import subprocess
  10. import sys
  11. import time
  12. from pathlib import Path
  13. from typing import NoReturn
  14. from hermes_cli.cli_output import line_input
  15. from hermes_cli.process_identity import is_desktop_owned_backend as _is_desktop_owned_backend
  16. _PRE_BUILD_HINT = " Pre-build first: npm install --workspace web && npm run build -w web"
  17. def _find_stale_dashboard_pids(*, exclude_pids: set[int] | None = None,
  18. scope_home: str | None = None) -> list[int]:
  19. """PIDs of running ``dashboard``/``serve`` backends the caller may stop.
  20. *scope_home*: keep only backends whose resolved Hermes home (see
  21. ``_hermes_home_for_pid``) is this home; unreadable ownership is spared, never guessed.
  22. ``--stop`` and the post-update cleanup pass their own home so another install's or
  23. profile's backend on the same machine is never a target (#113978).
  24. """
  25. from hermes_cli.dashboard_procs import (
  26. _caller_ancestor_pids,
  27. _is_caller_wrapper_shell,
  28. _pids_owned_by_hermes_home,
  29. _scan_dashboard_processes,
  30. )
  31. pids = [pid for pid, _cmd in _scan_dashboard_processes(exclude_pids=exclude_pids)]
  32. # The scan also selects the caller's own wrapper shell (``bash -c 'hermes dashboard --stop'``);
  33. # killing it takes down the invoking terminal.
  34. ancestors = _caller_ancestor_pids()
  35. pids = [pid for pid in pids if not _is_caller_wrapper_shell(pid, ancestors)]
  36. return _pids_owned_by_hermes_home(pids, scope_home) if scope_home else pids
  37. def _parse_dashboard_runtime(command: str) -> tuple[str, str, int] | None:
  38. """Best-effort parse of a dashboard/server cmdline into mode, host, and port.
  39. The mode is the canonical holder subcommand, never an argv substring: this gates the launchd
  40. backend inventory (a kill + kickstart path) and ``--status`` (#121156).
  41. """
  42. from hermes_cli.update_cmd_windows import _hermes_holder_subcommand
  43. mode = _hermes_holder_subcommand(command)
  44. if mode not in ("dashboard", "serve"):
  45. return None
  46. port = 9119
  47. host = "127.0.0.1"
  48. port_match = re.search(r"(?:^|\s)--port(?:=|\s+)(\d+)", command)
  49. if port_match:
  50. try:
  51. port = int(port_match.group(1))
  52. except ValueError:
  53. return None
  54. host_match = re.search(r"(?:^|\s)--host(?:=|\s+)(\"[^\"]+\"|'[^']+'|\S+)", command)
  55. if host_match:
  56. host = host_match.group(1).strip("\"'") or "127.0.0.1"
  57. return mode, host, port
  58. def _dashboard_probe_host(host: str | None) -> str:
  59. """Map wildcard binds to a loopback address suitable for local probing."""
  60. normalized = (host or "127.0.0.1").strip().strip("[]")
  61. if normalized in {"", "0.0.0.0", "::"}:
  62. return "127.0.0.1"
  63. return normalized
  64. _DASHBOARD_SYSTEMD_UNIT = "hermes-dashboard.service"
  65. _SYSTEMCTL_ERRORS = (FileNotFoundError, subprocess.TimeoutExpired, OSError)
  66. def _run_probe(cmd: list[str], *, timeout: int) -> subprocess.CompletedProcess:
  67. """Captured, text-decoded ``subprocess.run`` for short local probes (systemctl, ps)."""
  68. return subprocess.run(
  69. cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False)
  70. def _restart_managed_dashboard_service(reason: str, unit: str = _DASHBOARD_SYSTEMD_UNIT) -> bool:
  71. """Restart a systemd-managed dashboard instead of raw-killing its PID.
  72. True when a unit was found and handled (success or printed failure) — which
  73. deliberately stops the caller's ``os.kill`` fallback: systemd treats a direct
  74. SIGTERM as a clean stop, so ``Restart=on-failure`` won't bring it back.
  75. """
  76. if sys.platform == "win32":
  77. return False
  78. def _systemctl(*args: str, timeout: int = 10) -> subprocess.CompletedProcess:
  79. return _run_probe(["systemctl", *args], timeout=timeout)
  80. # User manager first (Hermes installs Linux services in the user scope by
  81. # default), system manager only when the unit isn't there. Keep the selected
  82. # scope for ALL probes and the restart — a user unit must never be restarted
  83. # through the system manager (or raw-killed).
  84. scope: tuple[str, ...] | None = None
  85. for candidate in (("--user",), ()):
  86. try:
  87. result = _systemctl(*candidate, "list-unit-files", unit, "--no-legend", "--no-pager")
  88. except _SYSTEMCTL_ERRORS:
  89. continue
  90. if result.returncode != 0:
  91. continue
  92. unit_rows = (result.stdout or "").splitlines()
  93. if any(row.split()[0:1] == [unit] for row in unit_rows if row.split()):
  94. scope = candidate
  95. break
  96. if scope is None:
  97. return False
  98. try:
  99. active = _systemctl(*scope, "is-active", unit)
  100. enabled = _systemctl(*scope, "is-enabled", unit)
  101. except _SYSTEMCTL_ERRORS:
  102. return False
  103. active_state = (active.stdout or "").strip()
  104. enabled_state = (enabled.stdout or "").strip()
  105. if active_state != "active" and enabled_state not in {
  106. "enabled", "enabled-runtime", "linked", "linked-runtime", "static", "generated",
  107. }:
  108. return False
  109. print(f"\n⟲ Restarting managed dashboard service ({reason})")
  110. scope_label = "systemctl --user" if scope else "sudo systemctl"
  111. commands = [("systemctl", *scope, "restart", unit)]
  112. if not scope:
  113. # System units may require privilege escalation; user units must use
  114. # the user manager directly and never prompt for sudo.
  115. commands.append(("sudo", "-n", "systemctl", "restart", unit))
  116. errors: list[str] = []
  117. for command in commands:
  118. try:
  119. result = _run_probe(list(command), timeout=60)
  120. except _SYSTEMCTL_ERRORS as e:
  121. errors.append(f"{' '.join(command)}: {e}")
  122. continue
  123. if result.returncode == 0:
  124. print(f" ✓ restarted {unit}")
  125. return True
  126. errors.append(f"{' '.join(command)}: {(result.stderr or result.stdout or '').strip()}")
  127. print(f" ✗ failed to restart {unit}")
  128. for err in errors:
  129. if err.strip():
  130. print(f" {err}")
  131. print(
  132. " Dashboard is managed by systemd; not raw-killing its PID because "
  133. "systemd would treat that as a clean stop."
  134. )
  135. print(f" Restart manually: {scope_label} restart {unit}")
  136. return True
  137. def _pid_unified_cgroup_entries(pid: int):
  138. """Yield the ``0::<path>`` cgroup paths from ``/proc/<pid>/cgroup``; nothing when unreadable."""
  139. try:
  140. cgroup_path = Path(f"/proc/{pid}/cgroup")
  141. if not cgroup_path.is_file():
  142. return
  143. text = cgroup_path.read_text(encoding="utf-8-sig", errors="replace")
  144. except (OSError, PermissionError):
  145. return
  146. for line in text.splitlines():
  147. parts = line.strip().split("::", 1)
  148. if len(parts) == 2:
  149. yield parts[1]
  150. def _get_systemd_service_for_pid(pid: int) -> str | None:
  151. """The systemd service unit that supervises *pid* (``hermes-serve.service``), or None.
  152. A ``.service`` cgroup alone only says where the process was started: a dashboard launched by
  153. hand from a shell that itself runs under some unit (a CI runner agent, ``cron.service``, a
  154. tmux or IDE user service, the gateway's own terminal tool) sits in THAT unit's cgroup. The
  155. unit owns the backend only when its live ``MainPID`` is this PID; otherwise restarting it
  156. restarts an unrelated service and leaves the dashboard down. None when the PID isn't part of
  157. a service, ownership can't be proved, the file is unreadable, or off Linux.
  158. """
  159. for cg_path in _pid_unified_cgroup_entries(pid):
  160. if cg_path.endswith(".service"):
  161. svc_name = cg_path.rsplit("/", 1)[-1]
  162. if svc_name and _unit_main_pid_is(svc_name, cg_path, pid):
  163. return svc_name
  164. return None
  165. def _unit_main_pid_is(svc_name: str, cgroup_path: str, pid: int) -> bool:
  166. """True when *svc_name*'s live ``MainPID`` is *pid* (read-only ``systemctl show``)."""
  167. scope = _extract_scope_from_cgroup(cgroup_path)
  168. scopes = {"user": [["--user"]], "system": [[]]}.get(scope or "", [[], ["--user"]])
  169. for scope_args in scopes:
  170. try:
  171. result = _run_probe(
  172. ["systemctl", *scope_args, "show", svc_name, "--property=MainPID", "--value"], timeout=10)
  173. except _SYSTEMCTL_ERRORS:
  174. continue
  175. if result.returncode == 0 and (result.stdout or "").strip() == str(pid):
  176. return True
  177. return False
  178. def _extract_scope_from_cgroup(cgroup_entry: str) -> str | None:
  179. """``user`` / ``system`` from a cgroup path (``/user.slice/…`` vs ``/system.slice/…``), else None."""
  180. if "/system.slice/" in cgroup_entry:
  181. return "system"
  182. if "/user.slice/" in cgroup_entry:
  183. return "user"
  184. return None
  185. def _get_pid_cgroup_path(pid: int) -> str | None:
  186. """The unified (``0::``) cgroup path from ``/proc/<pid>/cgroup``, or None."""
  187. return next(_pid_unified_cgroup_entries(pid), None)
  188. def _systemd_owner_for_pid(pid: int) -> tuple[str, str | None] | None:
  189. """``(unit, scope)`` of the systemd unit whose live ``MainPID`` is *pid*; ``None`` when no unit
  190. provably owns it (see :func:`_get_systemd_service_for_pid`). Scope is ``user``/``system`` or
  191. ``None`` when the cgroup path does not say."""
  192. unit = _get_systemd_service_for_pid(pid)
  193. if unit is None:
  194. return None
  195. return unit, _extract_scope_from_cgroup(_get_pid_cgroup_path(pid) or "")
  196. def _systemd_restart_commands(unit: str, scope: str | None) -> list[str]:
  197. """The command(s) that restart *unit* in its scope; both forms when the scope is unknown."""
  198. user_cmd, system_cmd = f"systemctl --user restart {unit}", f"sudo systemctl restart {unit}"
  199. return {"user": [user_cmd], "system": [system_cmd]}.get(scope or "", [user_cmd, system_cmd])
  200. def _format_restart_commands(commands: list[str]) -> str:
  201. """Join restart commands as a user-facing hint: each backticked, separated by " or "."""
  202. return " or ".join(f"`{c}`" for c in commands)
  203. def _try_restart_systemd_service(svc_name: str, cgroup_path: str | None = None) -> bool:
  204. """Restart *svc_name* via systemctl (``--user`` for user-scope units). True on success.
  205. Unknown scope tries system first, then user.
  206. """
  207. scope = _extract_scope_from_cgroup(cgroup_path) if cgroup_path else None
  208. system_cmd = ["systemctl", "restart", svc_name]
  209. user_cmd = ["systemctl", "--user", "restart", svc_name]
  210. candidates = {"user": [user_cmd], "system": [system_cmd]}.get(scope, [system_cmd, user_cmd])
  211. for cmd in candidates:
  212. try:
  213. if _run_probe(cmd, timeout=15).returncode == 0:
  214. return True
  215. except _SYSTEMCTL_ERRORS:
  216. continue
  217. return False
  218. # launchd plist directories that can supervise a ``hermes dashboard`` / ``hermes serve`` backend on
  219. # macOS, with the launchctl domain their jobs load into (LaunchAgents: ``gui/<uid>`` or ``user/<uid>``,
  220. # probed per label like the gateway helpers; LaunchDaemons: ``system``). Both LaunchAgents dirs are
  221. # per-user domains, so they share the ``agent`` kind.
  222. def _launchd_plist_dirs() -> list[tuple[str, Path]]:
  223. return [
  224. ("agent", Path.home() / "Library" / "LaunchAgents"),
  225. ("agent", Path("/Library/LaunchAgents")),
  226. ("daemon", Path("/Library/LaunchDaemons")),
  227. ]
  228. def _loaded_launchd_backend_jobs(
  229. plist_dirs: list[tuple[str, Path]] | None = None,
  230. ) -> list[tuple[str, str, list[str], int | None]]:
  231. """``(domain, label, program_arguments, live_pid)`` for every LOADED launchd job whose
  232. ``ProgramArguments`` is a ``hermes dashboard`` / ``hermes serve`` backend. macOS only (empty
  233. elsewhere). Reads the plists (unreadable/malformed ones are skipped) and asks ``launchctl print``
  234. per candidate label — a job that is not loaded in any domain is not returned, so an operator's
  235. stale plist never claims a process."""
  236. if sys.platform != "darwin":
  237. return []
  238. import plistlib
  239. from xml.parsers.expat import ExpatError
  240. from hermes_cli.gateway import _launchd_print_service_pid
  241. uid = os.getuid() # windows-footgun: ok — darwin-only branch
  242. jobs: list[tuple[str, str, list[str], int | None]] = []
  243. for kind, plist_dir in (plist_dirs if plist_dirs is not None else _launchd_plist_dirs()):
  244. try:
  245. plists = sorted(plist_dir.glob("*.plist"))
  246. except OSError:
  247. continue
  248. for plist_path in plists:
  249. try:
  250. with open(plist_path, "rb") as f:
  251. data = plistlib.load(f)
  252. # ExpatError is NOT a ValueError: plistlib propagates it unwrapped for
  253. # XML that is not well-formed (e.g. a hand-edited plist with a raw
  254. # `&` in `ProgramArguments`), and one such operator file must skip —
  255. # not abort — the whole post-pull cleanup scan.
  256. except (OSError, ValueError, plistlib.InvalidFileException, ExpatError):
  257. continue
  258. if not isinstance(data, dict):
  259. continue
  260. label = str(data.get("Label") or "").strip()
  261. args = data.get("ProgramArguments")
  262. if not label or not isinstance(args, list) or not args:
  263. continue
  264. argv = [str(a) for a in args]
  265. if _parse_dashboard_runtime(shlex.join(argv)) is None:
  266. continue
  267. domains = ("system",) if kind == "daemon" else (f"gui/{uid}", f"user/{uid}")
  268. for domain in domains:
  269. try:
  270. loaded, live_pid = _launchd_print_service_pid(domain, label)
  271. except _SYSTEMCTL_ERRORS:
  272. loaded, live_pid = False, None
  273. if loaded:
  274. jobs.append((domain, label, argv, live_pid))
  275. break
  276. return jobs
  277. def _launchd_job_owning_backend(
  278. pid: int, cmdline: list[str] | None, jobs: list[tuple[str, str, list[str], int | None]],
  279. ancestors: list[int] | tuple[int, ...] = (),
  280. ) -> tuple[str, str, int | None] | None:
  281. """``(domain, label, live_pid)`` of the loaded launchd job that owns *pid*: launchd reports *pid*
  282. (or one of its *ancestors* — a plist may wrap the backend in ``/bin/sh -c …`` without ``exec``)
  283. as the job's live process, OR the process runs the job's ``ProgramArguments`` — a detached copy
  284. of a supervised backend (an earlier respawn) holds the port the job needs, and respawning it again
  285. would only re-create that conflict. ``--no-open`` is ignored on both sides: the respawn path adds
  286. it, so an earlier respawn's argv is the plist's plus that flag. None when no loaded job claims
  287. the process."""
  288. def _norm(argv: list[str]) -> list[str]:
  289. return [a for a in argv if a != "--no-open"]
  290. for domain, label, argv, live_pid in jobs:
  291. if live_pid is not None and (live_pid == pid or live_pid in ancestors):
  292. return (domain, label, live_pid)
  293. if cmdline is not None and _norm(list(cmdline)) == _norm(argv):
  294. return (domain, label, live_pid)
  295. return None
  296. def _restart_launchd_job(domain: str, label: str, old_pid: int | None, *, timeout: float = 15.0) -> bool:
  297. """Bring a launchd-supervised backend back after its process was stopped: ``launchctl kickstart
  298. <domain>/<label>`` (no ``-k`` — a KeepAlive job may already have respawned it, and a kill would
  299. take that fresh process down), then require launchd to report a live PID other than *old_pid*
  300. within *timeout*. A kickstart that returns 0 only means "restart requested"; a job that is loaded
  301. but never comes back on a fresh PID is a failure the operator must hear about."""
  302. from hermes_cli.gateway import _wait_for_launchd_service_pid
  303. try:
  304. if _run_probe(["launchctl", "kickstart", f"{domain}/{label}"], timeout=30).returncode != 0:
  305. return False
  306. return _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=timeout, domain=domain)
  307. except _SYSTEMCTL_ERRORS:
  308. return False
  309. def _dashboard_cmdline_for_pid(pid: int) -> list[str] | None:
  310. """Exact argv of a running process: ``/proc/<pid>/cmdline`` (Linux), ``ps -o command=`` + shlex
  311. (macOS), None on Windows (no graceful taskkill window; Desktop manages its backend)."""
  312. if sys.platform == "win32":
  313. return None
  314. try:
  315. cmdline_path = f"/proc/{pid}/cmdline"
  316. if os.path.exists(cmdline_path):
  317. with open(cmdline_path, "rb") as f:
  318. raw = f.read()
  319. argv = [part.decode("utf-8", errors="replace") for part in raw.split(b"\x00") if part]
  320. return argv or None
  321. result = _run_probe(["ps", "-p", str(pid), "-o", "command="], timeout=10)
  322. if result.returncode != 0:
  323. return None
  324. command = (result.stdout or "").strip()
  325. if not command:
  326. return None
  327. try:
  328. argv = shlex.split(command)
  329. except ValueError:
  330. argv = command.split()
  331. return argv or None
  332. except (OSError, ValueError, subprocess.TimeoutExpired):
  333. return None
  334. _RESPAWN_LIVENESS_GRACE_SECONDS = 1.0
  335. def _respawnable_command_for_current_install(argv: list[str]) -> list[str]:
  336. """Rebuild a captured ``[<interpreter>, <hermes launcher>, ...]`` argv on this install's launcher.
  337. A pre-PM-takeover install left ``~/.local/bin/hermes`` as a symlink to a Python console
  338. script, so the kernel recorded a manual backend as ``[<old venv python>, <launcher>, dashboard,
  339. ...]``. The takeover then rewrote that launcher into a POSIX shell shim, and replaying the
  340. captured argv verbatim asks the old interpreter to parse a shell script (#124778). This
  341. checkout's own ``hermes`` entry script stays Python, so it and every other shape replay unchanged.
  342. """
  343. root = Path(__file__).resolve().parents[1]
  344. if (len(argv) > 2 and os.path.basename(argv[0]).startswith("python")
  345. and os.path.basename(argv[1]) == "hermes" and Path(argv[1]) != root / "hermes"):
  346. from hermes_cli._launchers import runtime_command
  347. return runtime_command(root, argv[2:])
  348. return list(argv)
  349. def _respawn_dashboard_processes(commands: list[list[str]]) -> list[list[str]]:
  350. """Respawn manually-started dashboards after ``hermes update``, detached, logging to
  351. ``logs/dashboard-restart.log``; returns the argvs that failed to spawn. Callers pre-filter via
  352. ``_filter_dashboard_respawn_candidates`` (no Desktop ``--port 0`` backends, capped per profile).
  353. See #78821.
  354. """
  355. from hermes_constants import get_hermes_home
  356. respawned: list[list[str]] = []
  357. spawned: list[tuple[list[str], list[str], subprocess.Popen]] = []
  358. failed: list[tuple[list[str], list[str], str]] = []
  359. log_path = get_hermes_home() / "logs" / "dashboard-restart.log"
  360. with contextlib.suppress(OSError):
  361. log_path.parent.mkdir(parents=True, exist_ok=True)
  362. for original in commands:
  363. command = _respawnable_command_for_current_install(original)
  364. # Keep restarted dashboards headless; reopening a browser after a
  365. # background update is noisy and fails in SSH/headless sessions.
  366. if "dashboard" in command and "--no-open" not in command:
  367. command = [*command, "--no-open"]
  368. try:
  369. with open(log_path, "ab") as log_f:
  370. proc = subprocess.Popen(
  371. command, stdin=subprocess.DEVNULL, stdout=log_f, stderr=subprocess.STDOUT,
  372. # detached-spawn: ok — dashboard service launch, meant to outlive the CLI that starts it
  373. start_new_session=True, close_fds=True)
  374. spawned.append((original, command, proc))
  375. except (OSError, ValueError) as exc:
  376. failed.append((original, command, str(exc)))
  377. # A respawned backend is a resident server: one that exits within the grace
  378. # window died at startup (SyntaxError on a stale argv, port already bound,
  379. # ...) and must surface as a failure, not as ``✓ restarted`` (#124778).
  380. if spawned:
  381. time.sleep(_RESPAWN_LIVENESS_GRACE_SECONDS)
  382. for original, command, proc in spawned:
  383. if proc.poll() is None:
  384. respawned.append(command)
  385. else:
  386. failed.append((original, command, f"child exited during the first "
  387. f"{_RESPAWN_LIVENESS_GRACE_SECONDS:.0f}s (code {proc.returncode})"))
  388. for command in respawned:
  389. print(f" ✓ restarted: {shlex.join(command)}")
  390. for _, command, err_msg in failed:
  391. print(f" ✗ failed to restart ({shlex.join(command)}): {err_msg}")
  392. # The caller's argv, not the spawned one: callers match it against the stopped PID's
  393. # captured cmdline to book the runtime as not brought back (#109290).
  394. return [original for original, _, _ in failed]
  395. class _UpdateOutputStream:
  396. """stdout/stderr wrapper for ``hermes update``: mirrors to ``logs/update.log`` and, once the
  397. terminal vanishes (BrokenPipe/OSError/ValueError), drops screen output instead of the update."""
  398. _BROKEN = (BrokenPipeError, OSError, ValueError)
  399. def __init__(self, original, log_file):
  400. self._original = original
  401. self._log = log_file
  402. self._original_broken = False
  403. def write(self, data):
  404. # Mirror to the log file first — it's the most reliable destination.
  405. if self._log is not None:
  406. with contextlib.suppress(Exception):
  407. self._log.write(data)
  408. if not self._original_broken:
  409. try:
  410. return self._original.write(data)
  411. except self._BROKEN:
  412. self._original_broken = True # terminal vanished; keep updating
  413. return len(data) if isinstance(data, (str, bytes)) else 0
  414. def flush(self):
  415. if self._log is not None:
  416. with contextlib.suppress(Exception):
  417. self._log.flush()
  418. if self._original_broken:
  419. return
  420. try:
  421. self._original.flush()
  422. except self._BROKEN:
  423. self._original_broken = True
  424. def isatty(self):
  425. if self._original_broken:
  426. return False
  427. try:
  428. return self._original.isatty()
  429. except Exception:
  430. return False
  431. def fileno(self):
  432. # Defer to the underlying stream; callers handle failures as when unwrapped.
  433. return self._original.fileno()
  434. def __getattr__(self, name):
  435. return getattr(self._original, name)
  436. def _install_hangup_protection(gateway_mode: bool = False):
  437. """Protect ``cmd_update`` from SIGHUP (→ SIG_IGN, inherited by pip/git children) and broken pipes
  438. (stdio wrapped in ``_UpdateOutputStream``). SIGINT/SIGTERM are left alone — legitimate cancels.
  439. Gateway updates are already detached, but still need the log mirror for the
  440. Desktop progress watchdog. Returns state for ``_finalize_update_output``."""
  441. state = {
  442. "prev_stdout": sys.stdout, "prev_stderr": sys.stderr, "log_file": None, "installed": False}
  443. import signal as _signal
  444. if not gateway_mode and hasattr(_signal, "SIGHUP"):
  445. # Non-main thread: update still runs, just without hangup protection.
  446. with contextlib.suppress(ValueError, OSError):
  447. _signal.signal(_signal.SIGHUP, _signal.SIG_IGN)
  448. # Any failure here is non-fatal; we just skip the wrap.
  449. try:
  450. # Late-bound import so tests can monkeypatch it to simulate setup failure. The ROOT
  451. # home, never a sticky profile's: the update mutates the shared checkout, and the
  452. # Desktop and the hand-off scripts read <root>/logs/update.log.
  453. from hermes_constants import get_default_hermes_root as _get_root_home
  454. logs_dir = _get_root_home() / "logs"
  455. logs_dir.mkdir(parents=True, exist_ok=True)
  456. log_file = open(logs_dir / "update.log", "a", buffering=1, encoding="utf-8")
  457. import datetime as _dt
  458. stage = "continued on the pulled code" if os.environ.get("HERMES_UPDATE_POST_SWAP") == "1" else "started"
  459. log_file.write(f"\n=== hermes update {stage} {_dt.datetime.now().isoformat(timespec='seconds')} ===\n")
  460. state["log_file"] = log_file
  461. sys.stdout = _UpdateOutputStream(state["prev_stdout"], log_file)
  462. sys.stderr = _UpdateOutputStream(state["prev_stderr"], log_file)
  463. state["installed"] = True
  464. except Exception:
  465. state["log_file"] = None
  466. return state
  467. def _finalize_update_output(state):
  468. """Restore stdio and close the update.log handle opened by ``_install_hangup_protection``."""
  469. if not state:
  470. return
  471. if state.get("installed"):
  472. with contextlib.suppress(Exception):
  473. sys.stdout = state.get("prev_stdout", sys.stdout)
  474. with contextlib.suppress(Exception):
  475. sys.stderr = state.get("prev_stderr", sys.stderr)
  476. log_file = state.get("log_file")
  477. if log_file is not None:
  478. with contextlib.suppress(Exception):
  479. log_file.flush()
  480. log_file.close()
  481. def _report_dashboard_status() -> int:
  482. """Print live listening dashboard/serve processes and return the count.
  483. Serve-mode backends are INCLUDED: ``--stop`` kills them, so hiding them from
  484. ``--status`` let an operator kill what they couldn't see.
  485. Ledger-registered serves (profiled launches the argv scan can't match) surface via the spawn-ledger
  486. augmentation in _scan_dashboard_processes, and the ledger's recorded bind replaces the argv port so
  487. ``--port 0`` backends are probed on the port the OS actually gave them. See #81564.
  488. """
  489. from hermes_cli.dashboard_procs import _ledger_serve_binds, _scan_dashboard_processes
  490. from gateway.status import _pid_exists
  491. binds = _ledger_serve_binds()
  492. live: list[tuple[int, str, str]] = []
  493. for pid, command in _scan_dashboard_processes():
  494. runtime = _parse_dashboard_runtime(command)
  495. if runtime is None:
  496. continue
  497. mode, host, port = runtime
  498. if pid in binds:
  499. ledger_host, port = binds[pid]
  500. host = ledger_host or host
  501. if port <= 0 or not _pid_exists(pid) or not _dashboard_listening(host, port):
  502. continue
  503. live.append((pid, command, mode))
  504. if not live:
  505. print("No hermes dashboard or serve processes running.")
  506. return 0
  507. print(f"{len(live)} hermes dashboard/serve process(es) running:")
  508. for pid, command, mode in live:
  509. print(f" PID {pid} [{mode}]: {command}")
  510. return len(live)
  511. def _dashboard_listening(host: str, port: int) -> bool:
  512. """True when something accepts TCP connections at host:port (even a 401 proves a dashboard is up)."""
  513. import socket
  514. try:
  515. with socket.create_connection((_dashboard_probe_host(host), port), timeout=1.5):
  516. return True
  517. except OSError:
  518. return False
  519. def _cancel(message: str = " Cancelled.") -> NoReturn:
  520. print(message)
  521. sys.exit(1)
  522. def _maybe_setup_dashboard_auth_interactively(args) -> None:
  523. """Offer to configure dashboard auth when the gate engages and no provider exists.
  524. ``start_server`` fails closed for a non-loopback bind / ``dashboard.public_url``
  525. without a ``DashboardAuthProvider``; prompt an interactive operator first.
  526. No-op (fail-closed backstop stays) when the gate doesn't engage, a provider
  527. exists, or stdin/stdout isn't a TTY.
  528. """
  529. host = getattr(args, "host", "127.0.0.1") or "127.0.0.1"
  530. try:
  531. from hermes_cli.web_server import should_require_dashboard_auth
  532. if not should_require_dashboard_auth(host):
  533. return
  534. except Exception:
  535. return # if we can't tell, defer to start_server's own gate
  536. try:
  537. from hermes_cli.dashboard_auth import list_providers
  538. if list_providers():
  539. return
  540. except Exception:
  541. return
  542. if not (sys.stdin.isatty() and sys.stdout.isatty()):
  543. return
  544. print(f"\n⚠ Dashboard authentication is required for this configuration ({host}).")
  545. print(
  546. " Non-loopback binds and configured external dashboard.public_url "
  547. "values require authentication (--insecure does not bypass this)."
  548. )
  549. print()
  550. print(" How do you want to authenticate the dashboard?")
  551. print(" [1] Username & password (quickest; for a trusted LAN / VPN)")
  552. print(" [2] OAuth via Nous Portal (run `hermes dashboard register`)\n [3] Cancel\n")
  553. try:
  554. choice = input(" Choice [1]: ").strip() or "1"
  555. except (EOFError, KeyboardInterrupt):
  556. _cancel("\n Cancelled.")
  557. if choice == "2":
  558. print()
  559. print(
  560. " Run this on the host where the dashboard lives, then start "
  561. "the dashboard again:\n"
  562. " hermes dashboard register\n"
  563. " It provisions a Nous Portal OAuth client and writes "
  564. "HERMES_DASHBOARD_OAUTH_CLIENT_ID into ~/.hermes/.env for you.\n"
  565. " Docs: https://hermes-agent.nousresearch.com/docs/"
  566. "user-guide/features/web-dashboard#authentication-gated-mode"
  567. )
  568. sys.exit(0)
  569. if choice != "1":
  570. _cancel()
  571. import getpass
  572. import secrets
  573. print()
  574. try:
  575. username = line_input(" Username [admin]: ").strip() or "admin"
  576. password = getpass.getpass(" Password: ")
  577. confirm = getpass.getpass(" Confirm password: ")
  578. except (EOFError, KeyboardInterrupt):
  579. _cancel("\n Cancelled.")
  580. if not password:
  581. _cancel(" ✗ Empty password — aborting.")
  582. if password != confirm:
  583. _cancel(" ✗ Passwords don't match — aborting.")
  584. try:
  585. from plugins.dashboard_auth.basic import hash_password
  586. except Exception as exc:
  587. _cancel(f" ✗ Could not load the password provider: {exc}")
  588. password_hash = hash_password(password)
  589. # A stable token-signing secret so sessions survive a dashboard restart.
  590. secret = secrets.token_urlsafe(32)
  591. try:
  592. from hermes_cli.config import load_config, save_config
  593. from hermes_cli.plugins_cmd import ensure_basic_auth_plugin_enabled_in_config
  594. cfg = load_config()
  595. basic = cfg.setdefault("dashboard", {}).setdefault("basic_auth", {})
  596. basic["username"] = username
  597. basic["password_hash"] = password_hash
  598. basic["password"] = "" # never persist plaintext
  599. if not str(basic.get("secret", "") or "").strip():
  600. basic["secret"] = secret
  601. # The bundled basic provider is a backend plugin that honours
  602. # plugins.disabled; unblock it so discover_plugins below registers it,
  603. # and tell an operator who deliberately disabled it.
  604. if ensure_basic_auth_plugin_enabled_in_config(cfg):
  605. print(" ✓ Re-enabled the bundled 'basic' auth plugin (was in plugins.disabled)")
  606. save_config(cfg)
  607. except Exception as exc:
  608. _cancel(f" ✗ Failed to write config.yaml: {exc}")
  609. # Re-run plugin discovery so the provider registers before start_server's gate.
  610. try:
  611. from hermes_cli.plugins import discover_plugins
  612. discover_plugins(force=True)
  613. except Exception as exc:
  614. print(f" ⚠ Plugin re-discovery failed ({exc}); the gate may still "
  615. "fail closed. Set the password again or restart the dashboard.")
  616. print()
  617. print(f" ✓ Username/password auth configured (user: {username}).")
  618. print(" Saved to config.yaml under dashboard.basic_auth.")
  619. print(" Sign in at the dashboard with these credentials.\n")
  620. def _read_ssh_session_token_file(path: str) -> str:
  621. """Read and unlink a Desktop SSH token from its private runtime directory."""
  622. if sys.platform == "win32":
  623. from hermes_cli.windows_ssh_runtime import read_token
  624. return read_token(path)
  625. import stat as _stat
  626. if not os.path.isabs(path):
  627. raise SystemExit("--ssh-session-token-file must be absolute")
  628. # The Desktop client writes the token under the account's $HOME/.hermes/
  629. # desktop-ssh, independent of HERMES_HOME and the active profile. Anchor
  630. # validation there, NOT get_hermes_home(): a non-default profile or a Docker
  631. # /opt/data root re-homes get_hermes_home() and would reject every token.
  632. # See #69551.
  633. token_root = Path.home() / ".hermes" / "desktop-ssh"
  634. try:
  635. relative = Path(path).relative_to(token_root)
  636. except ValueError as exc:
  637. raise SystemExit("--ssh-session-token-file must be under the desktop-ssh directory") from exc
  638. if len(relative.parts) != 2 or not re.fullmatch(r"[0-9a-f]{32}", relative.parts[0]):
  639. raise SystemExit("--ssh-session-token-file has an invalid runtime path")
  640. if not re.fullmatch(r"[0-9a-f]{16}\.token", relative.parts[1]):
  641. raise SystemExit("--ssh-session-token-file has an invalid filename")
  642. directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
  643. file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
  644. uid = os.getuid() if hasattr(os, "getuid") else None
  645. def _check_dir(fd: int, what: str) -> None:
  646. st = os.fstat(fd)
  647. if not _stat.S_ISDIR(st.st_mode):
  648. raise SystemExit(f"--ssh-session-token-file has an unsafe {what}")
  649. if uid is not None and st.st_uid != uid:
  650. raise SystemExit(f"--ssh-session-token-file {what} has the wrong owner")
  651. if what == "parent directory" and (st.st_mode & 0o777) != 0o700:
  652. raise SystemExit("--ssh-session-token-file parent has unsafe permissions")
  653. root_fd = -1
  654. directory_fd = -1
  655. file_fd = -1
  656. try:
  657. try:
  658. root_fd = os.open(token_root, directory_flags)
  659. _check_dir(root_fd, "runtime root")
  660. directory_fd = os.open(relative.parts[0], directory_flags, dir_fd=root_fd)
  661. _check_dir(directory_fd, "parent directory")
  662. file_fd = os.open(relative.parts[1], file_flags, dir_fd=directory_fd)
  663. except SystemExit:
  664. raise
  665. except OSError as exc:
  666. if exc.errno == getattr(__import__("errno"), "ELOOP", -1):
  667. raise SystemExit("--ssh-session-token-file is a symlink") from exc
  668. raise SystemExit("--ssh-session-token-file is not accessible") from exc
  669. file_stat = os.fstat(file_fd)
  670. if not _stat.S_ISREG(file_stat.st_mode):
  671. raise SystemExit("--ssh-session-token-file is not a regular file")
  672. if file_stat.st_size != 64:
  673. raise SystemExit("--ssh-session-token-file contains an invalid token")
  674. if uid is not None and file_stat.st_uid != uid:
  675. raise SystemExit("--ssh-session-token-file has the wrong owner")
  676. if uid is not None and (file_stat.st_mode & 0o777) & ~0o600:
  677. raise SystemExit("--ssh-session-token-file has unsafe permissions")
  678. with os.fdopen(file_fd, "r", encoding="utf-8-sig") as token_stream:
  679. file_fd = -1
  680. token = token_stream.read(65)
  681. if not re.fullmatch(r"[0-9a-f]{64}", token):
  682. raise SystemExit("--ssh-session-token-file contains an invalid token")
  683. return token
  684. finally:
  685. if file_fd >= 0:
  686. os.close(file_fd)
  687. if directory_fd >= 0:
  688. with contextlib.suppress(OSError):
  689. os.unlink(relative.parts[1], dir_fd=directory_fd)
  690. os.close(directory_fd)
  691. if root_fd >= 0:
  692. os.close(root_fd)
  693. def _is_electron_packaged_web_dist(path: str) -> bool:
  694. """True when *path* is an Electron-packaged renderer dist (``app.asar[.unpacked]/dist``).
  695. A standalone ``hermes dashboard`` inheriting that ``HERMES_WEB_DIST`` would
  696. serve the desktop frontend in the browser ("Desktop IPC bridge is unavailable").
  697. """
  698. if not path:
  699. return False
  700. return "app.asar" in path.replace("\\", "/")
  701. def _host_backend_attachment():
  702. """Live host serve/dashboard record to attach to, or ``None``.
  703. Record-based discovery replaces the old bare-TCP probe: "something accepts a connection on
  704. this port" proved nothing about WHO answers (a foreign service, or a recycled PID's new
  705. owner). The record carries ``(pid, createTime)`` so liveness is proved against the same
  706. incarnation, and its token fingerprint must still match the 0600 token file the owner wrote.
  707. The record only nominates a CANDIDATE; :func:`_attach_to_host_backend` makes it prove itself.
  708. """
  709. try:
  710. from gateway import host_rendezvous as hr
  711. record = hr.read_record(hr.ROLE_SERVE)
  712. if record is None or not record.port:
  713. return None
  714. return record if hr.record_token_is_consistent(record) else None
  715. except Exception:
  716. return None
  717. def _explicit_endpoint_flags(argv=None) -> set:
  718. """Which of ``--host``/``--port`` the operator actually typed.
  719. argparse defaults are indistinguishable from a typed value in ``args``, and the difference is
  720. load-bearing: an unset ``--port`` may attach to whatever port the host owner bound, but a
  721. typed ``--port 8899`` or ``--host 0.0.0.0`` (LAN access) must never be silently answered with
  722. a loopback attach on some other port.
  723. """
  724. typed = set()
  725. for token in (sys.argv[1:] if argv is None else argv):
  726. name = str(token).split("=", 1)[0]
  727. if name in ("--host", "--port"):
  728. typed.add(name[2:])
  729. return typed
  730. def _endpoint_conflict(args, record, typed: set) -> str:
  731. """Why an explicitly requested endpoint cannot be served by ``record`` ('' when it can)."""
  732. if "port" in typed:
  733. wanted_port = getattr(args, "port", None)
  734. # ``--port 0`` is "any free port", not a demand for a specific one.
  735. if isinstance(wanted_port, int) and wanted_port > 0 and wanted_port != record.port:
  736. return f"--port {wanted_port} (the host owner is on port {record.port})"
  737. if "host" in typed:
  738. wanted_host = str(getattr(args, "host", "") or "")
  739. owner_host = record.host or "127.0.0.1"
  740. loopback = {"127.0.0.1", "localhost", "::1"}
  741. wildcard = {"0.0.0.0", "::", "*"}
  742. # A wildcard owner already answers on loopback; anything else must match exactly.
  743. reachable = wanted_host == owner_host or (owner_host in wildcard and wanted_host in loopback)
  744. if not reachable:
  745. return f"--host {wanted_host} (the host owner is bound to {owner_host})"
  746. return ""
  747. def _attach_to_host_backend(args, headless_backend: bool) -> None:
  748. """Multiplex-only: a second `hermes serve`/`dashboard` attaches to the host backend.
  749. Exactly ONE backend runs per host and multiplexes every profile, so a second invocation —
  750. for ANY profile, the default included — reports the live one and exits 0 instead of binding
  751. a second port. ``--isolated`` opts out (Desktop's SSH backend proves ownership with it) and
  752. Desktop pool backends (HERMES_DESKTOP=1) keep their own lifecycle.
  753. Exit 0 means "the host backend answered and serves what you asked for", and nothing else:
  754. * the owner must ANSWER on its recorded port and identify itself (a record alone cannot see a
  755. graceful-shutdown window or a foreign listener that inherited the port) — a supervisor or
  756. `hermes update` relaunch landing in that window would otherwise exit 0 with NOTHING
  757. listening, reporting success for a dead service;
  758. * an explicitly typed ``--port``/``--host`` the owner cannot serve is a REFUSAL naming the
  759. owner, never a silent redirect. It exits 78 (EX_CONFIG), the deliberate-refusal code
  760. ``RestartPreventExitStatus=78`` parks on: exit 1 under ``Restart=always`` was an infinite
  761. restart loop with nothing listening on the ingress port (#119824);
  762. * a `hermes dashboard` user is never handed a headless backend's URL (no SPA behind it).
  763. Returns normally — leaving the caller to BIND — when no owner answers.
  764. """
  765. if getattr(args, "isolated", False) or _is_desktop_owned_backend():
  766. return
  767. record = _host_backend_attachment()
  768. if record is None:
  769. return
  770. from gateway import host_rendezvous as hr
  771. from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
  772. identity = hr.probe_owner(record)
  773. if identity is None:
  774. # Unprovable liveness (no psutil), a closed port, a foreign listener: all mean "no owner
  775. # answered". Fall through to the bind — never exit 0 on an attach that did not happen.
  776. return
  777. typed = _explicit_endpoint_flags()
  778. conflict = _endpoint_conflict(args, record, typed)
  779. if conflict:
  780. print(f"Refusing to start: this host is already served by {hr.describe(record)}.")
  781. print(f" You asked for {conflict}.")
  782. print(" Stop that backend, or drop the flag to use the running one.")
  783. sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
  784. if not headless_backend and not identity.get("servesSpa"):
  785. print(f"Refusing to start: this host is already served by {hr.describe(record)}, "
  786. "which is a headless `hermes serve` backend with no dashboard UI.")
  787. print(" Stop it and run `hermes dashboard`, or use --isolated for a dedicated server.")
  788. sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
  789. try:
  790. from hermes_cli.profiles import get_active_profile_name
  791. profile = get_active_profile_name()
  792. except Exception:
  793. profile = "default"
  794. wanted = getattr(args, "open_profile", "") or profile
  795. from hermes_cli.url_utils import format_url_host
  796. url = f"http://{format_url_host(hr.dial_host(record))}:{record.port}/?profile={wanted}"
  797. kind = "backend" if headless_backend else "dashboard"
  798. print(f"Hermes {kind} already running on this host: PID {record.pid}, port {record.port}.")
  799. print(f" Managing profile '{wanted}': {url}")
  800. if not headless_backend and not args.no_open:
  801. with contextlib.suppress(Exception):
  802. import webbrowser
  803. webbrowser.open(url)
  804. sys.exit(0)
  805. def _route_named_profile_dashboard(
  806. args, _headless_backend: bool, _ssh_owner_nonce: str, _token_file: str) -> None:
  807. """Route a named-profile launch to the single MACHINE dashboard (per-request ``?profile=`` scoping
  808. makes one server per profile pure fragmentation).
  809. No-record fallback to :func:`_attach_to_host_backend`, which already attached (and exited)
  810. when the host publishes a live rendezvous record: re-exec pinned to ``-p default`` (so
  811. ``_apply_profile_override`` can't re-route via the sticky active_profile file). ``--isolated``
  812. opts out; Desktop pool backends (HERMES_DESKTOP=1) stay per-profile. Returns normally when no
  813. routing applies.
  814. """
  815. try:
  816. from hermes_cli.profiles import get_active_profile_name
  817. _launch_profile = get_active_profile_name()
  818. except Exception:
  819. _launch_profile = "default"
  820. if (
  821. _launch_profile in ("default", "custom")
  822. or getattr(args, "isolated", False)
  823. or getattr(args, "open_profile", "")
  824. or _is_desktop_owned_backend()
  825. ):
  826. return
  827. print(
  828. f"Routing to the machine dashboard (profile '{_launch_profile}' "
  829. f"preselected). Use --isolated for a dedicated per-profile server."
  830. )
  831. reexec_argv = [
  832. sys.executable, "-m", "hermes_cli.main",
  833. "-p", "default",
  834. # Preserve the lean serve path so a named-profile `serve` doesn't
  835. # silently rebuild the UI as `dashboard`.
  836. "serve" if _headless_backend else "dashboard",
  837. "--port", str(args.port),
  838. "--host", args.host,
  839. "--open-profile", _launch_profile]
  840. for enabled, extra in (
  841. (_ssh_owner_nonce, ["--ssh-owner-nonce", _ssh_owner_nonce]),
  842. (_token_file, ["--ssh-session-token-file", _token_file]),
  843. (args.no_open, ["--no-open"]),
  844. (getattr(args, "insecure", False), ["--insecure"]),
  845. (getattr(args, "skip_build", False), ["--skip-build"])):
  846. if enabled:
  847. reexec_argv.extend(extra)
  848. from tools.environments.local import build_subprocess_env
  849. # HERMES_HOME is pinned to the machine root below — the factory must not
  850. # re-inject a profile home.
  851. env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False)
  852. # Pin the child to the machine ROOT, resolved explicitly rather than by
  853. # dropping HERMES_HOME: in the Docker layout the root is /opt/data, and an
  854. # unset HERMES_HOME would fall back to $HOME/.hermes = /opt/data/.hermes — an
  855. # empty auto-seeded home with only the default profile and no install stamp.
  856. # get_default_hermes_root() strips a trailing profiles/<name> for both layouts.
  857. try:
  858. from hermes_constants import get_default_hermes_root
  859. env["HERMES_HOME"] = str(get_default_hermes_root())
  860. except Exception:
  861. env.pop("HERMES_HOME", None) # prior behaviour rather than blocking the reroute
  862. # On Windows os.execvpe() spawns via CreateProcess then exits, which under
  863. # Python 3.14+ can crash with STATUS_ACCESS_VIOLATION; use Popen + exit.
  864. if sys.platform == "win32":
  865. proc = subprocess.Popen(reexec_argv, env=env)
  866. sys.exit(proc.wait())
  867. else:
  868. from hermes_cli.cpu_profile import write_cpu_profile
  869. write_cpu_profile() # exec skips atexit
  870. os.execvpe(sys.executable, reexec_argv, env)
  871. def _resolve_dashboard_web_dist(args, _headless_backend: bool) -> None:
  872. """Build or validate the web UI dist before the server imports.
  873. ``serve`` sets HERMES_SERVE_HEADLESS so mount_spa() stays off. Otherwise build
  874. unless HERMES_WEB_DIST / --skip-build promise a dist — then verify index.html
  875. (else the server serves 404s). --skip-build on the default location gets ONE
  876. recovery build; a caller-managed HERMES_WEB_DIST can't be populated.
  877. """
  878. from hermes_cli.main import PROJECT_ROOT
  879. from hermes_cli.main_web_build import _build_web_ui
  880. skip_build = getattr(args, "skip_build", False)
  881. if _headless_backend:
  882. os.environ["HERMES_SERVE_HEADLESS"] = "1" # set before web_server import
  883. elif "HERMES_WEB_DIST" not in os.environ and not skip_build:
  884. if not _build_web_ui(PROJECT_ROOT / "web", fatal=True):
  885. sys.exit(1)
  886. elif skip_build:
  887. _dist_root = (
  888. # --build-mode skip trusts the caller to have pre-built the web UI. Verify the dist actually
  889. # exists; otherwise the server will start and serve 404s with no obvious cause (issue #23817).
  890. Path(os.environ["HERMES_WEB_DIST"])
  891. if "HERMES_WEB_DIST" in os.environ
  892. else PROJECT_ROOT / "hermes_cli" / "web_dist"
  893. )
  894. if not (_dist_root / "index.html").exists():
  895. # Only the default dist location is recoverable (desktop launches with
  896. # --build-mode skip after a wipe of web_dist); a custom HERMES_WEB_DIST
  897. # is a caller-managed directory the build cannot populate.
  898. # The caller promised a pre-built dist but there isn't one. Instead of hard-failing (issue
  899. # #59288 — desktop launches with --build-mode skip after a wipe of web_dist), warn and attempt
  900. # ONE recovery build through the normal build path.
  901. _recoverable = "HERMES_WEB_DIST" not in os.environ
  902. if _recoverable:
  903. print(f"⚠ --skip-build was passed but no web dist found at: {_dist_root}")
  904. print(" Attempting one recovery build of the web UI...")
  905. _build_web_ui(PROJECT_ROOT / "web", fatal=True)
  906. if not (_dist_root / "index.html").exists():
  907. print(f"✗ --skip-build was passed but no web dist found at: {_dist_root}")
  908. if _recoverable:
  909. print(" The recovery build did not produce a usable dist.")
  910. print(_PRE_BUILD_HINT)
  911. print(" Or drop --skip-build to build automatically.")
  912. sys.exit(1)
  913. print(" ✓ Recovery build produced a web dist")
  914. print(f"→ Skipping web UI build (--skip-build); using dist at {_dist_root}")
  915. else:
  916. # HERMES_WEB_DIST without --skip-build: the env var points at a
  917. # caller-managed dist, so validate it like the --skip-build branch.
  918. # HERMES_WEB_DIST is set without --skip-build: the build is skipped (the env var points at a
  919. # caller-managed dist), so validate it the same way the --skip-build branch does — otherwise the
  920. # server starts and serves 404s with no obvious cause (same failure mode as #23817, via the env-var
  921. # path).
  922. _dist_root = Path(os.environ["HERMES_WEB_DIST"]).expanduser()
  923. if not (_dist_root / "index.html").exists():
  924. print(f"✗ HERMES_WEB_DIST is set but no web dist found at: {_dist_root}")
  925. print(_PRE_BUILD_HINT)
  926. print(" Or unset HERMES_WEB_DIST to build and use the default web UI dist.")
  927. sys.exit(1)
  928. # web_server reads HERMES_WEB_DIST raw at import (no expanduser), so a
  929. # validated "~/dist" would otherwise pass here and still 404 there.
  930. os.environ["HERMES_WEB_DIST"] = str(_dist_root)
  931. print(f"→ Using web dist from HERMES_WEB_DIST: {_dist_root}")
  932. def _dashboard_validate_serve_args(args, headless_backend, token_file):
  933. """Headless-serve argument checks -> ssh_owner_nonce (or None)."""
  934. # `hermes serve` is headless/non-interactive: fail closed on a corrupt
  935. # config.yaml instead of silently starting on defaults where provider
  936. # auto-detection can adopt unnamed .env credentials (issue #81952).
  937. # Same policy + escape hatch as _guard_noninteractive_user_config.
  938. if headless_backend:
  939. from hermes_cli.config import (
  940. InvalidUserConfigError,
  941. require_parseable_user_config,
  942. )
  943. try:
  944. require_parseable_user_config(
  945. ignore_user_config=bool(getattr(args, "ignore_user_config", False))
  946. )
  947. except InvalidUserConfigError as exc:
  948. print(f"Error: {exc}", file=sys.stderr)
  949. raise SystemExit(2) from exc
  950. ssh_owner_nonce = getattr(args, "ssh_owner_nonce", None)
  951. if ssh_owner_nonce and not re.fullmatch(r"[0-9a-f]{16}", ssh_owner_nonce):
  952. raise SystemExit("--ssh-owner-nonce must be 16 lowercase hex characters")
  953. if token_file and not headless_backend:
  954. raise SystemExit("--ssh-session-token-file is only valid with hermes serve")
  955. return ssh_owner_nonce