web_server.py 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528
  1. """Hermes Agent — Web UI server: FastAPI app assembly, auth/host middleware, ``start_server``.
  2. Route handlers live in ``web_routers/``; their helpers live in the sibling
  3. ``web_server_<concern>`` modules and are re-imported here so ``web_server.<name>``
  4. stays the single late-binding seam tests monkeypatch (``web_deps.late``).
  5. Usage: ``python -m hermes_cli.main web [--port 8080]``.
  6. """
  7. from contextlib import asynccontextmanager
  8. import asyncio
  9. from collections import deque
  10. import hmac
  11. import logging
  12. import os
  13. import re
  14. import secrets
  15. import subprocess
  16. import sys
  17. import sysconfig
  18. import threading
  19. import time
  20. import urllib.parse
  21. from hermes_cli.install_identity import get_install_id as _shared_get_install_id
  22. from hermes_cli.process_identity import is_desktop_owned_backend
  23. from hermes_cli.pty_session import run_reaper
  24. from hermes_cli import web_server_file_tickets as _file_tickets
  25. from hermes_cli.web_server_session_token import resolve_session_token
  26. from pathlib import Path
  27. from typing import Any, Dict, Optional, Tuple
  28. PROJECT_ROOT = Path(__file__).parent.parent.resolve()
  29. if str(PROJECT_ROOT) not in sys.path:
  30. sys.path.insert(0, str(PROJECT_ROOT))
  31. from hermes_constants import WEBAPP_ATTACHMENT_MAX_BYTES
  32. from hermes_cli.config import load_config
  33. from hermes_cli.version_info import get_version_info
  34. try:
  35. from fastapi import FastAPI, HTTPException, Request
  36. from fastapi.middleware.cors import CORSMiddleware
  37. from fastapi.responses import JSONResponse
  38. except ImportError:
  39. # First try lazy-installing the dashboard extras. Only the user actually
  40. # running `hermes dashboard` needs fastapi+uvicorn; lazy install keeps
  41. # them out of every other install path. After install, re-import.
  42. try:
  43. from pm import ensure_import
  44. ensure_import("web")
  45. from fastapi import (
  46. FastAPI, HTTPException, Request,
  47. )
  48. from fastapi.middleware.cors import CORSMiddleware
  49. from fastapi.responses import JSONResponse
  50. except Exception:
  51. raise SystemExit(
  52. "Web UI requires fastapi and uvicorn.\n"
  53. "Run hermes pm repair, then restart Hermes."
  54. )
  55. WEB_DIST = Path(os.environ["HERMES_WEB_DIST"]) if "HERMES_WEB_DIST" in os.environ else Path(__file__).parent / "web_dist"
  56. _log = logging.getLogger(__name__)
  57. from hermes_cli.web_server_lifecycle import (
  58. PORT_IN_USE_EXIT_CODE,
  59. _dashboard_forwarded_allow_ips,
  60. _eager_reconcile_own_session_db,
  61. _maybe_open_browser,
  62. _port_bind_conflict,
  63. _read_bound_port,
  64. _report_port_in_use,
  65. _start_parent_death_watchdog,
  66. _uvicorn_console_logging,
  67. _warm_gateway_module,
  68. _write_dashboard_ready_file,
  69. _write_machine_sentinel_line,
  70. )
  71. def _gateway_owns_cron(name: str, home) -> bool:
  72. """A gateway already ticks this profile's store with live adapters: its OWN process, or the
  73. live default multiplexer (a served satellite has no gateway.pid of its own). Winning the
  74. tick-lock race here would deliver through the standalone path (#52202, #100489, #107485)."""
  75. from hermes_cli.profiles import _check_gateway_running, _served_by_running_multiplexer
  76. return _check_gateway_running(Path(home)) or (
  77. name != "default" and _served_by_running_multiplexer(name))
  78. def _start_desktop_cron_ticker(stop_event: threading.Event, interval: int = 60) -> None:
  79. """Tick the cron scheduler from inside the desktop dashboard backend.
  80. The desktop spawns a ``hermes dashboard`` backend, not a gateway, so without
  81. this a cron created in the app would never fire (no live adapters; delivery
  82. falls back to the per-platform send path). The primary backend outlives the
  83. per-profile pool (reaped after ~10 idle minutes), so it ticks EVERY local
  84. profile's store like a multiplex gateway; external providers keep the
  85. single-store behavior (registries are not profile-scoped). Cross-process
  86. safe: the built-in tick takes the per-store ``cron/.tick.lock``.
  87. Every local profile's store is ticked, not just this backend's own (#69377's desktop sibling): the
  88. desktop pools per-profile backends and reaps them after ~10 idle minutes, so a secondary profile's
  89. ticker dies with its backend and that profile's jobs silently stop firing until the user next opens it
  90. ("tasks on the sleeping profile could be idle" — community report, Aug 2026).
  91. """
  92. from cron.scheduler_provider import InProcessCronScheduler, resolve_cron_scheduler
  93. from hermes_constants import get_hermes_home, profile_name_for_home
  94. provider = resolve_cron_scheduler()
  95. own_home = Path(get_hermes_home())
  96. own_name = profile_name_for_home(own_home) or "default"
  97. # Ownership is re-checked every tick, not once at startup, so Desktop takes over when the
  98. # gateway stops (#126822).
  99. profile_gate = lambda name, home: not _gateway_owns_cron(name, home)
  100. start_kwargs: dict = {"interval": interval}
  101. if isinstance(provider, InProcessCronScheduler):
  102. try:
  103. from hermes_cli.profiles import profiles_to_serve
  104. # Same served set as the multiplexer: default + every live profile under profiles/.
  105. # The ticker re-enumerates this callable every cycle. Passing a
  106. # startup snapshot leaves deleted profiles in the scheduler until
  107. # restart, which both writes their removed stores and keeps stale
  108. # profiles alive in Desktop's background work.
  109. profile_homes = lambda: list(profiles_to_serve(multiplex=True))
  110. initial_profile_homes = profile_homes()
  111. if initial_profile_homes:
  112. # Even one profile needs the per-tick gateway gate; otherwise
  113. # Desktop races its dedicated gateway for the same cron store.
  114. start_kwargs["profile_homes"] = profile_homes
  115. start_kwargs["profile_gate"] = profile_gate
  116. from hermes_logging import enable_profile_log_routing
  117. enable_profile_log_routing(initial_profile_homes)
  118. _log.info(
  119. "Desktop cron scheduler will tick %d profile(s): %s",
  120. len(initial_profile_homes),
  121. [name for name, _home in initial_profile_homes],
  122. )
  123. except Exception:
  124. # Fail open to the single-store ticker so the active profile keeps firing.
  125. _log.exception("Desktop cron: profile enumeration failed; ticking active profile only")
  126. if "profile_homes" not in start_kwargs:
  127. # Fail open to this backend's own store behind the same gate. A gated-out profile is
  128. # neither ticked nor heartbeated, so Desktop never marks the gateway's store healthy.
  129. start_kwargs["profile_homes"] = lambda: [(own_name, own_home)]
  130. start_kwargs["profile_gate"] = profile_gate
  131. else:
  132. # External providers take no per-tick gate: defer their start until the gateway is gone.
  133. def _owned() -> bool:
  134. try:
  135. return _gateway_owns_cron(own_name, own_home)
  136. except Exception:
  137. # Start the ticker rather than silently stand down.
  138. _log.warning("Desktop cron: gateway-ownership probe failed; starting the ticker", exc_info=True)
  139. return False
  140. if _owned():
  141. _log.info(
  142. "Desktop cron scheduler waiting: live gateway owns cron on this HERMES_HOME; "
  143. "the gateway ticks with live adapters (re-probing every %ds)", interval,
  144. )
  145. while True:
  146. if stop_event.wait(interval):
  147. return
  148. if not _owned():
  149. break
  150. _log.info("Desktop cron scheduler started (provider=%s, interval=%ds)", provider.name, interval)
  151. provider.start(stop_event, **start_kwargs)
  152. # Desktop `serve` only (start_server(start_mcp_discovery_after_bind=True)):
  153. # seconds after the READY sentinel before the MCP discovery thread starts.
  154. _DESKTOP_MCP_DISCOVERY_DELAY_S = 1.0
  155. from hermes_cli.web_server_app import app_lifespan
  156. def _app_state_default(app: FastAPI, name: str, factory):
  157. """Return ``app.state.<name>``, lazily creating it for non-``with`` TestClient usages.
  158. The lifespan normally initialises these on the running event loop (an
  159. asyncio.Lock created at import time binds to whatever loop was active then).
  160. """
  161. try:
  162. return getattr(app.state, name)
  163. except AttributeError:
  164. value = factory()
  165. setattr(app.state, name, value)
  166. return value
  167. def _get_chat_argv_lock(app: FastAPI) -> asyncio.Lock:
  168. return _app_state_default(app, "chat_argv_lock", asyncio.Lock)
  169. def _get_pty_active_session_files(app: FastAPI) -> dict[str, Path]:
  170. return _app_state_default(app, "pty_active_session_files", dict)
  171. app = FastAPI(title="Hermes Agent", version=get_version_info().base_version, lifespan=app_lifespan)
  172. from hermes_cli.dashboard_auth.body_limit import AuthBodyLimitMiddleware
  173. # Register first (innermost): auth gates run before this, JSON parsing after it.
  174. app.add_middleware(AuthBodyLimitMiddleware)
  175. # Memory-provider OAuth connect routes live in the memory layer, not here.
  176. from hermes_cli.memory_oauth import router as _memory_oauth_router
  177. app.include_router(_memory_oauth_router)
  178. # Session token for sensitive endpoints. Resolution lives in the
  179. # web_server_session_token sibling; it dies with the process and is injected
  180. # into the SPA HTML so only the web UI can use it.
  181. _SESSION_TOKEN = resolve_session_token()
  182. _SESSION_HEADER_NAME = "X-Hermes-Session-Token"
  183. _SSH_OWNER_NONCE: Optional[str] = None
  184. _SSH_RUNTIME_PURELIB: Optional[tuple[str, int, int]] = None
  185. _SSH_RUNTIME_MARKER: Optional[str] = None
  186. def _apply_ssh_session_token(token: str) -> None:
  187. global _SESSION_TOKEN
  188. if token:
  189. _SESSION_TOKEN = token
  190. def _apply_ssh_owner_nonce(nonce: Optional[str]) -> None:
  191. global _SSH_OWNER_NONCE, _SSH_RUNTIME_PURELIB, _SSH_RUNTIME_MARKER
  192. _SSH_OWNER_NONCE = nonce
  193. _SSH_RUNTIME_PURELIB = None
  194. _SSH_RUNTIME_MARKER = None
  195. if nonce:
  196. try:
  197. purelib = sysconfig.get_paths()["purelib"]
  198. except (KeyError, OSError):
  199. return
  200. # Primary identity: a marker FILE in site-packages. A replaced venv
  201. # loses it deterministically; pip installs leave it. A bare (dev, ino)
  202. # snapshot alone is NOT enough: ext4 reuses directory inodes at once,
  203. # so `rm -rf venv && uv venv` can land on the same inode undetected.
  204. try:
  205. marker = os.path.join(purelib, f".hermes-ssh-runtime-{nonce}")
  206. with open(marker, "w", encoding="utf-8") as fh:
  207. fh.write(f"pid={os.getpid()}\n")
  208. _SSH_RUNTIME_MARKER = marker
  209. except OSError:
  210. pass # read-only site-packages — fall back to the stat snapshot
  211. try:
  212. st = os.stat(purelib)
  213. _SSH_RUNTIME_PURELIB = (purelib, st.st_dev, st.st_ino)
  214. except OSError:
  215. pass
  216. def _ssh_runtime_intact() -> bool:
  217. if _SSH_RUNTIME_MARKER is not None:
  218. return os.path.isfile(_SSH_RUNTIME_MARKER)
  219. # Fallback (read-only site-packages): directory identity snapshot — weaker
  220. # (inode reuse) but catches cross-device moves and version-bump paths.
  221. if _SSH_RUNTIME_PURELIB is None:
  222. return True
  223. purelib, device, inode = _SSH_RUNTIME_PURELIB
  224. try:
  225. st = os.stat(purelib)
  226. except OSError:
  227. return False
  228. return (st.st_dev, st.st_ino) == (device, inode)
  229. # In-browser Chat tab (/chat, /api/pty, /api/ws): always enabled. A module
  230. # constant (not an inlined True) so the WS endpoints and SPA token injection
  231. # share one testable seam.
  232. _DASHBOARD_EMBEDDED_CHAT_ENABLED = True
  233. from hermes_cli.web_upload_limit import ChatFileUploadLimitMiddleware
  234. # Inside every auth/Host middleware: those checks run before body ingestion,
  235. # and the receive limit runs before FastAPI can spool a multipart attachment.
  236. app.add_middleware(ChatFileUploadLimitMiddleware)
  237. # Endpoints that do NOT require the session token; everything else under /api/
  238. # is gated below. Shared with the OAuth gate so the two allowlists cannot
  239. # drift (/api/status once 401'd under the OAuth gate, breaking the portal probe).
  240. from hermes_cli.dashboard_auth.public_paths import PUBLIC_API_PATHS as _PUBLIC_API_PATHS
  241. def _has_valid_session_token(request: Request) -> bool:
  242. """True if the request carries a valid dashboard session token.
  243. The dedicated header avoids collisions with reverse proxies that already use
  244. ``Authorization`` (Caddy ``basic_auth``); the legacy Bearer path stays for
  245. older dashboard bundles.
  246. """
  247. session_header = request.headers.get(_SESSION_HEADER_NAME, "")
  248. if session_header and hmac.compare_digest(session_header.encode(), _SESSION_TOKEN.encode()):
  249. return True
  250. auth = request.headers.get("authorization", "")
  251. return hmac.compare_digest(auth.encode(), f"Bearer {_SESSION_TOKEN}".encode())
  252. # Routes that may also authenticate via ``?token=`` for Electron's remote
  253. # "open externally" download link. Browser pages use path-bound file tickets
  254. # instead (``web_server_file_tickets``), so the session token never lands in
  255. # their download history or a copyable media URL. The OAuth gate still requires
  256. # its session cookie; query credentials are only accepted by the loopback gate.
  257. _QUERY_TOKEN_API_PATHS: frozenset[str] = frozenset({"/api/files/download"})
  258. def _has_valid_query_token(request: Request, path: str) -> bool:
  259. if _file_tickets.redeem(request):
  260. return True
  261. if path not in _QUERY_TOKEN_API_PATHS:
  262. return False
  263. token = request.query_params.get("token", "")
  264. return bool(token) and hmac.compare_digest(token.encode(), _SESSION_TOKEN.encode())
  265. def _require_token(request: Request) -> None:
  266. """Authorize a sensitive endpoint, raising 401 if the caller isn't allowed.
  267. Loopback mode (``auth_required`` False): validate the SPA-injected
  268. ``_SESSION_TOKEN``. Gated mode: the token is NOT injected (cookie auth), and
  269. ``gated_auth_middleware`` already 401'd anything without a verified
  270. ``request.state.session`` — requiring the absent token here would make every
  271. ``_require_token`` endpoint unreachable behind the gate, so defer to it.
  272. A verified native owner (``authenticate_native_http``) is the same-user
  273. principal the outer seams already accepted; local Desktop runs with no
  274. static token, so that principal must satisfy route-local policy too.
  275. Hosted plugins receive this guard's decision via private IPC and request-local
  276. state, not via a forwarded gateway credential or a process-wide bypass.
  277. """
  278. if (getattr(request.state, "native_http_principal", None)
  279. or getattr(request.state, "hosted_http_authenticated", None) is True):
  280. return
  281. if getattr(request.app.state, "auth_required", False):
  282. ok = getattr(request.state, "session", None) is not None
  283. else:
  284. ok = _has_valid_session_token(request)
  285. if not ok:
  286. raise HTTPException(status_code=401, detail="Unauthorized")
  287. from hermes_cli.web_server_boundary import ( # re-exported: web_server.<name> stays the public seam
  288. _DESKTOP_ATTACHMENT_WS_MAX_BYTES,
  289. _LOOPBACK_HOST_VALUES,
  290. _dashboard_public_hosts,
  291. _dashboard_public_origins,
  292. _desktop_loopback_auth_exempt,
  293. _host_header_hostname,
  294. _is_accepted_host,
  295. public_hosts_engage_gate,
  296. public_url_hosts,
  297. public_url_origins,
  298. should_require_auth,
  299. )
  300. def should_require_dashboard_auth(
  301. host: str,
  302. trusted_public_hosts: Optional[frozenset[str]] = None,
  303. ) -> bool:
  304. """Gate required for a non-loopback bind OR a non-loopback ``dashboard.public_url``.
  305. Callers may pass the already-resolved host set so startup and request
  306. validation share one snapshot.
  307. """
  308. if trusted_public_hosts is None:
  309. trusted_public_hosts = _dashboard_public_hosts()
  310. return should_require_auth(host) or public_hosts_engage_gate(trusted_public_hosts)
  311. @app.middleware("http")
  312. async def host_header_middleware(request: Request, call_next):
  313. """Reject requests whose Host header doesn't match the bound interface (DNS rebinding, GHSA-ppp5-vxwm-4cf7)."""
  314. # app.state.bound_host is set by start_server() at listen time.
  315. bound_host = getattr(app.state, "bound_host", None)
  316. if bound_host and not _is_accepted_host(
  317. request.headers.get("host", ""), bound_host, getattr(app.state, "trusted_public_hosts", frozenset())
  318. ):
  319. return JSONResponse(
  320. status_code=400,
  321. content={
  322. "detail": (
  323. "Invalid Host header. Dashboard requests must use the "
  324. "bound hostname or the configured public hostname."
  325. ),
  326. },
  327. )
  328. return await call_next(request)
  329. # Imported here, beside the plugin gates that use it, like the other mid-module web_server_*
  330. # groups. Not an import cycle: web_server_dashboard reaches back into this module only inside
  331. # functions, so the position is not load-bearing. What is load-bearing is the middleware
  332. # registration order below (the HTTP gate before the auth middlewares, so it runs after them).
  333. from hermes_cli.web_server_dashboard import (
  334. PluginApiWebSocketGate, _plugin_api_name, _plugin_api_request_refused,
  335. )
  336. from hermes_cli.web_server_plugin_profiles import forward_secondary_plugin_request, plugin_request_scope
  337. @app.middleware("http")
  338. async def _plugin_api_runtime_gate(request: Request, call_next):
  339. """Block requests to disabled plugin API routes at request time
  340. (:func:`_plugin_api_request_refused`).
  341. Registered BEFORE the auth middlewares (runs AFTER them): an unauthenticated
  342. caller must get auth's 401, never this 404, or the status code becomes a
  343. plugin-name oracle.
  344. """
  345. path = request.url.path
  346. plugin_name = _plugin_api_name(request.scope)
  347. # Only gate authenticated requests. Unauthenticated ones fall through so
  348. # auth_middleware / the OAuth gate return 401 first and this route can't
  349. # be used as a plugin-name oracle. Desktop authenticates with a native ticket.
  350. if plugin_name and (
  351. getattr(request.state, "token_authenticated", False)
  352. or getattr(request.state, "native_http_principal", None)
  353. or getattr(request.app.state, "auth_required", False)
  354. or _has_valid_session_token(request)
  355. or _has_valid_query_token(request, path)
  356. ):
  357. with plugin_request_scope(request.scope): # a secondary profile's plugins run in its host
  358. if await _plugin_api_request_refused(plugin_name):
  359. return JSONResponse(status_code=404, content={"detail": "Plugin not found"})
  360. hosted = await forward_secondary_plugin_request(request, plugin_name)
  361. if hosted is not None:
  362. return hosted
  363. return await call_next(request)
  364. # The same gate for WebSocket upgrades: ``@app.middleware("http")`` passes those through untouched.
  365. app.add_middleware(PluginApiWebSocketGate)
  366. @app.middleware("http")
  367. async def _dashboard_auth_gate(request: Request, call_next):
  368. """OAuth gate — active only when start_server flags ``auth_required``; pass-through on loopback.
  369. Registered between host_header and auth_middleware: host check → cookie auth → token auth.
  370. """
  371. from hermes_cli.dashboard_auth.middleware import gated_auth_middleware
  372. return await gated_auth_middleware(request, call_next)
  373. @app.middleware("http")
  374. async def auth_middleware(request: Request, call_next):
  375. """Require the session token on all /api/ routes except the public list.
  376. Skipped for requests the token-auth seam already authenticated
  377. (``token_authenticated``) and when the OAuth gate is active — cookie auth is
  378. then authoritative and the loopback-only token path must not override it.
  379. """
  380. path = request.url.path
  381. if (
  382. not getattr(request.state, "token_authenticated", False)
  383. and not getattr(request.state, "native_http_principal", None)
  384. and not getattr(request.app.state, "auth_required", False)
  385. and path.startswith("/api/")
  386. and path not in _PUBLIC_API_PATHS
  387. and not path.startswith("/api/mcp/oauth/callback/")
  388. and not _has_valid_session_token(request)
  389. and not _has_valid_query_token(request, path)
  390. ):
  391. return JSONResponse(status_code=401, content={"detail": "Unauthorized"})
  392. return await call_next(request)
  393. @app.middleware("http")
  394. async def _token_auth_seam(request: Request, call_next):
  395. """Outermost auth seam: bearer-token auth for opted-in routes (registered LAST = runs FIRST).
  396. A registered token route is owned here — authenticate, attach the principal
  397. + ``token_authenticated`` so downstream gates skip enforcement. Non-token
  398. routes pass through untouched.
  399. """
  400. from hermes_cli.dashboard_auth.native_http import authenticate_native_http, native_profile_scope
  401. from hermes_cli.dashboard_auth.token_auth import token_auth_middleware
  402. rejection = await authenticate_native_http(request)
  403. if rejection is not None:
  404. return rejection
  405. with native_profile_scope(request):
  406. return await token_auth_middleware(request, call_next)
  407. _DASHBOARD_HEALTH_WINDOW_SECONDS = 300.0
  408. class DashboardHealth:
  409. """Dashboard-process health: rolling unhandled-error/5xx window + periodic self-test result.
  410. Feeds ``components`` on the PUBLIC ``/api/status``, so :meth:`snapshot`
  411. exports counts and enums only — never ``last_error_type``/``last_error_path``.
  412. """
  413. def __init__(self, window_seconds: float = _DASHBOARD_HEALTH_WINDOW_SECONDS) -> None:
  414. self.window_seconds = window_seconds
  415. self._error_times: deque[float] = deque(maxlen=256)
  416. self.last_error_type: Optional[str] = None
  417. self.last_error_path: Optional[str] = None # internal-only, never serialized
  418. self.last_error_at: Optional[float] = None
  419. self.selftest_status: str = "unknown" # unknown | ok | failing
  420. self.selftest_http_status: Optional[int] = None
  421. self.selftest_at: Optional[float] = None
  422. def record_error(self, exc_type: str, path: str) -> None:
  423. now = time.time()
  424. self._error_times.append(now)
  425. self.last_error_type = exc_type
  426. self.last_error_path = path
  427. self.last_error_at = now
  428. def record_selftest(self, passed: bool, http_status: Optional[int]) -> None:
  429. self.selftest_status = "ok" if passed else "failing"
  430. self.selftest_http_status = http_status
  431. self.selftest_at = time.time()
  432. def recent_error_count(self) -> int:
  433. cutoff = time.time() - self.window_seconds
  434. while self._error_times and self._error_times[0] < cutoff:
  435. self._error_times.popleft()
  436. return len(self._error_times)
  437. def snapshot(self) -> dict[str, Any]:
  438. """Public component payload: status enum + counts + timestamps only."""
  439. errors = self.recent_error_count()
  440. status = "degraded" if (errors or self.selftest_status == "failing") else "ok"
  441. return {
  442. "status": status,
  443. "recent_unhandled_errors": errors,
  444. "last_error_at": self.last_error_at,
  445. "selftest": self.selftest_status,
  446. }
  447. DASHBOARD_HEALTH = DashboardHealth()
  448. @app.middleware("http")
  449. async def _dashboard_health_middleware(request: Request, call_next):
  450. """Outermost non-CORS middleware: count unhandled exceptions and 5xx; re-raises, never alters."""
  451. try:
  452. response = await call_next(request)
  453. except Exception as exc:
  454. DASHBOARD_HEALTH.record_error(type(exc).__name__, request.url.path)
  455. raise
  456. if response.status_code >= 500:
  457. DASHBOARD_HEALTH.record_error(f"http_{response.status_code}", request.url.path)
  458. return response
  459. def _is_trusted_local_origin(origin: str) -> bool:
  460. """The dashboard app's :func:`hermes_cli.web_server_boundary.is_trusted_local_origin`."""
  461. from hermes_cli.web_server_boundary import is_trusted_local_origin
  462. return is_trusted_local_origin(origin, app.state)
  463. class _LocalOriginCORSMiddleware(CORSMiddleware):
  464. def is_allowed_origin(self, origin: str) -> bool:
  465. return _is_trusted_local_origin(origin)
  466. # CORS: the trusted loopback origins only (web_server_boundary.is_trusted_local_origin, the same
  467. # rule the WebSocket Origin gate applies). allow_origins=["*"] on 0.0.0.0 would let any website
  468. # read/modify config and secrets; any local port would let a page on another port read
  469. # index.html's session token.
  470. #
  471. # Registered AFTER all ``@app.middleware("http")`` decorators so it is the
  472. # *outermost* middleware (Starlette's onion: last-added runs first).
  473. # Without this, an OPTIONS preflight from a cross-origin SPA hits the auth
  474. # middlewares before CORS can answer, producing 401 instead of 204 + headers.
  475. app.add_middleware(_LocalOriginCORSMiddleware, allow_methods=["*"], allow_headers=["*"])
  476. # Authenticated-route self-test: one in-process request per minute against a
  477. # cheap DB-touching route, catching "liveness fine but every authed request 500s".
  478. _DASHBOARD_SELFTEST_INTERVAL_SECONDS = 60.0
  479. _DASHBOARD_SELFTEST_ROUTE = "/api/sessions?limit=1"
  480. async def _dashboard_selftest_once() -> None:
  481. """Run one authenticated in-process self-test request and record it."""
  482. try:
  483. import httpx
  484. except ImportError:
  485. return # optional dependency — leave status "unknown"
  486. try:
  487. # Loopback base_url so the Host-header middleware accepts the request.
  488. async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://127.0.0.1") as client:
  489. resp = await client.get(_DASHBOARD_SELFTEST_ROUTE, headers={_SESSION_HEADER_NAME: _SESSION_TOKEN})
  490. DASHBOARD_HEALTH.record_selftest(resp.status_code == 200, resp.status_code)
  491. except Exception:
  492. DASHBOARD_HEALTH.record_selftest(False, None)
  493. async def _dashboard_selftest_loop() -> None:
  494. """Periodic self-test driver started from the lifespan."""
  495. try:
  496. import httpx
  497. except ImportError:
  498. _log.debug("httpx unavailable — dashboard self-test disabled")
  499. return
  500. while True:
  501. await asyncio.sleep(_DASHBOARD_SELFTEST_INTERVAL_SECONDS)
  502. # OAuth-gated binds don't honour the session token; the probe would false-alarm 401.
  503. if getattr(app.state, "auth_required", False):
  504. continue
  505. await _dashboard_selftest_once()
  506. # Action registries/spawner are owned by web_server_gateway; routers and tests reach them
  507. # there, so this module reads them through the module too (one patch seam).
  508. from hermes_cli import web_server_gateway as _gateway_mod
  509. from hermes_cli.web_server_gateway import _ACTION_LOG_FILES, _terminate_desktop_managed_gateway
  510. from hermes_cli.web_server_sessions import _auto_archive_ticker_loop
  511. from hermes_cli.web_server_chat import PTY_REGISTRY
  512. from hermes_cli.web_server_dashboard import (
  513. _discover_dashboard_plugins, _mount_plugin_api_routes, mount_spa,
  514. )
  515. _GATEWAY_HEALTH_URL = os.getenv("GATEWAY_HEALTH_URL")
  516. _GATEWAY_HEALTH_TIMEOUT_MAX = 1.0
  517. try:
  518. _GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "1"))
  519. except (ValueError, TypeError):
  520. _log.warning(
  521. "Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 1.0s",
  522. os.getenv("GATEWAY_HEALTH_TIMEOUT"),
  523. )
  524. _GATEWAY_HEALTH_TIMEOUT = 1.0
  525. if _GATEWAY_HEALTH_TIMEOUT <= 0:
  526. _log.warning(
  527. "Invalid non-positive GATEWAY_HEALTH_TIMEOUT value %.3fs — using default 1.0s",
  528. _GATEWAY_HEALTH_TIMEOUT,
  529. )
  530. _GATEWAY_HEALTH_TIMEOUT = 1.0
  531. elif _GATEWAY_HEALTH_TIMEOUT > _GATEWAY_HEALTH_TIMEOUT_MAX:
  532. _log.warning(
  533. "Capping GATEWAY_HEALTH_TIMEOUT %.3fs to %.3fs for dashboard liveness probes",
  534. _GATEWAY_HEALTH_TIMEOUT,
  535. _GATEWAY_HEALTH_TIMEOUT_MAX,
  536. )
  537. _GATEWAY_HEALTH_TIMEOUT = _GATEWAY_HEALTH_TIMEOUT_MAX
  538. _MANAGED_FILE_MAX_BYTES = 100 * 1024 * 1024
  539. _FS_DATA_URL_MAX_BYTES = WEBAPP_ATTACHMENT_MAX_BYTES
  540. # Multipart uploads stream to a temp file in fixed chunks and rename into
  541. # place: constant memory, no base64 inflation, no proxy body-size 502s (NS-501).
  542. _UPLOAD_CHUNK_BYTES = 1024 * 1024
  543. # Stable install identity for /api/status: one uuid4 hex per physical install,
  544. # persisted under the ROOT Hermes home (not the profile HERMES_HOME) so every
  545. # profile reports the same id and the desktop can collapse duplicate roster rows
  546. # for one backend. Must never change across restarts, so cached per process.
  547. _INSTALL_ID_CACHE: dict[str, Optional[str]] = {"root": None, "value": None}
  548. def get_install_id() -> Optional[str]:
  549. """Process-lifetime-cached stable install id."""
  550. return _shared_get_install_id(cache=_INSTALL_ID_CACHE)
  551. # Serializes config.yaml read-modify-write cycles for handlers on worker threads
  552. # (asyncio.to_thread): config.py's _CONFIG_LOCK covers each load/save call, not
  553. # the span between them, so two off-loop updates could drop each other's writes.
  554. # RLock so nested helpers that also take it can't self-deadlock.
  555. _CONFIG_MUTATION_LOCK = threading.RLock()
  556. # A finished ``gateway-restart`` child does not mean the gateway is back (it
  557. # exits once the restart is handed off), so in-flight reuse stops coalescing
  558. # exactly when a stale frontend re-fires every few seconds (#89034: 77 restarts,
  559. # state.db corrupted mid-FTS5-write). MAINTAINER DECISION: a fixed window, not
  560. # "until healthy" — a gateway that never returns must not leave the action
  561. # inert. 10s is above the ~3.5s storm spacing and below an operator's retry.
  562. GATEWAY_RESTART_COOLDOWN_SECONDS = 10.0
  563. # ``(monotonic spawn time, Popen, command)`` of the last restart. Deliberately
  564. # NOT read from ``_ACTION_PROCS``: entries there vanish when the child exits.
  565. _LAST_GATEWAY_RESTART: Optional[tuple[float, subprocess.Popen, tuple[str, ...]]] = None
  566. def _spawn_gateway_restart(profile: Optional[str] = None, *, interrupt: bool = False) -> tuple[subprocess.Popen, bool]:
  567. """Spawn ``hermes gateway restart``, reusing an in-flight or recent restart.
  568. Concurrent children race each other on the kill-and-start path, so a live
  569. child is reused; requests within ``GATEWAY_RESTART_COOLDOWN_SECONDS`` for the
  570. same profile coalesce onto the last spawn too (#89034). This process stops
  571. nothing itself: the child decides whether the restart is even allowed
  572. (``_cmd_restart``'s multiplexer guard), so the orphan reap (#77276) runs
  573. there, after that guard — reaping here killed the profile's gateway and then
  574. the child refused to start a replacement (#125394). Returns ``(proc, reused)``.
  575. """
  576. global _LAST_GATEWAY_RESTART
  577. # ``interrupt``: ``POST /api/gateway/restart`` is the user-commanded force path that supersedes a
  578. # drain (D6 of the safe-shutdown plan, 19b26244046); a config-save auto-restart is not, and its
  579. # child refuses while sessions are live.
  580. subcommand = _gateway_mod._gateway_subcommand(profile, "restart") + ["--interrupt"] * interrupt
  581. existing = _gateway_mod._ACTION_PROCS.get("gateway-restart")
  582. if existing is not None and existing.poll() is None:
  583. existing_command = _gateway_mod._ACTION_COMMANDS.get("gateway-restart")
  584. if existing_command is None or existing_command == tuple(subcommand):
  585. return existing, True
  586. raise RuntimeError("gateway restart already in progress for another profile")
  587. recent = _LAST_GATEWAY_RESTART
  588. if recent is not None:
  589. spawned_at, recent_proc, recent_command = recent
  590. age = time.monotonic() - spawned_at if recent_command == tuple(subcommand) else None
  591. if age is not None and age < GATEWAY_RESTART_COOLDOWN_SECONDS:
  592. _log.info(
  593. "Coalescing gateway restart: one was started %.1fs ago "
  594. "(pid %s) and the gateway may still be coming back; not "
  595. "spawning another (#89034).",
  596. age,
  597. getattr(recent_proc, "pid", "?"),
  598. )
  599. return recent_proc, True
  600. proc = _gateway_mod._spawn_hermes_action(subcommand, "gateway-restart")
  601. _LAST_GATEWAY_RESTART = (time.monotonic(), proc, tuple(subcommand))
  602. return proc, False
  603. # Collapses repeated identical ElevenLabs voice-list failures (the desktop
  604. # re-polls on every settings focus) to one log line; re-arms on success or a
  605. # changed signature.
  606. _voice_list_last_error: Optional[str] = None
  607. def _voice_list_error_logged_once(signature: Optional[str]) -> bool:
  608. """True if ``signature`` is new and should be logged now; ``None`` clears the latch."""
  609. global _voice_list_last_error
  610. if signature is None:
  611. _voice_list_last_error = None
  612. return False
  613. if signature == _voice_list_last_error:
  614. return False
  615. _voice_list_last_error = signature
  616. return True
  617. _ACTION_LOG_FILES.setdefault("computer-use-grant", "action-computer-use-grant.log")
  618. # Cache discovered plugins per-process (refresh on explicit re-scan).
  619. _dashboard_plugins_cache: Optional[list] = None
  620. def _get_dashboard_plugins(force_rescan: bool = False) -> list:
  621. """The active scope's dashboard plugins: a secondary profile's are its own (PLG-3)."""
  622. global _dashboard_plugins_cache
  623. from hermes_cli.web_server_plugin_profiles import profile_dashboard_plugins, secondary_profile_home
  624. home = secondary_profile_home()
  625. if home is not None:
  626. return profile_dashboard_plugins(home, force_rescan)
  627. stale = _dashboard_plugins_cache is None or force_rescan or any(
  628. not Path(p["_dir"]).is_dir() for p in _dashboard_plugins_cache
  629. )
  630. if stale:
  631. _dashboard_plugins_cache = _discover_dashboard_plugins()
  632. return _dashboard_plugins_cache
  633. # Router mounting. ORDER IS ROUTE-MATCHING ORDER: literal paths must land before
  634. # templated siblings (e.g. /api/sessions/bulk-delete before /api/sessions/{id}).
  635. from hermes_cli.web_routers import (
  636. files as _files_routes,
  637. git as _git_routes,
  638. local_models as _local_models_routes,
  639. status as _status_routes,
  640. actions as _actions_routes,
  641. audio as _audio_routes,
  642. display as _display_routes,
  643. sessions as _sessions_routes,
  644. profiles as _profiles_routes,
  645. memory_providers as _memory_providers_routes,
  646. config_env as _config_env_routes,
  647. models as _models_routes,
  648. messaging as _messaging_routes,
  649. messaging_relay as _messaging_relay_routes,
  650. oauth as _oauth_routes,
  651. cron as _cron_routes,
  652. mcp as _mcp_routes,
  653. ops as _ops_routes,
  654. skills as _skills_routes,
  655. tools as _tools_routes,
  656. analytics as _analytics_routes,
  657. chat_ws as _chat_ws_routes,
  658. chat_workspaces as _chat_workspaces_routes,
  659. dashboard_ui as _dashboard_ui_routes,
  660. webapp as _webapp_routes,
  661. shared_metrics as _shared_metrics_routes,
  662. delegation as _delegation_routes,
  663. webapp_access as _webapp_access_routes,
  664. ui_preferences as _ui_preferences_routes,
  665. )
  666. import hermes_cli.web_routers.desktop_plugins as _desktop_plugin_routes
  667. import hermes_cli.web_routers.uploads as _upload_routes
  668. from fastapi import Depends as _Depends
  669. from hermes_cli.web_server_surface import route_on as _route_on
  670. app.include_router(_files_routes.router)
  671. # The Webapp's routes exist only on the surfaces whose policy names them (404 elsewhere).
  672. app.include_router(_upload_routes.router, dependencies=[_Depends(_route_on("browser_files"))])
  673. app.include_router(_desktop_plugin_routes.router, dependencies=[_Depends(_route_on("desktop_plugins"))])
  674. app.include_router(_git_routes.router)
  675. app.include_router(_local_models_routes.router)
  676. app.include_router(_status_routes.router)
  677. app.include_router(_actions_routes.router)
  678. app.include_router(_audio_routes.router)
  679. app.include_router(_display_routes.router)
  680. app.include_router(_actions_routes.status_router)
  681. app.include_router(_sessions_routes.list_router)
  682. app.include_router(_profiles_routes.sessions_router)
  683. app.include_router(_sessions_routes.search_router)
  684. app.include_router(_memory_providers_routes.router)
  685. app.include_router(_config_env_routes.config_router)
  686. app.include_router(_models_routes.router)
  687. app.include_router(_config_env_routes.router)
  688. app.include_router(_messaging_routes.router)
  689. app.include_router(_messaging_relay_routes.router)
  690. app.include_router(_oauth_routes.router)
  691. app.include_router(_sessions_routes.manage_router)
  692. app.include_router(_status_routes.logs_router)
  693. app.include_router(_cron_routes.router)
  694. app.include_router(_mcp_routes.router)
  695. app.include_router(_ops_routes.router)
  696. app.include_router(_skills_routes.hub_router)
  697. app.include_router(_profiles_routes.router)
  698. app.include_router(_skills_routes.router)
  699. app.include_router(_tools_routes.router)
  700. app.include_router(_analytics_routes.router)
  701. app.include_router(_chat_ws_routes.router)
  702. app.include_router(_chat_workspaces_routes.router)
  703. app.include_router(_dashboard_ui_routes.router)
  704. app.include_router(_webapp_routes.router)
  705. app.include_router(_shared_metrics_routes.router)
  706. app.include_router(_delegation_routes.router)
  707. app.include_router(_webapp_access_routes.router, dependencies=[_Depends(_route_on("web_access"))])
  708. app.include_router(_ui_preferences_routes.router, dependencies=[_Depends(_route_on("ui_preferences"))])
  709. # Plugin API routes and the dashboard auth routes (/login, /auth/*, /api/auth/*)
  710. # mount before the SPA catch-all so /{full_path:path} doesn't swallow them. Auth
  711. # routes are always mounted — the gate middleware decides enforcement.
  712. _mount_plugin_api_routes()
  713. from hermes_cli.dashboard_auth.routes import router as _dashboard_auth_router
  714. app.include_router(_dashboard_auth_router)
  715. mount_spa(app)
  716. def _no_auth_provider_message(host: str) -> str:
  717. """Actionable SystemExit text for a gated bind with no registered auth provider.
  718. Names the exact trigger: on a loopback bind the ONLY trigger is the public URL
  719. (dashboard.public_url, or webapp.public_url for the gateway's Webapp listener),
  720. so print the offending URL and the remove-it exit.
  721. Bundled providers expose ``LAST_SKIP_REASON`` so an installed-but-
  722. unconfigured provider is not reported as merely "no providers".
  723. """
  724. skip_reasons: list[str] = []
  725. try:
  726. from plugins.dashboard_auth import nous as _nous_plugin
  727. if _nous_plugin.LAST_SKIP_REASON:
  728. skip_reasons.append(f" • nous: {_nous_plugin.LAST_SKIP_REASON}")
  729. except Exception:
  730. pass
  731. # A listener that declares its own public URL (the gateway's Webapp: webapp.public_url) is
  732. # gated by that setting, never by dashboard.public_url.
  733. own_url = getattr(app.state, "listener_public_url", None)
  734. setting = "dashboard.public_url" if own_url is None else "webapp.public_url"
  735. if host in _LOOPBACK_HOST_VALUES:
  736. public_url = own_url or ""
  737. if own_url is None:
  738. try:
  739. from hermes_cli.dashboard_auth.prefix import resolve_public_url
  740. public_url = resolve_public_url()
  741. except Exception:
  742. pass
  743. gate_reason = (
  744. f"{setting} is set to "
  745. f"{public_url or '<a non-loopback URL>'} — an "
  746. f"operator-declared external URL engages the auth gate "
  747. f"even on a loopback bind"
  748. )
  749. fix_hint = (
  750. "If this dashboard should be LOCAL-ONLY (no reverse "
  751. f"proxy), remove {setting} from config.yaml "
  752. + ("(and unset HERMES_DASHBOARD_PUBLIC_URL) " if own_url is None else "")
  753. + "to restore the unauthenticated loopback mode.\n"
  754. )
  755. else:
  756. gate_reason = f"the auth gate engages on non-loopback binds ({host})"
  757. fix_hint = ""
  758. fix_hint += (
  759. "Configure an auth provider before exposing the dashboard:\n"
  760. " • Password: set dashboard.basic_auth.username + "
  761. "password_hash in config.yaml\n"
  762. " (hash with: python -c \"from "
  763. "plugins.dashboard_auth.basic import hash_password; "
  764. "print(hash_password('your-password'))\")\n"
  765. " • OAuth: run `hermes dashboard register` (Nous Portal) or "
  766. "install a DashboardAuthProvider plugin.\n"
  767. " • Deployment token: provision HERMES_DASHBOARD_SESSION_TOKEN "
  768. "(a literal token, or 'file:<path>' naming a secret-manager-decrypted "
  769. "token file) — the bundled session-token provider then authorizes "
  770. "bearer REST + /api/ws ?token= for token-mode clients such as the "
  771. "Desktop's remote-gateway 'Session token' field.\n"
  772. "There is no unauthenticated public-dashboard option. For "
  773. f"local-only use, bind 127.0.0.1 and leave {setting} "
  774. "unset; a configured external public URL requires auth even "
  775. "when a local reverse proxy reaches a loopback backend."
  776. )
  777. # Credentials exist but the bundled provider is disabled (#54489). Basic
  778. # auth needs a username AND a credential; a half-configured block is silent.
  779. try:
  780. from hermes_cli.config import load_config as _load_cfg
  781. from hermes_cli.plugins_cmd import _BASIC_AUTH_PLUGIN_KEYS
  782. cfg = _load_cfg()
  783. ba = (cfg.get("dashboard") or {}).get("basic_auth") or {}
  784. disabled = (cfg.get("plugins") or {}).get("disabled") or []
  785. has_creds = bool(ba.get("username")) and bool(ba.get("password_hash") or ba.get("password"))
  786. if has_creds and (set(disabled) & _BASIC_AUTH_PLUGIN_KEYS):
  787. fix_hint = (
  788. "The 'basic' dashboard-auth plugin is in "
  789. "plugins.disabled but dashboard.basic_auth is "
  790. "configured.\n"
  791. "Remove 'basic' from plugins.disabled (or run "
  792. "`hermes plugins enable basic`), then restart the "
  793. "dashboard.\n\n"
  794. ) + fix_hint
  795. except Exception:
  796. pass
  797. msg = (
  798. f"Refusing to bind dashboard to {host} — {gate_reason}, "
  799. f"but no auth providers are registered.\n\n"
  800. )
  801. if skip_reasons:
  802. msg += "Bundled providers reported these issues:\n" + "\n".join(skip_reasons) + "\n\n"
  803. return msg + fix_hint
  804. def _configure_auth_gate(
  805. host: str,
  806. allow_public: bool,
  807. ssh_session_token: Optional[str],
  808. ssh_owner_nonce: Optional[str],
  809. ) -> None:
  810. """Resolve the trusted public hosts + auth-gate flag onto ``app.state``.
  811. Fails closed (``SystemExit`` with an actionable message) when the gate
  812. engages but no dashboard auth provider is registered.
  813. """
  814. # dashboard.public_url is also the exact Host/Origin trust declaration for
  815. # reverse-proxy deployments; resolved once so middleware never reloads
  816. # config. A non-loopback public hostname engages the gate even on a loopback
  817. # backend, else the SPA's local session token becomes remotely reachable.
  818. # A listener that declares its own public URL (``listener_public_url``: the
  819. # gateway's, from webapp.public_url) is judged by that one instead.
  820. own_url = getattr(app.state, "listener_public_url", None)
  821. app.state.trusted_public_hosts = (
  822. _dashboard_public_hosts() if own_url is None else public_url_hosts(own_url))
  823. app.state.trusted_public_origins = ( # its exact loopback origin: a proxy on another local port
  824. _dashboard_public_origins() if own_url is None else public_url_origins(own_url))
  825. # auth_required drives middleware, SPA-token injection, WS auth, the
  826. # startup refusal, the gate-on banner and uvicorn proxy_headers.
  827. if own_url is None and _desktop_loopback_auth_exempt(host, ssh_session_token, ssh_owner_nonce):
  828. # public_url describes the operator's PUBLIC deployment, not this
  829. # Desktop-owned loopback backend (#96490), which authenticates with the
  830. # per-spawn session token the ticket-only gate would refuse.
  831. app.state.auth_required = should_require_auth(host)
  832. _log.info(
  833. "Desktop-owned loopback backend: dashboard.public_url does not "
  834. "engage the ticket gate for this process; the public deployment "
  835. "keeps its own gate.",
  836. )
  837. else:
  838. app.state.auth_required = should_require_dashboard_auth(host, app.state.trusted_public_hosts)
  839. # ``--insecure`` no longer disables the gate (June 2026 hermes-0day
  840. # hardening); warn that it is a no-op rather than silently ignore it.
  841. if allow_public and host not in _LOOPBACK_HOST_VALUES:
  842. _log.warning(
  843. "--insecure no longer bypasses dashboard authentication. A "
  844. "non-loopback bind (%s) now ALWAYS requires an auth provider "
  845. "(OAuth or the bundled password provider). Configure one — see "
  846. "below — or bind to 127.0.0.1 and reach it over an SSH tunnel / "
  847. "Tailscale.", host,
  848. )
  849. if app.state.auth_required:
  850. # No escape hatch serves a gated dashboard without a provider.
  851. from hermes_cli.dashboard_auth import list_providers
  852. if not list_providers():
  853. raise SystemExit(_no_auth_provider_message(host))
  854. _log.info(
  855. "Dashboard binding to %s with auth gate enabled. Providers: %s",
  856. host,
  857. ", ".join(p.name for p in list_providers()),
  858. )
  859. def _build_uvicorn_server(host: str, port: int, *, ssh_isolated: bool = False):
  860. """Build the uvicorn ``Config`` + ``Server`` for this bind (reads ``app.state.auth_required``).
  861. uvicorn.Server is driven directly (not uvicorn.run) so startup is split from
  862. the main loop: after startup() the socket is bound and held by uvicorn, so the
  863. OS-assigned port can be read with no pre-bind-then-close TOCTOU. Explicit
  864. taken ports are caught by the #93608 preflight probe; uvicorn's own bind
  865. error stays the fallback for races.
  866. """
  867. import uvicorn
  868. # WS keepalive ping runs ON the agent event loop; a GIL-holding worker call
  869. # can starve it for minutes, so uvicorn misses the pong and drops a healthy
  870. # local socket (#53773/#48445/#50005). The ping only detects half-open
  871. # connections (proxy 524, dropped tunnels), impossible on loopback where a
  872. # dead client sends a real FIN/RST -> WebSocketDisconnect. So: no ping on
  873. # loopback; non-loopback sits behind a Cloudflare Tunnel (~100s idle) and
  874. # keeps a config-driven cadence (dashboard.ws_ping_interval/_timeout,
  875. # #79635) defaulting to 20/20.
  876. _is_loopback = host in _LOOPBACK_HOST_VALUES
  877. try:
  878. _dash_cfg = load_config().get("dashboard") or {}
  879. except Exception:
  880. _dash_cfg = {}
  881. def _ws_ping_setting(key: str, default: float = 20.0) -> float:
  882. try:
  883. return float(_dash_cfg.get(key, default))
  884. except (TypeError, ValueError):
  885. return default
  886. # A Desktop-owned SSH-isolated backend is loopback on the SERVER, but the client sits at the far
  887. # end of a tunnel: the local socket stays healthy while the laptop sleeps, so only a slow WS ping
  888. # notices the half-open tunnel (#101626). Its client count is tracked at the ASGI boundary so
  889. # the idle watchdog can retire the backend once nothing is connected.
  890. served_app = app
  891. ping_interval, ping_timeout = (None, None) if _is_loopback else (
  892. _ws_ping_setting("ws_ping_interval"), _ws_ping_setting("ws_ping_timeout"))
  893. if ssh_isolated:
  894. from hermes_cli.web_server_idle_exit import (
  895. TUNNEL_WS_PING_INTERVAL_S, TUNNEL_WS_PING_TIMEOUT_S, IdleClientTracker, wrap_asgi_with_ws_tracking)
  896. app.state.ssh_isolated_clients = IdleClientTracker()
  897. served_app = wrap_asgi_with_ws_tracking(app, app.state.ssh_isolated_clients)
  898. ping_interval, ping_timeout = TUNNEL_WS_PING_INTERVAL_S, TUNNEL_WS_PING_TIMEOUT_S
  899. _uvicorn_console_logging() # log_config=None: uvicorn's dictConfig would close every live handler
  900. config = uvicorn.Config(
  901. served_app, host=host, port=port, log_level="warning", log_config=None,
  902. # Off by default so _ws_client_is_allowed sees the real peer, not
  903. # X-Forwarded-For. Gated mode runs behind a TLS terminator and needs
  904. # X-Forwarded-Proto for cookie Secure flags.
  905. proxy_headers=bool(app.state.auth_required),
  906. # Loopback-only unless the operator trusts a bounded upstream proxy, so
  907. # spoofed X-Forwarded-* from arbitrary callers is never honoured.
  908. forwarded_allow_ips=_dashboard_forwarded_allow_ips(_dash_cfg),
  909. ws_ping_interval=ping_interval,
  910. ws_ping_timeout=ping_timeout,
  911. # Per-frame zlib on the loop buys nothing for a local client (build_listener_server); a peer
  912. # behind a tunnel (SSH-isolated backend, public gate) keeps it.
  913. ws_per_message_deflate=ssh_isolated or not _is_loopback or bool(app.state.auth_required),
  914. ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,
  915. # Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;
  916. # uvicorn's default (None) waits on lingering ASGI tasks forever, so a
  917. # mid-turn request orphans the backend past that budget (#76244). 3s
  918. # leaves room for lifespan shutdown + cron_stop before the kill.
  919. timeout_graceful_shutdown=3,
  920. )
  921. return config, uvicorn.Server(config)
  922. def _best_effort(what: str, fn) -> None:
  923. """Run a best-effort startup step; any failure (import included) is a debug line."""
  924. try:
  925. fn()
  926. except Exception as exc:
  927. _log.debug("%s skipped: %s", what, exc)
  928. def _reclaim_host_from_orphaned_owner(role: str) -> bool:
  929. """True when the conflicting host owner was reaped/cleared and the claim is worth retrying.
  930. HELD_BY_OTHER helper for #121964. A stale record with no live owner is retracted via
  931. ``discard_dead_record`` (which only removes provably-gone owners); a live owner is reaped
  932. only when the spawn ledger proves it is a dead session's orphan. Anything else returns
  933. False and the caller stays observe-only. Never raises.
  934. """
  935. from gateway import host_rendezvous as hr
  936. from hermes_cli.process_identity import reap_orphaned_backend_owner
  937. try:
  938. owner = hr.read_record(role)
  939. except Exception:
  940. return False
  941. if owner is None:
  942. try:
  943. return bool(hr.discard_dead_record(role))
  944. except Exception:
  945. return False
  946. if owner.pid == os.getpid():
  947. return False # never reap our own incarnation on a re-entrant claim
  948. try:
  949. if reap_orphaned_backend_owner(owner.pid, owner.create_time) is None:
  950. return False
  951. except Exception:
  952. return False
  953. try:
  954. hr.discard_dead_record(role)
  955. except Exception:
  956. pass
  957. return True
  958. def _publish_host_rendezvous(host: str, port: int) -> None:
  959. """Publish this backend's host record: ``ROLE_SERVE`` for the machine-level owner,
  960. ``ROLE_DESKTOP_SERVE`` for a Desktop-owned child."""
  961. # Desktop-spawned backends (flag + per-spawn credential; the bare flag is inherited by every
  962. # Desktop shell) are loopback, random-port and per-profile. Recording one as the HOST owner
  963. # made a later independently supervised `dashboard --host 0.0.0.0 --port N` refuse behind
  964. # the private child on every restart (#119824): the attach/refuse ladder reads ROLE_SERVE
  965. # only. They still publish under their own role so `hermes plugins install` from a terminal
  966. # can reach the backend hosting the open chats on a Desktop-only box (#119644).
  967. from gateway import host_rendezvous as hr
  968. desktop_child = is_desktop_owned_backend()
  969. role = hr.ROLE_DESKTOP_SERVE if desktop_child else hr.ROLE_SERVE
  970. outcome, error = hr.claim_host_lock(role)
  971. if outcome is hr.HostLockOutcome.COULD_NOT_OPEN:
  972. _log.warning(
  973. "Host backend lock could not be opened (%s); this backend is not discoverable. "
  974. "This is NOT another backend holding it.", error)
  975. return
  976. if outcome is hr.HostLockOutcome.HELD_BY_OTHER and not desktop_child:
  977. # A prior backend whose session died (dead spawner, PID re-parented to init) is still
  978. # alive, so it still holds the flock and its record passes every staleness check: the
  979. # pre-#121964 code returned observe-only here forever, and every attach retried against
  980. # the same orphan. Reap exactly that corpse and re-claim instead.
  981. if _reclaim_host_from_orphaned_owner(role):
  982. outcome, error = hr.claim_host_lock(role)
  983. if outcome is hr.HostLockOutcome.HELD_BY_OTHER:
  984. owner = hr.read_record(role)
  985. if desktop_child:
  986. # A second pool child is Desktop's own topology, not a conflict.
  987. _log.debug("another Desktop backend holds the %s record (%s)", role,
  988. hr.describe(owner) if owner else "owner unknown")
  989. return
  990. _log.warning(
  991. "Another backend already owns this host (%s); this one bound anyway "
  992. "(observe-only). Multiplex-only expects exactly one backend per host.",
  993. hr.describe(owner) if owner else "owner unknown",
  994. )
  995. return
  996. app.state.host_role = role
  997. hr.publish_record(
  998. role,
  999. host=host,
  1000. port=port,
  1001. profiles=hr.served_profiles(),
  1002. # The live session token, so an attaching client of the same OS user can
  1003. # authenticate even when the backend is gated and `GET /` withholds it.
  1004. token=_SESSION_TOKEN,
  1005. )
  1006. # SIGTERM included: it is the normal stop, and it does not run atexit here.
  1007. hr.cleanup_on_exit(role)
  1008. def _on_server_started(
  1009. server,
  1010. *,
  1011. host: str,
  1012. port: int,
  1013. headless: bool,
  1014. isolated: bool,
  1015. open_browser: bool,
  1016. initial_profile: str,
  1017. start_mcp_discovery_after_bind: bool,
  1018. ssh_lock_path: Optional[Path] = None,
  1019. ) -> None:
  1020. """Post-bind arming on the serving loop right after ``server.startup()``.
  1021. Reap prior corpses, parent-death watchdog, process identity, READY
  1022. announcement, browser open, deferred MCP discovery, loop-noise filter,
  1023. loop heartbeat.
  1024. """
  1025. # Clear corpses from a previous unclean Desktop exit (crash/SIGKILL/update
  1026. # handoff leaves an orphaned backend + its MCP subtree) before stacking a
  1027. # new tree (EMFILE / missing tabs). The watchdog only protects *this*
  1028. # process going forward.
  1029. def _reap_desktop_serves() -> None:
  1030. from hermes_cli.dashboard_procs import _reap_orphaned_desktop_local_serves
  1031. _reap_orphaned_desktop_local_serves()
  1032. def _reap_mcp_helpers() -> None:
  1033. from hermes_cli.process_identity import reap_orphaned_mcp_helpers
  1034. reap_orphaned_mcp_helpers()
  1035. if is_desktop_owned_backend():
  1036. _best_effort("orphan desktop-local serve reap", _reap_desktop_serves)
  1037. # Same sweep for stdio MCP helpers (#61514): positive identity only (spawn
  1038. # ledger + spawner provably dead); anything alive or unprovable is untouched.
  1039. _best_effort("orphan MCP helper reap", _reap_mcp_helpers)
  1040. # No-op for standalone `hermes serve` (no HERMES_PARENT_PID).
  1041. _start_parent_death_watchdog()
  1042. # SSH-isolated backends are detached from any parent on purpose (#91668); their liveness signal
  1043. # is "does a client still hold a WebSocket" (#101626).
  1044. if getattr(app.state, "ssh_isolated_clients", None) is not None:
  1045. from hermes_cli.web_server_idle_exit import DEFAULT_IDLE_GRACE_S, start_idle_watchdog
  1046. try:
  1047. grace = float((load_config().get("dashboard") or {}).get("ssh_isolated_idle_grace_s", DEFAULT_IDLE_GRACE_S))
  1048. except (TypeError, ValueError):
  1049. grace = DEFAULT_IDLE_GRACE_S
  1050. start_idle_watchdog(server, app.state.ssh_isolated_clients, grace_s=grace)
  1051. if getattr(app.state, "ssh_isolated_clients", None) is not None or is_desktop_owned_backend():
  1052. # The host's updater never restarts this backend (SSH-isolated: only the remote Desktop
  1053. # client holds its token and owner nonce, #91668/#101626; Desktop-owned local serve:
  1054. # the updater defers to the app's ledger-verified restart and the backend otherwise
  1055. # outlives the handoff, #99859), so it retires itself (between turns) when the install
  1056. # moves on. The retirement fence closes admission process-wide before the exit, so a
  1057. # connected Desktop just sees its next request reconnect-respawn the backend on new code.
  1058. from hermes_cli.web_server_skew_exit import start_code_skew_watchdog
  1059. start_code_skew_watchdog(server)
  1060. if getattr(app.state, "ssh_isolated_clients", None) is not None and ssh_lock_path and _SSH_OWNER_NONCE:
  1061. # A reconnect that cannot prove this pid is its own drops the lock without signalling us
  1062. # and spawns a new nonce (#132034); retire (between turns) once the lock names that spawn.
  1063. from hermes_cli.web_server_owner_exit import start_owner_watchdog
  1064. start_owner_watchdog(server, lock_path=ssh_lock_path, nonce=_SSH_OWNER_NONCE)
  1065. actual_port = _read_bound_port(server, fallback=port)
  1066. app.state.bound_port = actual_port
  1067. # Published by /api/host/identity: an attaching `hermes dashboard` must never be routed to a
  1068. # headless backend (a URL with no UI behind it).
  1069. app.state.serves_spa = not headless
  1070. # Positive process identity in the machine spawn ledger (+ Windows
  1071. # kill-on-close job). Registered AFTER the bind so the entry carries the
  1072. # ACTUAL port — what lets `hermes update` relaunch a manually-started serve
  1073. # on its real endpoint (#63206).
  1074. def _register_identity() -> None:
  1075. from hermes_cli.process_identity import attach_self_to_kill_on_close_job, register_self
  1076. register_self(
  1077. "serve" if headless else "dashboard",
  1078. detail={"host": host, "port": actual_port, "profile": initial_profile or "", "isolated": isolated},
  1079. )
  1080. attach_self_to_kill_on_close_job()
  1081. _best_effort("process-identity registration", _register_identity)
  1082. # Host rendezvous (multiplex-only): the host lock + record that let a SECOND `hermes serve`
  1083. # for any profile find this process and attach instead of binding a second port. Published
  1084. # after the bind so the record carries the real port, and beside — not instead of — the
  1085. # spawn-ledger entry above, which Desktop's attach ladder reads.
  1086. _best_effort("host rendezvous publish", lambda: _publish_host_rendezvous(host, actual_port))
  1087. _write_dashboard_ready_file(actual_port)
  1088. # Port-discovery sentinel parsed by the Desktop spawn. Written to fd 1:
  1089. # tui_gateway.server redirects sys.stdout to stderr at import, and the
  1090. # Desktop watches child.stdout (#96282). A headless `serve` announces the
  1091. # neutral token FIRST and the legacy HERMES_DASHBOARD_READY one after it:
  1092. # a packaged Desktop artifact whose parser predates the neutral token
  1093. # (#60772) still matches the legacy sentinel, while current parsers match
  1094. # either. The legacy `dashboard` backend keeps its own single token.
  1095. if headless:
  1096. _write_machine_sentinel_line(f"HERMES_BACKEND_READY port={actual_port}")
  1097. _write_machine_sentinel_line(f"HERMES_DASHBOARD_READY port={actual_port}")
  1098. else:
  1099. _write_machine_sentinel_line(f"HERMES_DASHBOARD_READY port={actual_port}")
  1100. if headless:
  1101. # Auth-gated JSON-RPC/WS only — announce the bind, not a URL. flush:
  1102. # a piped stdout otherwise surfaces this minutes after the sentinel.
  1103. print(f" Hermes backend listening on {host}:{actual_port}", flush=True)
  1104. else:
  1105. from hermes_cli.url_utils import format_url_host
  1106. print(f" Hermes Web UI → http://{format_url_host(host)}:{actual_port}")
  1107. _maybe_open_browser(host, actual_port, open_browser, initial_profile)
  1108. if start_mcp_discovery_after_bind:
  1109. # Desktop `serve`: the ~350ms `mcp` SDK import holds the GIL while the
  1110. # renderer does its WS handshake + first hydration reads, so arm it one
  1111. # second later when the shell is painted and idle. An agent build inside
  1112. # that second fires the deferred start itself (wait_for_mcp_discovery).
  1113. try:
  1114. from hermes_cli.mcp_startup import defer_background_mcp_discovery
  1115. defer_background_mcp_discovery(
  1116. logger=_log,
  1117. thread_name="dashboard-mcp-discovery",
  1118. delay=_DESKTOP_MCP_DISCOVERY_DELAY_S,
  1119. )
  1120. except Exception:
  1121. _log.debug("Deferred MCP discovery arm failed", exc_info=True)
  1122. # Collapse the peer-hangup teardown flood (#50005): 50+ identical WinError
  1123. # 10054 tracebacks per Desktop disconnect become one debug line.
  1124. def _install_noise_filter() -> None:
  1125. from tui_gateway.loop_noise import install_loop_noise_filter
  1126. install_loop_noise_filter(asyncio.get_running_loop())
  1127. _best_effort("loop noise filter install", _install_noise_filter)
  1128. # Loop heartbeat watchdog (CF-1): a 2s call_later tick whose drift equals
  1129. # any GIL stall, so a stalled-loop WS drop is diagnosable from the log.
  1130. # call_later (not a task) dies with the loop — nothing to cancel.
  1131. _hb_interval = 2.0
  1132. _hb_stall_threshold = 5.0
  1133. _hb_loop = asyncio.get_running_loop()
  1134. def _loop_heartbeat(expected: float) -> None:
  1135. now = _hb_loop.time()
  1136. drift = now - expected
  1137. if drift > _hb_stall_threshold:
  1138. _log.warning("event loop stalled %.1fs (GIL pressure suspected)", drift)
  1139. _hb_loop.call_later(_hb_interval, _loop_heartbeat, now + _hb_interval)
  1140. _hb_loop.call_later(_hb_interval, _loop_heartbeat, _hb_loop.time() + _hb_interval)
  1141. def _windows_serve_loop_factory(config):
  1142. """Loop factory for serve on Windows: always a selector loop.
  1143. uvicorn 0.41's ``asyncio_loop_factory`` returns ProactorEventLoop on
  1144. win32, on which uvicorn's socket stack binds-but-never-accepts (READY
  1145. prints, then WinError 10014 accept failures, exit 1, desktop
  1146. ECONNREFUSED — #120164, regression of #50641). A factory that already
  1147. yields selector loops (older uvicorn, explicit ``--loop``) passes through.
  1148. """
  1149. factory = config.get_loop_factory()
  1150. if factory is None or factory is asyncio.ProactorEventLoop: # type: ignore[attr-defined]
  1151. return asyncio.SelectorEventLoop
  1152. return factory
  1153. def _run_serve(serve, config, host: str, port: int) -> None:
  1154. """Drive ``serve()`` on the loop uvicorn expects.
  1155. POSIX keeps ``asyncio.run`` (already a SelectorEventLoop / uvloop). On
  1156. Windows ``asyncio.run`` defaults to a ProactorEventLoop, on which uvicorn
  1157. binds a socket that never accepts (#50641), so mirror uvicorn's own runner +
  1158. loop factory there (hand-installed selector policy for uvicorn < 0.36).
  1159. Ctrl+C -> clean return; probe-to-bind port race -> sentinel + exit code.
  1160. """
  1161. runner = asyncio.run
  1162. runner_kwargs: dict = {}
  1163. if sys.platform == "win32":
  1164. # Resolved FIRST; the serve call is outside this try so genuine
  1165. # serve-time errors (port in use) propagate instead of double-running.
  1166. try:
  1167. from uvicorn._compat import asyncio_run as runner
  1168. runner_kwargs = {"loop_factory": _windows_serve_loop_factory(config)}
  1169. except Exception:
  1170. runner = asyncio.run
  1171. runner_kwargs = {}
  1172. try:
  1173. asyncio.set_event_loop_policy(
  1174. asyncio.WindowsSelectorEventLoopPolicy() # type: ignore[attr-defined]
  1175. )
  1176. except Exception:
  1177. pass
  1178. # ``capture_signals()`` re-raises the captured signal after graceful
  1179. # shutdown; console Ctrl+C lands as KeyboardInterrupt = clean exit.
  1180. # (Re-raised SIGTERM/SIGBREAK keep their terminate disposition.)
  1181. try:
  1182. runner(serve(), **runner_kwargs)
  1183. except KeyboardInterrupt:
  1184. return
  1185. except SystemExit as exc:
  1186. # Probe-to-bind race (#93608): uvicorn's bind_socket() exits 1 — re-check
  1187. # and translate a confirmed conflict into the sentinel + distinct code.
  1188. if exc.code == 1 and _port_bind_conflict(host, port):
  1189. _report_port_in_use(host, port)
  1190. raise SystemExit(PORT_IN_USE_EXIT_CODE) from None
  1191. raise
  1192. def start_server(
  1193. host: str = "127.0.0.1",
  1194. port: int = 9119,
  1195. open_browser: bool = True,
  1196. allow_public: bool = False,
  1197. initial_profile: str = "",
  1198. headless: bool = False,
  1199. isolated: bool = False,
  1200. ssh_session_token: Optional[str] = None,
  1201. ssh_owner_nonce: Optional[str] = None,
  1202. start_mcp_discovery_after_bind: bool = False,
  1203. ssh_lock_path: Optional[Path] = None,
  1204. ):
  1205. """Start the web UI server.
  1206. ``initial_profile`` is appended to the auto-opened URL as ``?profile=<name>``
  1207. (profile alias ``<profile> dashboard``). ``headless`` is the ``serve`` path:
  1208. JSON-RPC/WS backend, no UI build, no SPA mount (``HERMES_SERVE_HEADLESS``).
  1209. ``isolated`` (``--isolated``) is recorded in the spawn ledger so attach-first
  1210. discovery never adopts this process.
  1211. ``ssh_session_token``/``ssh_owner_nonce`` are process-local Desktop SSH
  1212. bootstrap state, never persisted or exported to children; ``ssh_lock_path`` is the
  1213. Desktop's ``backend.lock.json`` for that ownership slot (supersession watchdog).
  1214. ``start_mcp_discovery_after_bind`` (Desktop ``serve``) defers MCP discovery
  1215. until the ready sentinel is written so its SDK import can't hold the GIL
  1216. against the pre-bind path.
  1217. """
  1218. from hermes_cli.update_proxy_env import restore_user_proxy_env
  1219. restore_user_proxy_env() # a backend an update replayed keeps the user's proxy env, not the update's
  1220. _apply_ssh_session_token(ssh_session_token or "")
  1221. _apply_ssh_owner_nonce(ssh_owner_nonce)
  1222. # Dashboard-mode starts don't route through main.py's `serve` path, which
  1223. # applies the same RLIMIT_NOFILE floor (policy in resource_limits, #81547).
  1224. from hermes_cli.resource_limits import apply_nofile_soft_limit
  1225. apply_nofile_soft_limit()
  1226. import uvicorn
  1227. try:
  1228. from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive
  1229. start_nous_auth_keepalive()
  1230. except Exception as exc:
  1231. _log.debug("Nous auth keepalive did not start: %s", exc)
  1232. _configure_auth_gate(host, allow_public, ssh_session_token, ssh_owner_nonce)
  1233. # host_header_middleware validates Host against this (DNS rebinding,
  1234. # GHSA-ppp5-vxwm-4cf7).
  1235. app.state.bound_host = host
  1236. # The SPA bootstrap reads this so profile-less deep links (/chat?resume=<id>) inherit the
  1237. # launcher's preselected profile instead of silently running in the launch scope (#73085).
  1238. app.state.initial_profile = str(initial_profile or "")
  1239. config, server = _build_uvicorn_server(host, port, ssh_isolated=bool(ssh_session_token))
  1240. # Flush-on-kill guard (#94724): chaining SIGTERM/SIGINT handlers persist
  1241. # in-memory transcripts to state.db before shutdown. Installed BEFORE
  1242. # uvicorn's capture_signals() so uvicorn re-raises into them as the
  1243. # "original" handlers — kills outside the serve window are covered too.
  1244. try:
  1245. from tui_gateway.server import install_exit_flush_signal_handlers
  1246. install_exit_flush_signal_handlers()
  1247. except Exception as exc:
  1248. _log.debug("exit-flush signal handlers not installed: %s", exc)
  1249. # #93608: uvicorn's bind_socket() would exit 1 with a bare ERROR line,
  1250. # indistinguishable from "backend broken". Probe first so a conflict
  1251. # surfaces as the BACKEND_PORT_IN_USE sentinel + distinct exit code.
  1252. # ``--port 0`` is skipped by the probe.
  1253. if _port_bind_conflict(host, port):
  1254. _report_port_in_use(host, port)
  1255. raise SystemExit(PORT_IN_USE_EXIT_CODE)
  1256. # LAST boot step, deliberately. One host process serves every profile and this one can be asked
  1257. # for any of them via ``?profile=``, so the decision is made here instead of on the first such
  1258. # request — activation is one-way, and everything the backend had already done by then
  1259. # (idle-reaper flushes, hosted rooms, cron) stayed on single-profile assumptions. It runs after
  1260. # the keepalive / auth gate / uvicorn build because activation FREEZES ``os.environ`` as the
  1261. # launch profile's credentials, and that snapshot is the only source for launch keys with no
  1262. # ``.env`` to rebuild from (systemd ``Environment=``, ``op run``, Compose): anything injected or
  1263. # rotated by a later boot step would otherwise be invisible for the process lifetime. No-op on a
  1264. # single-profile host; `gateway.multiplex_profiles: false` is retired and no longer skips it.
  1265. try:
  1266. from tui_gateway.launch_profile_policy import activate_multi_profile_hosting_eagerly
  1267. activate_multi_profile_hosting_eagerly()
  1268. except Exception:
  1269. _log.warning("eager multi-profile activation failed", exc_info=True)
  1270. async def _serve():
  1271. # startup split from main_loop so the bound (ephemeral) port is readable.
  1272. if not config.loaded:
  1273. config.load()
  1274. server.lifespan = config.lifespan_class(config)
  1275. with server.capture_signals():
  1276. await server.startup()
  1277. if server.should_exit:
  1278. return
  1279. _on_server_started(
  1280. server,
  1281. host=host,
  1282. port=port,
  1283. headless=headless,
  1284. isolated=isolated,
  1285. open_browser=open_browser,
  1286. initial_profile=initial_profile,
  1287. start_mcp_discovery_after_bind=start_mcp_discovery_after_bind,
  1288. ssh_lock_path=ssh_lock_path,
  1289. )
  1290. if headless:
  1291. from hermes_cli.observability.shared_metrics_startup import record_process_ready
  1292. record_process_ready("serve_boot", background=True)
  1293. await server.main_loop()
  1294. if server.started:
  1295. await server.shutdown()
  1296. _run_serve(_serve, config, host, port)