| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528 |
- """Hermes Agent — Web UI server: FastAPI app assembly, auth/host middleware, ``start_server``.
- Route handlers live in ``web_routers/``; their helpers live in the sibling
- ``web_server_<concern>`` modules and are re-imported here so ``web_server.<name>``
- stays the single late-binding seam tests monkeypatch (``web_deps.late``).
- Usage: ``python -m hermes_cli.main web [--port 8080]``.
- """
- from contextlib import asynccontextmanager
- import asyncio
- from collections import deque
- import hmac
- import logging
- import os
- import re
- import secrets
- import subprocess
- import sys
- import sysconfig
- import threading
- import time
- import urllib.parse
- from hermes_cli.install_identity import get_install_id as _shared_get_install_id
- from hermes_cli.process_identity import is_desktop_owned_backend
- from hermes_cli.pty_session import run_reaper
- from hermes_cli import web_server_file_tickets as _file_tickets
- from hermes_cli.web_server_session_token import resolve_session_token
- from pathlib import Path
- from typing import Any, Dict, Optional, Tuple
- PROJECT_ROOT = Path(__file__).parent.parent.resolve()
- if str(PROJECT_ROOT) not in sys.path:
- sys.path.insert(0, str(PROJECT_ROOT))
- from hermes_constants import WEBAPP_ATTACHMENT_MAX_BYTES
- from hermes_cli.config import load_config
- from hermes_cli.version_info import get_version_info
- try:
- from fastapi import FastAPI, HTTPException, Request
- from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import JSONResponse
- except ImportError:
- # First try lazy-installing the dashboard extras. Only the user actually
- # running `hermes dashboard` needs fastapi+uvicorn; lazy install keeps
- # them out of every other install path. After install, re-import.
- try:
- from pm import ensure_import
- ensure_import("web")
- from fastapi import (
- FastAPI, HTTPException, Request,
- )
- from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import JSONResponse
- except Exception:
- raise SystemExit(
- "Web UI requires fastapi and uvicorn.\n"
- "Run hermes pm repair, then restart Hermes."
- )
- WEB_DIST = Path(os.environ["HERMES_WEB_DIST"]) if "HERMES_WEB_DIST" in os.environ else Path(__file__).parent / "web_dist"
- _log = logging.getLogger(__name__)
- from hermes_cli.web_server_lifecycle import (
- PORT_IN_USE_EXIT_CODE,
- _dashboard_forwarded_allow_ips,
- _eager_reconcile_own_session_db,
- _maybe_open_browser,
- _port_bind_conflict,
- _read_bound_port,
- _report_port_in_use,
- _start_parent_death_watchdog,
- _uvicorn_console_logging,
- _warm_gateway_module,
- _write_dashboard_ready_file,
- _write_machine_sentinel_line,
- )
- def _gateway_owns_cron(name: str, home) -> bool:
- """A gateway already ticks this profile's store with live adapters: its OWN process, or the
- live default multiplexer (a served satellite has no gateway.pid of its own). Winning the
- tick-lock race here would deliver through the standalone path (#52202, #100489, #107485)."""
- from hermes_cli.profiles import _check_gateway_running, _served_by_running_multiplexer
- return _check_gateway_running(Path(home)) or (
- name != "default" and _served_by_running_multiplexer(name))
- def _start_desktop_cron_ticker(stop_event: threading.Event, interval: int = 60) -> None:
- """Tick the cron scheduler from inside the desktop dashboard backend.
- The desktop spawns a ``hermes dashboard`` backend, not a gateway, so without
- this a cron created in the app would never fire (no live adapters; delivery
- falls back to the per-platform send path). The primary backend outlives the
- per-profile pool (reaped after ~10 idle minutes), so it ticks EVERY local
- profile's store like a multiplex gateway; external providers keep the
- single-store behavior (registries are not profile-scoped). Cross-process
- safe: the built-in tick takes the per-store ``cron/.tick.lock``.
- Every local profile's store is ticked, not just this backend's own (#69377's desktop sibling): the
- desktop pools per-profile backends and reaps them after ~10 idle minutes, so a secondary profile's
- ticker dies with its backend and that profile's jobs silently stop firing until the user next opens it
- ("tasks on the sleeping profile could be idle" — community report, Aug 2026).
- """
- from cron.scheduler_provider import InProcessCronScheduler, resolve_cron_scheduler
- from hermes_constants import get_hermes_home, profile_name_for_home
- provider = resolve_cron_scheduler()
- own_home = Path(get_hermes_home())
- own_name = profile_name_for_home(own_home) or "default"
- # Ownership is re-checked every tick, not once at startup, so Desktop takes over when the
- # gateway stops (#126822).
- profile_gate = lambda name, home: not _gateway_owns_cron(name, home)
- start_kwargs: dict = {"interval": interval}
- if isinstance(provider, InProcessCronScheduler):
- try:
- from hermes_cli.profiles import profiles_to_serve
- # Same served set as the multiplexer: default + every live profile under profiles/.
- # The ticker re-enumerates this callable every cycle. Passing a
- # startup snapshot leaves deleted profiles in the scheduler until
- # restart, which both writes their removed stores and keeps stale
- # profiles alive in Desktop's background work.
- profile_homes = lambda: list(profiles_to_serve(multiplex=True))
- initial_profile_homes = profile_homes()
- if initial_profile_homes:
- # Even one profile needs the per-tick gateway gate; otherwise
- # Desktop races its dedicated gateway for the same cron store.
- start_kwargs["profile_homes"] = profile_homes
- start_kwargs["profile_gate"] = profile_gate
- from hermes_logging import enable_profile_log_routing
- enable_profile_log_routing(initial_profile_homes)
- _log.info(
- "Desktop cron scheduler will tick %d profile(s): %s",
- len(initial_profile_homes),
- [name for name, _home in initial_profile_homes],
- )
- except Exception:
- # Fail open to the single-store ticker so the active profile keeps firing.
- _log.exception("Desktop cron: profile enumeration failed; ticking active profile only")
- if "profile_homes" not in start_kwargs:
- # Fail open to this backend's own store behind the same gate. A gated-out profile is
- # neither ticked nor heartbeated, so Desktop never marks the gateway's store healthy.
- start_kwargs["profile_homes"] = lambda: [(own_name, own_home)]
- start_kwargs["profile_gate"] = profile_gate
- else:
- # External providers take no per-tick gate: defer their start until the gateway is gone.
- def _owned() -> bool:
- try:
- return _gateway_owns_cron(own_name, own_home)
- except Exception:
- # Start the ticker rather than silently stand down.
- _log.warning("Desktop cron: gateway-ownership probe failed; starting the ticker", exc_info=True)
- return False
- if _owned():
- _log.info(
- "Desktop cron scheduler waiting: live gateway owns cron on this HERMES_HOME; "
- "the gateway ticks with live adapters (re-probing every %ds)", interval,
- )
- while True:
- if stop_event.wait(interval):
- return
- if not _owned():
- break
- _log.info("Desktop cron scheduler started (provider=%s, interval=%ds)", provider.name, interval)
- provider.start(stop_event, **start_kwargs)
- # Desktop `serve` only (start_server(start_mcp_discovery_after_bind=True)):
- # seconds after the READY sentinel before the MCP discovery thread starts.
- _DESKTOP_MCP_DISCOVERY_DELAY_S = 1.0
- from hermes_cli.web_server_app import app_lifespan
- def _app_state_default(app: FastAPI, name: str, factory):
- """Return ``app.state.<name>``, lazily creating it for non-``with`` TestClient usages.
- The lifespan normally initialises these on the running event loop (an
- asyncio.Lock created at import time binds to whatever loop was active then).
- """
- try:
- return getattr(app.state, name)
- except AttributeError:
- value = factory()
- setattr(app.state, name, value)
- return value
- def _get_chat_argv_lock(app: FastAPI) -> asyncio.Lock:
- return _app_state_default(app, "chat_argv_lock", asyncio.Lock)
- def _get_pty_active_session_files(app: FastAPI) -> dict[str, Path]:
- return _app_state_default(app, "pty_active_session_files", dict)
- app = FastAPI(title="Hermes Agent", version=get_version_info().base_version, lifespan=app_lifespan)
- from hermes_cli.dashboard_auth.body_limit import AuthBodyLimitMiddleware
- # Register first (innermost): auth gates run before this, JSON parsing after it.
- app.add_middleware(AuthBodyLimitMiddleware)
- # Memory-provider OAuth connect routes live in the memory layer, not here.
- from hermes_cli.memory_oauth import router as _memory_oauth_router
- app.include_router(_memory_oauth_router)
- # Session token for sensitive endpoints. Resolution lives in the
- # web_server_session_token sibling; it dies with the process and is injected
- # into the SPA HTML so only the web UI can use it.
- _SESSION_TOKEN = resolve_session_token()
- _SESSION_HEADER_NAME = "X-Hermes-Session-Token"
- _SSH_OWNER_NONCE: Optional[str] = None
- _SSH_RUNTIME_PURELIB: Optional[tuple[str, int, int]] = None
- _SSH_RUNTIME_MARKER: Optional[str] = None
- def _apply_ssh_session_token(token: str) -> None:
- global _SESSION_TOKEN
- if token:
- _SESSION_TOKEN = token
- def _apply_ssh_owner_nonce(nonce: Optional[str]) -> None:
- global _SSH_OWNER_NONCE, _SSH_RUNTIME_PURELIB, _SSH_RUNTIME_MARKER
- _SSH_OWNER_NONCE = nonce
- _SSH_RUNTIME_PURELIB = None
- _SSH_RUNTIME_MARKER = None
- if nonce:
- try:
- purelib = sysconfig.get_paths()["purelib"]
- except (KeyError, OSError):
- return
- # Primary identity: a marker FILE in site-packages. A replaced venv
- # loses it deterministically; pip installs leave it. A bare (dev, ino)
- # snapshot alone is NOT enough: ext4 reuses directory inodes at once,
- # so `rm -rf venv && uv venv` can land on the same inode undetected.
- try:
- marker = os.path.join(purelib, f".hermes-ssh-runtime-{nonce}")
- with open(marker, "w", encoding="utf-8") as fh:
- fh.write(f"pid={os.getpid()}\n")
- _SSH_RUNTIME_MARKER = marker
- except OSError:
- pass # read-only site-packages — fall back to the stat snapshot
- try:
- st = os.stat(purelib)
- _SSH_RUNTIME_PURELIB = (purelib, st.st_dev, st.st_ino)
- except OSError:
- pass
- def _ssh_runtime_intact() -> bool:
- if _SSH_RUNTIME_MARKER is not None:
- return os.path.isfile(_SSH_RUNTIME_MARKER)
- # Fallback (read-only site-packages): directory identity snapshot — weaker
- # (inode reuse) but catches cross-device moves and version-bump paths.
- if _SSH_RUNTIME_PURELIB is None:
- return True
- purelib, device, inode = _SSH_RUNTIME_PURELIB
- try:
- st = os.stat(purelib)
- except OSError:
- return False
- return (st.st_dev, st.st_ino) == (device, inode)
- # In-browser Chat tab (/chat, /api/pty, /api/ws): always enabled. A module
- # constant (not an inlined True) so the WS endpoints and SPA token injection
- # share one testable seam.
- _DASHBOARD_EMBEDDED_CHAT_ENABLED = True
- from hermes_cli.web_upload_limit import ChatFileUploadLimitMiddleware
- # Inside every auth/Host middleware: those checks run before body ingestion,
- # and the receive limit runs before FastAPI can spool a multipart attachment.
- app.add_middleware(ChatFileUploadLimitMiddleware)
- # Endpoints that do NOT require the session token; everything else under /api/
- # is gated below. Shared with the OAuth gate so the two allowlists cannot
- # drift (/api/status once 401'd under the OAuth gate, breaking the portal probe).
- from hermes_cli.dashboard_auth.public_paths import PUBLIC_API_PATHS as _PUBLIC_API_PATHS
- def _has_valid_session_token(request: Request) -> bool:
- """True if the request carries a valid dashboard session token.
- The dedicated header avoids collisions with reverse proxies that already use
- ``Authorization`` (Caddy ``basic_auth``); the legacy Bearer path stays for
- older dashboard bundles.
- """
- session_header = request.headers.get(_SESSION_HEADER_NAME, "")
- if session_header and hmac.compare_digest(session_header.encode(), _SESSION_TOKEN.encode()):
- return True
- auth = request.headers.get("authorization", "")
- return hmac.compare_digest(auth.encode(), f"Bearer {_SESSION_TOKEN}".encode())
- # Routes that may also authenticate via ``?token=`` for Electron's remote
- # "open externally" download link. Browser pages use path-bound file tickets
- # instead (``web_server_file_tickets``), so the session token never lands in
- # their download history or a copyable media URL. The OAuth gate still requires
- # its session cookie; query credentials are only accepted by the loopback gate.
- _QUERY_TOKEN_API_PATHS: frozenset[str] = frozenset({"/api/files/download"})
- def _has_valid_query_token(request: Request, path: str) -> bool:
- if _file_tickets.redeem(request):
- return True
- if path not in _QUERY_TOKEN_API_PATHS:
- return False
- token = request.query_params.get("token", "")
- return bool(token) and hmac.compare_digest(token.encode(), _SESSION_TOKEN.encode())
- def _require_token(request: Request) -> None:
- """Authorize a sensitive endpoint, raising 401 if the caller isn't allowed.
- Loopback mode (``auth_required`` False): validate the SPA-injected
- ``_SESSION_TOKEN``. Gated mode: the token is NOT injected (cookie auth), and
- ``gated_auth_middleware`` already 401'd anything without a verified
- ``request.state.session`` — requiring the absent token here would make every
- ``_require_token`` endpoint unreachable behind the gate, so defer to it.
- A verified native owner (``authenticate_native_http``) is the same-user
- principal the outer seams already accepted; local Desktop runs with no
- static token, so that principal must satisfy route-local policy too.
- Hosted plugins receive this guard's decision via private IPC and request-local
- state, not via a forwarded gateway credential or a process-wide bypass.
- """
- if (getattr(request.state, "native_http_principal", None)
- or getattr(request.state, "hosted_http_authenticated", None) is True):
- return
- if getattr(request.app.state, "auth_required", False):
- ok = getattr(request.state, "session", None) is not None
- else:
- ok = _has_valid_session_token(request)
- if not ok:
- raise HTTPException(status_code=401, detail="Unauthorized")
- from hermes_cli.web_server_boundary import ( # re-exported: web_server.<name> stays the public seam
- _DESKTOP_ATTACHMENT_WS_MAX_BYTES,
- _LOOPBACK_HOST_VALUES,
- _dashboard_public_hosts,
- _dashboard_public_origins,
- _desktop_loopback_auth_exempt,
- _host_header_hostname,
- _is_accepted_host,
- public_hosts_engage_gate,
- public_url_hosts,
- public_url_origins,
- should_require_auth,
- )
- def should_require_dashboard_auth(
- host: str,
- trusted_public_hosts: Optional[frozenset[str]] = None,
- ) -> bool:
- """Gate required for a non-loopback bind OR a non-loopback ``dashboard.public_url``.
- Callers may pass the already-resolved host set so startup and request
- validation share one snapshot.
- """
- if trusted_public_hosts is None:
- trusted_public_hosts = _dashboard_public_hosts()
- return should_require_auth(host) or public_hosts_engage_gate(trusted_public_hosts)
- @app.middleware("http")
- async def host_header_middleware(request: Request, call_next):
- """Reject requests whose Host header doesn't match the bound interface (DNS rebinding, GHSA-ppp5-vxwm-4cf7)."""
- # app.state.bound_host is set by start_server() at listen time.
- bound_host = getattr(app.state, "bound_host", None)
- if bound_host and not _is_accepted_host(
- request.headers.get("host", ""), bound_host, getattr(app.state, "trusted_public_hosts", frozenset())
- ):
- return JSONResponse(
- status_code=400,
- content={
- "detail": (
- "Invalid Host header. Dashboard requests must use the "
- "bound hostname or the configured public hostname."
- ),
- },
- )
- return await call_next(request)
- # Imported here, beside the plugin gates that use it, like the other mid-module web_server_*
- # groups. Not an import cycle: web_server_dashboard reaches back into this module only inside
- # functions, so the position is not load-bearing. What is load-bearing is the middleware
- # registration order below (the HTTP gate before the auth middlewares, so it runs after them).
- from hermes_cli.web_server_dashboard import (
- PluginApiWebSocketGate, _plugin_api_name, _plugin_api_request_refused,
- )
- from hermes_cli.web_server_plugin_profiles import forward_secondary_plugin_request, plugin_request_scope
- @app.middleware("http")
- async def _plugin_api_runtime_gate(request: Request, call_next):
- """Block requests to disabled plugin API routes at request time
- (:func:`_plugin_api_request_refused`).
- Registered BEFORE the auth middlewares (runs AFTER them): an unauthenticated
- caller must get auth's 401, never this 404, or the status code becomes a
- plugin-name oracle.
- """
- path = request.url.path
- plugin_name = _plugin_api_name(request.scope)
- # Only gate authenticated requests. Unauthenticated ones fall through so
- # auth_middleware / the OAuth gate return 401 first and this route can't
- # be used as a plugin-name oracle. Desktop authenticates with a native ticket.
- if plugin_name and (
- getattr(request.state, "token_authenticated", False)
- or getattr(request.state, "native_http_principal", None)
- or getattr(request.app.state, "auth_required", False)
- or _has_valid_session_token(request)
- or _has_valid_query_token(request, path)
- ):
- with plugin_request_scope(request.scope): # a secondary profile's plugins run in its host
- if await _plugin_api_request_refused(plugin_name):
- return JSONResponse(status_code=404, content={"detail": "Plugin not found"})
- hosted = await forward_secondary_plugin_request(request, plugin_name)
- if hosted is not None:
- return hosted
- return await call_next(request)
- # The same gate for WebSocket upgrades: ``@app.middleware("http")`` passes those through untouched.
- app.add_middleware(PluginApiWebSocketGate)
- @app.middleware("http")
- async def _dashboard_auth_gate(request: Request, call_next):
- """OAuth gate — active only when start_server flags ``auth_required``; pass-through on loopback.
- Registered between host_header and auth_middleware: host check → cookie auth → token auth.
- """
- from hermes_cli.dashboard_auth.middleware import gated_auth_middleware
- return await gated_auth_middleware(request, call_next)
- @app.middleware("http")
- async def auth_middleware(request: Request, call_next):
- """Require the session token on all /api/ routes except the public list.
- Skipped for requests the token-auth seam already authenticated
- (``token_authenticated``) and when the OAuth gate is active — cookie auth is
- then authoritative and the loopback-only token path must not override it.
- """
- path = request.url.path
- if (
- not getattr(request.state, "token_authenticated", False)
- and not getattr(request.state, "native_http_principal", None)
- and not getattr(request.app.state, "auth_required", False)
- and path.startswith("/api/")
- and path not in _PUBLIC_API_PATHS
- and not path.startswith("/api/mcp/oauth/callback/")
- and not _has_valid_session_token(request)
- and not _has_valid_query_token(request, path)
- ):
- return JSONResponse(status_code=401, content={"detail": "Unauthorized"})
- return await call_next(request)
- @app.middleware("http")
- async def _token_auth_seam(request: Request, call_next):
- """Outermost auth seam: bearer-token auth for opted-in routes (registered LAST = runs FIRST).
- A registered token route is owned here — authenticate, attach the principal
- + ``token_authenticated`` so downstream gates skip enforcement. Non-token
- routes pass through untouched.
- """
- from hermes_cli.dashboard_auth.native_http import authenticate_native_http, native_profile_scope
- from hermes_cli.dashboard_auth.token_auth import token_auth_middleware
- rejection = await authenticate_native_http(request)
- if rejection is not None:
- return rejection
- with native_profile_scope(request):
- return await token_auth_middleware(request, call_next)
- _DASHBOARD_HEALTH_WINDOW_SECONDS = 300.0
- class DashboardHealth:
- """Dashboard-process health: rolling unhandled-error/5xx window + periodic self-test result.
- Feeds ``components`` on the PUBLIC ``/api/status``, so :meth:`snapshot`
- exports counts and enums only — never ``last_error_type``/``last_error_path``.
- """
- def __init__(self, window_seconds: float = _DASHBOARD_HEALTH_WINDOW_SECONDS) -> None:
- self.window_seconds = window_seconds
- self._error_times: deque[float] = deque(maxlen=256)
- self.last_error_type: Optional[str] = None
- self.last_error_path: Optional[str] = None # internal-only, never serialized
- self.last_error_at: Optional[float] = None
- self.selftest_status: str = "unknown" # unknown | ok | failing
- self.selftest_http_status: Optional[int] = None
- self.selftest_at: Optional[float] = None
- def record_error(self, exc_type: str, path: str) -> None:
- now = time.time()
- self._error_times.append(now)
- self.last_error_type = exc_type
- self.last_error_path = path
- self.last_error_at = now
- def record_selftest(self, passed: bool, http_status: Optional[int]) -> None:
- self.selftest_status = "ok" if passed else "failing"
- self.selftest_http_status = http_status
- self.selftest_at = time.time()
- def recent_error_count(self) -> int:
- cutoff = time.time() - self.window_seconds
- while self._error_times and self._error_times[0] < cutoff:
- self._error_times.popleft()
- return len(self._error_times)
- def snapshot(self) -> dict[str, Any]:
- """Public component payload: status enum + counts + timestamps only."""
- errors = self.recent_error_count()
- status = "degraded" if (errors or self.selftest_status == "failing") else "ok"
- return {
- "status": status,
- "recent_unhandled_errors": errors,
- "last_error_at": self.last_error_at,
- "selftest": self.selftest_status,
- }
- DASHBOARD_HEALTH = DashboardHealth()
- @app.middleware("http")
- async def _dashboard_health_middleware(request: Request, call_next):
- """Outermost non-CORS middleware: count unhandled exceptions and 5xx; re-raises, never alters."""
- try:
- response = await call_next(request)
- except Exception as exc:
- DASHBOARD_HEALTH.record_error(type(exc).__name__, request.url.path)
- raise
- if response.status_code >= 500:
- DASHBOARD_HEALTH.record_error(f"http_{response.status_code}", request.url.path)
- return response
- def _is_trusted_local_origin(origin: str) -> bool:
- """The dashboard app's :func:`hermes_cli.web_server_boundary.is_trusted_local_origin`."""
- from hermes_cli.web_server_boundary import is_trusted_local_origin
- return is_trusted_local_origin(origin, app.state)
- class _LocalOriginCORSMiddleware(CORSMiddleware):
- def is_allowed_origin(self, origin: str) -> bool:
- return _is_trusted_local_origin(origin)
- # CORS: the trusted loopback origins only (web_server_boundary.is_trusted_local_origin, the same
- # rule the WebSocket Origin gate applies). allow_origins=["*"] on 0.0.0.0 would let any website
- # read/modify config and secrets; any local port would let a page on another port read
- # index.html's session token.
- #
- # Registered AFTER all ``@app.middleware("http")`` decorators so it is the
- # *outermost* middleware (Starlette's onion: last-added runs first).
- # Without this, an OPTIONS preflight from a cross-origin SPA hits the auth
- # middlewares before CORS can answer, producing 401 instead of 204 + headers.
- app.add_middleware(_LocalOriginCORSMiddleware, allow_methods=["*"], allow_headers=["*"])
- # Authenticated-route self-test: one in-process request per minute against a
- # cheap DB-touching route, catching "liveness fine but every authed request 500s".
- _DASHBOARD_SELFTEST_INTERVAL_SECONDS = 60.0
- _DASHBOARD_SELFTEST_ROUTE = "/api/sessions?limit=1"
- async def _dashboard_selftest_once() -> None:
- """Run one authenticated in-process self-test request and record it."""
- try:
- import httpx
- except ImportError:
- return # optional dependency — leave status "unknown"
- try:
- # Loopback base_url so the Host-header middleware accepts the request.
- async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://127.0.0.1") as client:
- resp = await client.get(_DASHBOARD_SELFTEST_ROUTE, headers={_SESSION_HEADER_NAME: _SESSION_TOKEN})
- DASHBOARD_HEALTH.record_selftest(resp.status_code == 200, resp.status_code)
- except Exception:
- DASHBOARD_HEALTH.record_selftest(False, None)
- async def _dashboard_selftest_loop() -> None:
- """Periodic self-test driver started from the lifespan."""
- try:
- import httpx
- except ImportError:
- _log.debug("httpx unavailable — dashboard self-test disabled")
- return
- while True:
- await asyncio.sleep(_DASHBOARD_SELFTEST_INTERVAL_SECONDS)
- # OAuth-gated binds don't honour the session token; the probe would false-alarm 401.
- if getattr(app.state, "auth_required", False):
- continue
- await _dashboard_selftest_once()
- # Action registries/spawner are owned by web_server_gateway; routers and tests reach them
- # there, so this module reads them through the module too (one patch seam).
- from hermes_cli import web_server_gateway as _gateway_mod
- from hermes_cli.web_server_gateway import _ACTION_LOG_FILES, _terminate_desktop_managed_gateway
- from hermes_cli.web_server_sessions import _auto_archive_ticker_loop
- from hermes_cli.web_server_chat import PTY_REGISTRY
- from hermes_cli.web_server_dashboard import (
- _discover_dashboard_plugins, _mount_plugin_api_routes, mount_spa,
- )
- _GATEWAY_HEALTH_URL = os.getenv("GATEWAY_HEALTH_URL")
- _GATEWAY_HEALTH_TIMEOUT_MAX = 1.0
- try:
- _GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "1"))
- except (ValueError, TypeError):
- _log.warning(
- "Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 1.0s",
- os.getenv("GATEWAY_HEALTH_TIMEOUT"),
- )
- _GATEWAY_HEALTH_TIMEOUT = 1.0
- if _GATEWAY_HEALTH_TIMEOUT <= 0:
- _log.warning(
- "Invalid non-positive GATEWAY_HEALTH_TIMEOUT value %.3fs — using default 1.0s",
- _GATEWAY_HEALTH_TIMEOUT,
- )
- _GATEWAY_HEALTH_TIMEOUT = 1.0
- elif _GATEWAY_HEALTH_TIMEOUT > _GATEWAY_HEALTH_TIMEOUT_MAX:
- _log.warning(
- "Capping GATEWAY_HEALTH_TIMEOUT %.3fs to %.3fs for dashboard liveness probes",
- _GATEWAY_HEALTH_TIMEOUT,
- _GATEWAY_HEALTH_TIMEOUT_MAX,
- )
- _GATEWAY_HEALTH_TIMEOUT = _GATEWAY_HEALTH_TIMEOUT_MAX
- _MANAGED_FILE_MAX_BYTES = 100 * 1024 * 1024
- _FS_DATA_URL_MAX_BYTES = WEBAPP_ATTACHMENT_MAX_BYTES
- # Multipart uploads stream to a temp file in fixed chunks and rename into
- # place: constant memory, no base64 inflation, no proxy body-size 502s (NS-501).
- _UPLOAD_CHUNK_BYTES = 1024 * 1024
- # Stable install identity for /api/status: one uuid4 hex per physical install,
- # persisted under the ROOT Hermes home (not the profile HERMES_HOME) so every
- # profile reports the same id and the desktop can collapse duplicate roster rows
- # for one backend. Must never change across restarts, so cached per process.
- _INSTALL_ID_CACHE: dict[str, Optional[str]] = {"root": None, "value": None}
- def get_install_id() -> Optional[str]:
- """Process-lifetime-cached stable install id."""
- return _shared_get_install_id(cache=_INSTALL_ID_CACHE)
- # Serializes config.yaml read-modify-write cycles for handlers on worker threads
- # (asyncio.to_thread): config.py's _CONFIG_LOCK covers each load/save call, not
- # the span between them, so two off-loop updates could drop each other's writes.
- # RLock so nested helpers that also take it can't self-deadlock.
- _CONFIG_MUTATION_LOCK = threading.RLock()
- # A finished ``gateway-restart`` child does not mean the gateway is back (it
- # exits once the restart is handed off), so in-flight reuse stops coalescing
- # exactly when a stale frontend re-fires every few seconds (#89034: 77 restarts,
- # state.db corrupted mid-FTS5-write). MAINTAINER DECISION: a fixed window, not
- # "until healthy" — a gateway that never returns must not leave the action
- # inert. 10s is above the ~3.5s storm spacing and below an operator's retry.
- GATEWAY_RESTART_COOLDOWN_SECONDS = 10.0
- # ``(monotonic spawn time, Popen, command)`` of the last restart. Deliberately
- # NOT read from ``_ACTION_PROCS``: entries there vanish when the child exits.
- _LAST_GATEWAY_RESTART: Optional[tuple[float, subprocess.Popen, tuple[str, ...]]] = None
- def _spawn_gateway_restart(profile: Optional[str] = None, *, interrupt: bool = False) -> tuple[subprocess.Popen, bool]:
- """Spawn ``hermes gateway restart``, reusing an in-flight or recent restart.
- Concurrent children race each other on the kill-and-start path, so a live
- child is reused; requests within ``GATEWAY_RESTART_COOLDOWN_SECONDS`` for the
- same profile coalesce onto the last spawn too (#89034). This process stops
- nothing itself: the child decides whether the restart is even allowed
- (``_cmd_restart``'s multiplexer guard), so the orphan reap (#77276) runs
- there, after that guard — reaping here killed the profile's gateway and then
- the child refused to start a replacement (#125394). Returns ``(proc, reused)``.
- """
- global _LAST_GATEWAY_RESTART
- # ``interrupt``: ``POST /api/gateway/restart`` is the user-commanded force path that supersedes a
- # drain (D6 of the safe-shutdown plan, 19b26244046); a config-save auto-restart is not, and its
- # child refuses while sessions are live.
- subcommand = _gateway_mod._gateway_subcommand(profile, "restart") + ["--interrupt"] * interrupt
- existing = _gateway_mod._ACTION_PROCS.get("gateway-restart")
- if existing is not None and existing.poll() is None:
- existing_command = _gateway_mod._ACTION_COMMANDS.get("gateway-restart")
- if existing_command is None or existing_command == tuple(subcommand):
- return existing, True
- raise RuntimeError("gateway restart already in progress for another profile")
- recent = _LAST_GATEWAY_RESTART
- if recent is not None:
- spawned_at, recent_proc, recent_command = recent
- age = time.monotonic() - spawned_at if recent_command == tuple(subcommand) else None
- if age is not None and age < GATEWAY_RESTART_COOLDOWN_SECONDS:
- _log.info(
- "Coalescing gateway restart: one was started %.1fs ago "
- "(pid %s) and the gateway may still be coming back; not "
- "spawning another (#89034).",
- age,
- getattr(recent_proc, "pid", "?"),
- )
- return recent_proc, True
- proc = _gateway_mod._spawn_hermes_action(subcommand, "gateway-restart")
- _LAST_GATEWAY_RESTART = (time.monotonic(), proc, tuple(subcommand))
- return proc, False
- # Collapses repeated identical ElevenLabs voice-list failures (the desktop
- # re-polls on every settings focus) to one log line; re-arms on success or a
- # changed signature.
- _voice_list_last_error: Optional[str] = None
- def _voice_list_error_logged_once(signature: Optional[str]) -> bool:
- """True if ``signature`` is new and should be logged now; ``None`` clears the latch."""
- global _voice_list_last_error
- if signature is None:
- _voice_list_last_error = None
- return False
- if signature == _voice_list_last_error:
- return False
- _voice_list_last_error = signature
- return True
- _ACTION_LOG_FILES.setdefault("computer-use-grant", "action-computer-use-grant.log")
- # Cache discovered plugins per-process (refresh on explicit re-scan).
- _dashboard_plugins_cache: Optional[list] = None
- def _get_dashboard_plugins(force_rescan: bool = False) -> list:
- """The active scope's dashboard plugins: a secondary profile's are its own (PLG-3)."""
- global _dashboard_plugins_cache
- from hermes_cli.web_server_plugin_profiles import profile_dashboard_plugins, secondary_profile_home
- home = secondary_profile_home()
- if home is not None:
- return profile_dashboard_plugins(home, force_rescan)
- stale = _dashboard_plugins_cache is None or force_rescan or any(
- not Path(p["_dir"]).is_dir() for p in _dashboard_plugins_cache
- )
- if stale:
- _dashboard_plugins_cache = _discover_dashboard_plugins()
- return _dashboard_plugins_cache
- # Router mounting. ORDER IS ROUTE-MATCHING ORDER: literal paths must land before
- # templated siblings (e.g. /api/sessions/bulk-delete before /api/sessions/{id}).
- from hermes_cli.web_routers import (
- files as _files_routes,
- git as _git_routes,
- local_models as _local_models_routes,
- status as _status_routes,
- actions as _actions_routes,
- audio as _audio_routes,
- display as _display_routes,
- sessions as _sessions_routes,
- profiles as _profiles_routes,
- memory_providers as _memory_providers_routes,
- config_env as _config_env_routes,
- models as _models_routes,
- messaging as _messaging_routes,
- messaging_relay as _messaging_relay_routes,
- oauth as _oauth_routes,
- cron as _cron_routes,
- mcp as _mcp_routes,
- ops as _ops_routes,
- skills as _skills_routes,
- tools as _tools_routes,
- analytics as _analytics_routes,
- chat_ws as _chat_ws_routes,
- chat_workspaces as _chat_workspaces_routes,
- dashboard_ui as _dashboard_ui_routes,
- webapp as _webapp_routes,
- shared_metrics as _shared_metrics_routes,
- delegation as _delegation_routes,
- webapp_access as _webapp_access_routes,
- ui_preferences as _ui_preferences_routes,
- )
- import hermes_cli.web_routers.desktop_plugins as _desktop_plugin_routes
- import hermes_cli.web_routers.uploads as _upload_routes
- from fastapi import Depends as _Depends
- from hermes_cli.web_server_surface import route_on as _route_on
- app.include_router(_files_routes.router)
- # The Webapp's routes exist only on the surfaces whose policy names them (404 elsewhere).
- app.include_router(_upload_routes.router, dependencies=[_Depends(_route_on("browser_files"))])
- app.include_router(_desktop_plugin_routes.router, dependencies=[_Depends(_route_on("desktop_plugins"))])
- app.include_router(_git_routes.router)
- app.include_router(_local_models_routes.router)
- app.include_router(_status_routes.router)
- app.include_router(_actions_routes.router)
- app.include_router(_audio_routes.router)
- app.include_router(_display_routes.router)
- app.include_router(_actions_routes.status_router)
- app.include_router(_sessions_routes.list_router)
- app.include_router(_profiles_routes.sessions_router)
- app.include_router(_sessions_routes.search_router)
- app.include_router(_memory_providers_routes.router)
- app.include_router(_config_env_routes.config_router)
- app.include_router(_models_routes.router)
- app.include_router(_config_env_routes.router)
- app.include_router(_messaging_routes.router)
- app.include_router(_messaging_relay_routes.router)
- app.include_router(_oauth_routes.router)
- app.include_router(_sessions_routes.manage_router)
- app.include_router(_status_routes.logs_router)
- app.include_router(_cron_routes.router)
- app.include_router(_mcp_routes.router)
- app.include_router(_ops_routes.router)
- app.include_router(_skills_routes.hub_router)
- app.include_router(_profiles_routes.router)
- app.include_router(_skills_routes.router)
- app.include_router(_tools_routes.router)
- app.include_router(_analytics_routes.router)
- app.include_router(_chat_ws_routes.router)
- app.include_router(_chat_workspaces_routes.router)
- app.include_router(_dashboard_ui_routes.router)
- app.include_router(_webapp_routes.router)
- app.include_router(_shared_metrics_routes.router)
- app.include_router(_delegation_routes.router)
- app.include_router(_webapp_access_routes.router, dependencies=[_Depends(_route_on("web_access"))])
- app.include_router(_ui_preferences_routes.router, dependencies=[_Depends(_route_on("ui_preferences"))])
- # Plugin API routes and the dashboard auth routes (/login, /auth/*, /api/auth/*)
- # mount before the SPA catch-all so /{full_path:path} doesn't swallow them. Auth
- # routes are always mounted — the gate middleware decides enforcement.
- _mount_plugin_api_routes()
- from hermes_cli.dashboard_auth.routes import router as _dashboard_auth_router
- app.include_router(_dashboard_auth_router)
- mount_spa(app)
- def _no_auth_provider_message(host: str) -> str:
- """Actionable SystemExit text for a gated bind with no registered auth provider.
- Names the exact trigger: on a loopback bind the ONLY trigger is the public URL
- (dashboard.public_url, or webapp.public_url for the gateway's Webapp listener),
- so print the offending URL and the remove-it exit.
- Bundled providers expose ``LAST_SKIP_REASON`` so an installed-but-
- unconfigured provider is not reported as merely "no providers".
- """
- skip_reasons: list[str] = []
- try:
- from plugins.dashboard_auth import nous as _nous_plugin
- if _nous_plugin.LAST_SKIP_REASON:
- skip_reasons.append(f" • nous: {_nous_plugin.LAST_SKIP_REASON}")
- except Exception:
- pass
- # A listener that declares its own public URL (the gateway's Webapp: webapp.public_url) is
- # gated by that setting, never by dashboard.public_url.
- own_url = getattr(app.state, "listener_public_url", None)
- setting = "dashboard.public_url" if own_url is None else "webapp.public_url"
- if host in _LOOPBACK_HOST_VALUES:
- public_url = own_url or ""
- if own_url is None:
- try:
- from hermes_cli.dashboard_auth.prefix import resolve_public_url
- public_url = resolve_public_url()
- except Exception:
- pass
- gate_reason = (
- f"{setting} is set to "
- f"{public_url or '<a non-loopback URL>'} — an "
- f"operator-declared external URL engages the auth gate "
- f"even on a loopback bind"
- )
- fix_hint = (
- "If this dashboard should be LOCAL-ONLY (no reverse "
- f"proxy), remove {setting} from config.yaml "
- + ("(and unset HERMES_DASHBOARD_PUBLIC_URL) " if own_url is None else "")
- + "to restore the unauthenticated loopback mode.\n"
- )
- else:
- gate_reason = f"the auth gate engages on non-loopback binds ({host})"
- fix_hint = ""
- fix_hint += (
- "Configure an auth provider before exposing the dashboard:\n"
- " • Password: set dashboard.basic_auth.username + "
- "password_hash in config.yaml\n"
- " (hash with: python -c \"from "
- "plugins.dashboard_auth.basic import hash_password; "
- "print(hash_password('your-password'))\")\n"
- " • OAuth: run `hermes dashboard register` (Nous Portal) or "
- "install a DashboardAuthProvider plugin.\n"
- " • Deployment token: provision HERMES_DASHBOARD_SESSION_TOKEN "
- "(a literal token, or 'file:<path>' naming a secret-manager-decrypted "
- "token file) — the bundled session-token provider then authorizes "
- "bearer REST + /api/ws ?token= for token-mode clients such as the "
- "Desktop's remote-gateway 'Session token' field.\n"
- "There is no unauthenticated public-dashboard option. For "
- f"local-only use, bind 127.0.0.1 and leave {setting} "
- "unset; a configured external public URL requires auth even "
- "when a local reverse proxy reaches a loopback backend."
- )
- # Credentials exist but the bundled provider is disabled (#54489). Basic
- # auth needs a username AND a credential; a half-configured block is silent.
- try:
- from hermes_cli.config import load_config as _load_cfg
- from hermes_cli.plugins_cmd import _BASIC_AUTH_PLUGIN_KEYS
- cfg = _load_cfg()
- ba = (cfg.get("dashboard") or {}).get("basic_auth") or {}
- disabled = (cfg.get("plugins") or {}).get("disabled") or []
- has_creds = bool(ba.get("username")) and bool(ba.get("password_hash") or ba.get("password"))
- if has_creds and (set(disabled) & _BASIC_AUTH_PLUGIN_KEYS):
- fix_hint = (
- "The 'basic' dashboard-auth plugin is in "
- "plugins.disabled but dashboard.basic_auth is "
- "configured.\n"
- "Remove 'basic' from plugins.disabled (or run "
- "`hermes plugins enable basic`), then restart the "
- "dashboard.\n\n"
- ) + fix_hint
- except Exception:
- pass
- msg = (
- f"Refusing to bind dashboard to {host} — {gate_reason}, "
- f"but no auth providers are registered.\n\n"
- )
- if skip_reasons:
- msg += "Bundled providers reported these issues:\n" + "\n".join(skip_reasons) + "\n\n"
- return msg + fix_hint
- def _configure_auth_gate(
- host: str,
- allow_public: bool,
- ssh_session_token: Optional[str],
- ssh_owner_nonce: Optional[str],
- ) -> None:
- """Resolve the trusted public hosts + auth-gate flag onto ``app.state``.
- Fails closed (``SystemExit`` with an actionable message) when the gate
- engages but no dashboard auth provider is registered.
- """
- # dashboard.public_url is also the exact Host/Origin trust declaration for
- # reverse-proxy deployments; resolved once so middleware never reloads
- # config. A non-loopback public hostname engages the gate even on a loopback
- # backend, else the SPA's local session token becomes remotely reachable.
- # A listener that declares its own public URL (``listener_public_url``: the
- # gateway's, from webapp.public_url) is judged by that one instead.
- own_url = getattr(app.state, "listener_public_url", None)
- app.state.trusted_public_hosts = (
- _dashboard_public_hosts() if own_url is None else public_url_hosts(own_url))
- app.state.trusted_public_origins = ( # its exact loopback origin: a proxy on another local port
- _dashboard_public_origins() if own_url is None else public_url_origins(own_url))
- # auth_required drives middleware, SPA-token injection, WS auth, the
- # startup refusal, the gate-on banner and uvicorn proxy_headers.
- if own_url is None and _desktop_loopback_auth_exempt(host, ssh_session_token, ssh_owner_nonce):
- # public_url describes the operator's PUBLIC deployment, not this
- # Desktop-owned loopback backend (#96490), which authenticates with the
- # per-spawn session token the ticket-only gate would refuse.
- app.state.auth_required = should_require_auth(host)
- _log.info(
- "Desktop-owned loopback backend: dashboard.public_url does not "
- "engage the ticket gate for this process; the public deployment "
- "keeps its own gate.",
- )
- else:
- app.state.auth_required = should_require_dashboard_auth(host, app.state.trusted_public_hosts)
- # ``--insecure`` no longer disables the gate (June 2026 hermes-0day
- # hardening); warn that it is a no-op rather than silently ignore it.
- if allow_public and host not in _LOOPBACK_HOST_VALUES:
- _log.warning(
- "--insecure no longer bypasses dashboard authentication. A "
- "non-loopback bind (%s) now ALWAYS requires an auth provider "
- "(OAuth or the bundled password provider). Configure one — see "
- "below — or bind to 127.0.0.1 and reach it over an SSH tunnel / "
- "Tailscale.", host,
- )
- if app.state.auth_required:
- # No escape hatch serves a gated dashboard without a provider.
- from hermes_cli.dashboard_auth import list_providers
- if not list_providers():
- raise SystemExit(_no_auth_provider_message(host))
- _log.info(
- "Dashboard binding to %s with auth gate enabled. Providers: %s",
- host,
- ", ".join(p.name for p in list_providers()),
- )
- def _build_uvicorn_server(host: str, port: int, *, ssh_isolated: bool = False):
- """Build the uvicorn ``Config`` + ``Server`` for this bind (reads ``app.state.auth_required``).
- uvicorn.Server is driven directly (not uvicorn.run) so startup is split from
- the main loop: after startup() the socket is bound and held by uvicorn, so the
- OS-assigned port can be read with no pre-bind-then-close TOCTOU. Explicit
- taken ports are caught by the #93608 preflight probe; uvicorn's own bind
- error stays the fallback for races.
- """
- import uvicorn
- # WS keepalive ping runs ON the agent event loop; a GIL-holding worker call
- # can starve it for minutes, so uvicorn misses the pong and drops a healthy
- # local socket (#53773/#48445/#50005). The ping only detects half-open
- # connections (proxy 524, dropped tunnels), impossible on loopback where a
- # dead client sends a real FIN/RST -> WebSocketDisconnect. So: no ping on
- # loopback; non-loopback sits behind a Cloudflare Tunnel (~100s idle) and
- # keeps a config-driven cadence (dashboard.ws_ping_interval/_timeout,
- # #79635) defaulting to 20/20.
- _is_loopback = host in _LOOPBACK_HOST_VALUES
- try:
- _dash_cfg = load_config().get("dashboard") or {}
- except Exception:
- _dash_cfg = {}
- def _ws_ping_setting(key: str, default: float = 20.0) -> float:
- try:
- return float(_dash_cfg.get(key, default))
- except (TypeError, ValueError):
- return default
- # A Desktop-owned SSH-isolated backend is loopback on the SERVER, but the client sits at the far
- # end of a tunnel: the local socket stays healthy while the laptop sleeps, so only a slow WS ping
- # notices the half-open tunnel (#101626). Its client count is tracked at the ASGI boundary so
- # the idle watchdog can retire the backend once nothing is connected.
- served_app = app
- ping_interval, ping_timeout = (None, None) if _is_loopback else (
- _ws_ping_setting("ws_ping_interval"), _ws_ping_setting("ws_ping_timeout"))
- if ssh_isolated:
- from hermes_cli.web_server_idle_exit import (
- TUNNEL_WS_PING_INTERVAL_S, TUNNEL_WS_PING_TIMEOUT_S, IdleClientTracker, wrap_asgi_with_ws_tracking)
- app.state.ssh_isolated_clients = IdleClientTracker()
- served_app = wrap_asgi_with_ws_tracking(app, app.state.ssh_isolated_clients)
- ping_interval, ping_timeout = TUNNEL_WS_PING_INTERVAL_S, TUNNEL_WS_PING_TIMEOUT_S
- _uvicorn_console_logging() # log_config=None: uvicorn's dictConfig would close every live handler
- config = uvicorn.Config(
- served_app, host=host, port=port, log_level="warning", log_config=None,
- # Off by default so _ws_client_is_allowed sees the real peer, not
- # X-Forwarded-For. Gated mode runs behind a TLS terminator and needs
- # X-Forwarded-Proto for cookie Secure flags.
- proxy_headers=bool(app.state.auth_required),
- # Loopback-only unless the operator trusts a bounded upstream proxy, so
- # spoofed X-Forwarded-* from arbitrary callers is never honoured.
- forwarded_allow_ips=_dashboard_forwarded_allow_ips(_dash_cfg),
- ws_ping_interval=ping_interval,
- ws_ping_timeout=ping_timeout,
- # Per-frame zlib on the loop buys nothing for a local client (build_listener_server); a peer
- # behind a tunnel (SSH-isolated backend, public gate) keeps it.
- ws_per_message_deflate=ssh_isolated or not _is_loopback or bool(app.state.auth_required),
- ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,
- # Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;
- # uvicorn's default (None) waits on lingering ASGI tasks forever, so a
- # mid-turn request orphans the backend past that budget (#76244). 3s
- # leaves room for lifespan shutdown + cron_stop before the kill.
- timeout_graceful_shutdown=3,
- )
- return config, uvicorn.Server(config)
- def _best_effort(what: str, fn) -> None:
- """Run a best-effort startup step; any failure (import included) is a debug line."""
- try:
- fn()
- except Exception as exc:
- _log.debug("%s skipped: %s", what, exc)
- def _reclaim_host_from_orphaned_owner(role: str) -> bool:
- """True when the conflicting host owner was reaped/cleared and the claim is worth retrying.
- HELD_BY_OTHER helper for #121964. A stale record with no live owner is retracted via
- ``discard_dead_record`` (which only removes provably-gone owners); a live owner is reaped
- only when the spawn ledger proves it is a dead session's orphan. Anything else returns
- False and the caller stays observe-only. Never raises.
- """
- from gateway import host_rendezvous as hr
- from hermes_cli.process_identity import reap_orphaned_backend_owner
- try:
- owner = hr.read_record(role)
- except Exception:
- return False
- if owner is None:
- try:
- return bool(hr.discard_dead_record(role))
- except Exception:
- return False
- if owner.pid == os.getpid():
- return False # never reap our own incarnation on a re-entrant claim
- try:
- if reap_orphaned_backend_owner(owner.pid, owner.create_time) is None:
- return False
- except Exception:
- return False
- try:
- hr.discard_dead_record(role)
- except Exception:
- pass
- return True
- def _publish_host_rendezvous(host: str, port: int) -> None:
- """Publish this backend's host record: ``ROLE_SERVE`` for the machine-level owner,
- ``ROLE_DESKTOP_SERVE`` for a Desktop-owned child."""
- # Desktop-spawned backends (flag + per-spawn credential; the bare flag is inherited by every
- # Desktop shell) are loopback, random-port and per-profile. Recording one as the HOST owner
- # made a later independently supervised `dashboard --host 0.0.0.0 --port N` refuse behind
- # the private child on every restart (#119824): the attach/refuse ladder reads ROLE_SERVE
- # only. They still publish under their own role so `hermes plugins install` from a terminal
- # can reach the backend hosting the open chats on a Desktop-only box (#119644).
- from gateway import host_rendezvous as hr
- desktop_child = is_desktop_owned_backend()
- role = hr.ROLE_DESKTOP_SERVE if desktop_child else hr.ROLE_SERVE
- outcome, error = hr.claim_host_lock(role)
- if outcome is hr.HostLockOutcome.COULD_NOT_OPEN:
- _log.warning(
- "Host backend lock could not be opened (%s); this backend is not discoverable. "
- "This is NOT another backend holding it.", error)
- return
- if outcome is hr.HostLockOutcome.HELD_BY_OTHER and not desktop_child:
- # A prior backend whose session died (dead spawner, PID re-parented to init) is still
- # alive, so it still holds the flock and its record passes every staleness check: the
- # pre-#121964 code returned observe-only here forever, and every attach retried against
- # the same orphan. Reap exactly that corpse and re-claim instead.
- if _reclaim_host_from_orphaned_owner(role):
- outcome, error = hr.claim_host_lock(role)
- if outcome is hr.HostLockOutcome.HELD_BY_OTHER:
- owner = hr.read_record(role)
- if desktop_child:
- # A second pool child is Desktop's own topology, not a conflict.
- _log.debug("another Desktop backend holds the %s record (%s)", role,
- hr.describe(owner) if owner else "owner unknown")
- return
- _log.warning(
- "Another backend already owns this host (%s); this one bound anyway "
- "(observe-only). Multiplex-only expects exactly one backend per host.",
- hr.describe(owner) if owner else "owner unknown",
- )
- return
- app.state.host_role = role
- hr.publish_record(
- role,
- host=host,
- port=port,
- profiles=hr.served_profiles(),
- # The live session token, so an attaching client of the same OS user can
- # authenticate even when the backend is gated and `GET /` withholds it.
- token=_SESSION_TOKEN,
- )
- # SIGTERM included: it is the normal stop, and it does not run atexit here.
- hr.cleanup_on_exit(role)
- def _on_server_started(
- server,
- *,
- host: str,
- port: int,
- headless: bool,
- isolated: bool,
- open_browser: bool,
- initial_profile: str,
- start_mcp_discovery_after_bind: bool,
- ssh_lock_path: Optional[Path] = None,
- ) -> None:
- """Post-bind arming on the serving loop right after ``server.startup()``.
- Reap prior corpses, parent-death watchdog, process identity, READY
- announcement, browser open, deferred MCP discovery, loop-noise filter,
- loop heartbeat.
- """
- # Clear corpses from a previous unclean Desktop exit (crash/SIGKILL/update
- # handoff leaves an orphaned backend + its MCP subtree) before stacking a
- # new tree (EMFILE / missing tabs). The watchdog only protects *this*
- # process going forward.
- def _reap_desktop_serves() -> None:
- from hermes_cli.dashboard_procs import _reap_orphaned_desktop_local_serves
- _reap_orphaned_desktop_local_serves()
- def _reap_mcp_helpers() -> None:
- from hermes_cli.process_identity import reap_orphaned_mcp_helpers
- reap_orphaned_mcp_helpers()
- if is_desktop_owned_backend():
- _best_effort("orphan desktop-local serve reap", _reap_desktop_serves)
- # Same sweep for stdio MCP helpers (#61514): positive identity only (spawn
- # ledger + spawner provably dead); anything alive or unprovable is untouched.
- _best_effort("orphan MCP helper reap", _reap_mcp_helpers)
- # No-op for standalone `hermes serve` (no HERMES_PARENT_PID).
- _start_parent_death_watchdog()
- # SSH-isolated backends are detached from any parent on purpose (#91668); their liveness signal
- # is "does a client still hold a WebSocket" (#101626).
- if getattr(app.state, "ssh_isolated_clients", None) is not None:
- from hermes_cli.web_server_idle_exit import DEFAULT_IDLE_GRACE_S, start_idle_watchdog
- try:
- grace = float((load_config().get("dashboard") or {}).get("ssh_isolated_idle_grace_s", DEFAULT_IDLE_GRACE_S))
- except (TypeError, ValueError):
- grace = DEFAULT_IDLE_GRACE_S
- start_idle_watchdog(server, app.state.ssh_isolated_clients, grace_s=grace)
- if getattr(app.state, "ssh_isolated_clients", None) is not None or is_desktop_owned_backend():
- # The host's updater never restarts this backend (SSH-isolated: only the remote Desktop
- # client holds its token and owner nonce, #91668/#101626; Desktop-owned local serve:
- # the updater defers to the app's ledger-verified restart and the backend otherwise
- # outlives the handoff, #99859), so it retires itself (between turns) when the install
- # moves on. The retirement fence closes admission process-wide before the exit, so a
- # connected Desktop just sees its next request reconnect-respawn the backend on new code.
- from hermes_cli.web_server_skew_exit import start_code_skew_watchdog
- start_code_skew_watchdog(server)
- if getattr(app.state, "ssh_isolated_clients", None) is not None and ssh_lock_path and _SSH_OWNER_NONCE:
- # A reconnect that cannot prove this pid is its own drops the lock without signalling us
- # and spawns a new nonce (#132034); retire (between turns) once the lock names that spawn.
- from hermes_cli.web_server_owner_exit import start_owner_watchdog
- start_owner_watchdog(server, lock_path=ssh_lock_path, nonce=_SSH_OWNER_NONCE)
- actual_port = _read_bound_port(server, fallback=port)
- app.state.bound_port = actual_port
- # Published by /api/host/identity: an attaching `hermes dashboard` must never be routed to a
- # headless backend (a URL with no UI behind it).
- app.state.serves_spa = not headless
- # Positive process identity in the machine spawn ledger (+ Windows
- # kill-on-close job). Registered AFTER the bind so the entry carries the
- # ACTUAL port — what lets `hermes update` relaunch a manually-started serve
- # on its real endpoint (#63206).
- def _register_identity() -> None:
- from hermes_cli.process_identity import attach_self_to_kill_on_close_job, register_self
- register_self(
- "serve" if headless else "dashboard",
- detail={"host": host, "port": actual_port, "profile": initial_profile or "", "isolated": isolated},
- )
- attach_self_to_kill_on_close_job()
- _best_effort("process-identity registration", _register_identity)
- # Host rendezvous (multiplex-only): the host lock + record that let a SECOND `hermes serve`
- # for any profile find this process and attach instead of binding a second port. Published
- # after the bind so the record carries the real port, and beside — not instead of — the
- # spawn-ledger entry above, which Desktop's attach ladder reads.
- _best_effort("host rendezvous publish", lambda: _publish_host_rendezvous(host, actual_port))
- _write_dashboard_ready_file(actual_port)
- # Port-discovery sentinel parsed by the Desktop spawn. Written to fd 1:
- # tui_gateway.server redirects sys.stdout to stderr at import, and the
- # Desktop watches child.stdout (#96282). A headless `serve` announces the
- # neutral token FIRST and the legacy HERMES_DASHBOARD_READY one after it:
- # a packaged Desktop artifact whose parser predates the neutral token
- # (#60772) still matches the legacy sentinel, while current parsers match
- # either. The legacy `dashboard` backend keeps its own single token.
- if headless:
- _write_machine_sentinel_line(f"HERMES_BACKEND_READY port={actual_port}")
- _write_machine_sentinel_line(f"HERMES_DASHBOARD_READY port={actual_port}")
- else:
- _write_machine_sentinel_line(f"HERMES_DASHBOARD_READY port={actual_port}")
- if headless:
- # Auth-gated JSON-RPC/WS only — announce the bind, not a URL. flush:
- # a piped stdout otherwise surfaces this minutes after the sentinel.
- print(f" Hermes backend listening on {host}:{actual_port}", flush=True)
- else:
- from hermes_cli.url_utils import format_url_host
- print(f" Hermes Web UI → http://{format_url_host(host)}:{actual_port}")
- _maybe_open_browser(host, actual_port, open_browser, initial_profile)
- if start_mcp_discovery_after_bind:
- # Desktop `serve`: the ~350ms `mcp` SDK import holds the GIL while the
- # renderer does its WS handshake + first hydration reads, so arm it one
- # second later when the shell is painted and idle. An agent build inside
- # that second fires the deferred start itself (wait_for_mcp_discovery).
- try:
- from hermes_cli.mcp_startup import defer_background_mcp_discovery
- defer_background_mcp_discovery(
- logger=_log,
- thread_name="dashboard-mcp-discovery",
- delay=_DESKTOP_MCP_DISCOVERY_DELAY_S,
- )
- except Exception:
- _log.debug("Deferred MCP discovery arm failed", exc_info=True)
- # Collapse the peer-hangup teardown flood (#50005): 50+ identical WinError
- # 10054 tracebacks per Desktop disconnect become one debug line.
- def _install_noise_filter() -> None:
- from tui_gateway.loop_noise import install_loop_noise_filter
- install_loop_noise_filter(asyncio.get_running_loop())
- _best_effort("loop noise filter install", _install_noise_filter)
- # Loop heartbeat watchdog (CF-1): a 2s call_later tick whose drift equals
- # any GIL stall, so a stalled-loop WS drop is diagnosable from the log.
- # call_later (not a task) dies with the loop — nothing to cancel.
- _hb_interval = 2.0
- _hb_stall_threshold = 5.0
- _hb_loop = asyncio.get_running_loop()
- def _loop_heartbeat(expected: float) -> None:
- now = _hb_loop.time()
- drift = now - expected
- if drift > _hb_stall_threshold:
- _log.warning("event loop stalled %.1fs (GIL pressure suspected)", drift)
- _hb_loop.call_later(_hb_interval, _loop_heartbeat, now + _hb_interval)
- _hb_loop.call_later(_hb_interval, _loop_heartbeat, _hb_loop.time() + _hb_interval)
- def _windows_serve_loop_factory(config):
- """Loop factory for serve on Windows: always a selector loop.
- uvicorn 0.41's ``asyncio_loop_factory`` returns ProactorEventLoop on
- win32, on which uvicorn's socket stack binds-but-never-accepts (READY
- prints, then WinError 10014 accept failures, exit 1, desktop
- ECONNREFUSED — #120164, regression of #50641). A factory that already
- yields selector loops (older uvicorn, explicit ``--loop``) passes through.
- """
- factory = config.get_loop_factory()
- if factory is None or factory is asyncio.ProactorEventLoop: # type: ignore[attr-defined]
- return asyncio.SelectorEventLoop
- return factory
- def _run_serve(serve, config, host: str, port: int) -> None:
- """Drive ``serve()`` on the loop uvicorn expects.
- POSIX keeps ``asyncio.run`` (already a SelectorEventLoop / uvloop). On
- Windows ``asyncio.run`` defaults to a ProactorEventLoop, on which uvicorn
- binds a socket that never accepts (#50641), so mirror uvicorn's own runner +
- loop factory there (hand-installed selector policy for uvicorn < 0.36).
- Ctrl+C -> clean return; probe-to-bind port race -> sentinel + exit code.
- """
- runner = asyncio.run
- runner_kwargs: dict = {}
- if sys.platform == "win32":
- # Resolved FIRST; the serve call is outside this try so genuine
- # serve-time errors (port in use) propagate instead of double-running.
- try:
- from uvicorn._compat import asyncio_run as runner
- runner_kwargs = {"loop_factory": _windows_serve_loop_factory(config)}
- except Exception:
- runner = asyncio.run
- runner_kwargs = {}
- try:
- asyncio.set_event_loop_policy(
- asyncio.WindowsSelectorEventLoopPolicy() # type: ignore[attr-defined]
- )
- except Exception:
- pass
- # ``capture_signals()`` re-raises the captured signal after graceful
- # shutdown; console Ctrl+C lands as KeyboardInterrupt = clean exit.
- # (Re-raised SIGTERM/SIGBREAK keep their terminate disposition.)
- try:
- runner(serve(), **runner_kwargs)
- except KeyboardInterrupt:
- return
- except SystemExit as exc:
- # Probe-to-bind race (#93608): uvicorn's bind_socket() exits 1 — re-check
- # and translate a confirmed conflict into the sentinel + distinct code.
- if exc.code == 1 and _port_bind_conflict(host, port):
- _report_port_in_use(host, port)
- raise SystemExit(PORT_IN_USE_EXIT_CODE) from None
- raise
- def start_server(
- host: str = "127.0.0.1",
- port: int = 9119,
- open_browser: bool = True,
- allow_public: bool = False,
- initial_profile: str = "",
- headless: bool = False,
- isolated: bool = False,
- ssh_session_token: Optional[str] = None,
- ssh_owner_nonce: Optional[str] = None,
- start_mcp_discovery_after_bind: bool = False,
- ssh_lock_path: Optional[Path] = None,
- ):
- """Start the web UI server.
- ``initial_profile`` is appended to the auto-opened URL as ``?profile=<name>``
- (profile alias ``<profile> dashboard``). ``headless`` is the ``serve`` path:
- JSON-RPC/WS backend, no UI build, no SPA mount (``HERMES_SERVE_HEADLESS``).
- ``isolated`` (``--isolated``) is recorded in the spawn ledger so attach-first
- discovery never adopts this process.
- ``ssh_session_token``/``ssh_owner_nonce`` are process-local Desktop SSH
- bootstrap state, never persisted or exported to children; ``ssh_lock_path`` is the
- Desktop's ``backend.lock.json`` for that ownership slot (supersession watchdog).
- ``start_mcp_discovery_after_bind`` (Desktop ``serve``) defers MCP discovery
- until the ready sentinel is written so its SDK import can't hold the GIL
- against the pre-bind path.
- """
- from hermes_cli.update_proxy_env import restore_user_proxy_env
- restore_user_proxy_env() # a backend an update replayed keeps the user's proxy env, not the update's
- _apply_ssh_session_token(ssh_session_token or "")
- _apply_ssh_owner_nonce(ssh_owner_nonce)
- # Dashboard-mode starts don't route through main.py's `serve` path, which
- # applies the same RLIMIT_NOFILE floor (policy in resource_limits, #81547).
- from hermes_cli.resource_limits import apply_nofile_soft_limit
- apply_nofile_soft_limit()
- import uvicorn
- try:
- from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive
- start_nous_auth_keepalive()
- except Exception as exc:
- _log.debug("Nous auth keepalive did not start: %s", exc)
- _configure_auth_gate(host, allow_public, ssh_session_token, ssh_owner_nonce)
- # host_header_middleware validates Host against this (DNS rebinding,
- # GHSA-ppp5-vxwm-4cf7).
- app.state.bound_host = host
- # The SPA bootstrap reads this so profile-less deep links (/chat?resume=<id>) inherit the
- # launcher's preselected profile instead of silently running in the launch scope (#73085).
- app.state.initial_profile = str(initial_profile or "")
- config, server = _build_uvicorn_server(host, port, ssh_isolated=bool(ssh_session_token))
- # Flush-on-kill guard (#94724): chaining SIGTERM/SIGINT handlers persist
- # in-memory transcripts to state.db before shutdown. Installed BEFORE
- # uvicorn's capture_signals() so uvicorn re-raises into them as the
- # "original" handlers — kills outside the serve window are covered too.
- try:
- from tui_gateway.server import install_exit_flush_signal_handlers
- install_exit_flush_signal_handlers()
- except Exception as exc:
- _log.debug("exit-flush signal handlers not installed: %s", exc)
- # #93608: uvicorn's bind_socket() would exit 1 with a bare ERROR line,
- # indistinguishable from "backend broken". Probe first so a conflict
- # surfaces as the BACKEND_PORT_IN_USE sentinel + distinct exit code.
- # ``--port 0`` is skipped by the probe.
- if _port_bind_conflict(host, port):
- _report_port_in_use(host, port)
- raise SystemExit(PORT_IN_USE_EXIT_CODE)
- # LAST boot step, deliberately. One host process serves every profile and this one can be asked
- # for any of them via ``?profile=``, so the decision is made here instead of on the first such
- # request — activation is one-way, and everything the backend had already done by then
- # (idle-reaper flushes, hosted rooms, cron) stayed on single-profile assumptions. It runs after
- # the keepalive / auth gate / uvicorn build because activation FREEZES ``os.environ`` as the
- # launch profile's credentials, and that snapshot is the only source for launch keys with no
- # ``.env`` to rebuild from (systemd ``Environment=``, ``op run``, Compose): anything injected or
- # rotated by a later boot step would otherwise be invisible for the process lifetime. No-op on a
- # single-profile host; `gateway.multiplex_profiles: false` is retired and no longer skips it.
- try:
- from tui_gateway.launch_profile_policy import activate_multi_profile_hosting_eagerly
- activate_multi_profile_hosting_eagerly()
- except Exception:
- _log.warning("eager multi-profile activation failed", exc_info=True)
- async def _serve():
- # startup split from main_loop so the bound (ephemeral) port is readable.
- if not config.loaded:
- config.load()
- server.lifespan = config.lifespan_class(config)
- with server.capture_signals():
- await server.startup()
- if server.should_exit:
- return
- _on_server_started(
- server,
- host=host,
- port=port,
- headless=headless,
- isolated=isolated,
- open_browser=open_browser,
- initial_profile=initial_profile,
- start_mcp_discovery_after_bind=start_mcp_discovery_after_bind,
- ssh_lock_path=ssh_lock_path,
- )
- if headless:
- from hermes_cli.observability.shared_metrics_startup import record_process_ready
- record_process_ready("serve_boot", background=True)
- await server.main_loop()
- if server.started:
- await server.shutdown()
- _run_serve(_serve, config, host, port)
|