status.py 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934
  1. """Status dashboard routes: health, /api/status, system stats, curator, learning graph,
  2. portal and diagnostics actions.
  3. Extracted from ``hermes_cli.web_server``; app state and helpers are late-bound through
  4. :mod:`hermes_cli.web_deps` (cycle-safe, monkeypatch-friendly).
  5. """
  6. import concurrent.futures
  7. import importlib
  8. import logging
  9. import re
  10. import asyncio
  11. import os
  12. import sys
  13. import time
  14. from fastapi import APIRouter
  15. from hermes_cli.web_deps import LateState, late
  16. from hermes_cli.web_server_gateway import _display_system_platform
  17. from starlette.concurrency import run_in_threadpool
  18. from fastapi import HTTPException, Request
  19. from gateway.status import (
  20. derive_gateway_busy, derive_gateway_drainable, normalize_updated_at, parse_active_agents,
  21. profile_platforms_from_multiplexer, resolve_gateway_liveness,
  22. retained_gateway_state,
  23. runtime_status_heartbeat_age_s, runtime_status_is_stale)
  24. from gateway.status_served import local_liveness_pid
  25. from hermes_cli import __release_date__
  26. from hermes_cli.config import get_config_path, get_env_path
  27. from hermes_cli.version_info import get_version_info
  28. from hermes_constants import get_process_hermes_home, profile_name_for_home
  29. from hermes_cli.web_models import CuratorPause, LearningNodeRef, LearningNodeEdit, DebugShareRequest
  30. from hermes_cli.web_routers._common import config_scoped_to_thread, destructive_profile, scoped_to_thread
  31. from pathlib import Path
  32. from typing import Any, Dict, Optional
  33. _log = logging.getLogger("hermes_cli.web_server")
  34. router = APIRouter()
  35. # Mounted separately by web_server so /api/logs keeps its original route-table position.
  36. logs_router = APIRouter()
  37. # Late-bound so a test's monkeypatch on the owning module wins at call time.
  38. _collect_profile_gateway_topology_cached = late("_collect_profile_gateway_topology_cached", "hermes_cli.web_server_gateway")
  39. _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
  40. _dashboard_local_update_managed_externally = late("_dashboard_local_update_managed_externally", "hermes_cli.web_server_files")
  41. _load_configured_gateway_platforms = late("_load_configured_gateway_platforms", "hermes_cli.web_server_gateway")
  42. _probe_gateway_health = late("_probe_gateway_health", "hermes_cli.web_server_gateway")
  43. _require_token = late("_require_token")
  44. _resolve_profile_dir = late("_resolve_profile_dir", "hermes_cli.web_server_profiles")
  45. _resolve_restart_drain_timeout = late("_resolve_restart_drain_timeout", "hermes_cli.web_server_lifecycle")
  46. _spawn_hermes_action = late("_spawn_hermes_action", "hermes_cli.web_server_gateway")
  47. _ssh_runtime_intact = late("_ssh_runtime_intact")
  48. app = LateState("app") # the FastAPI instance (app.state.*)
  49. check_config_version = late("check_config_version", "hermes_cli.config")
  50. get_hermes_home = late("get_hermes_home", "hermes_cli.config")
  51. _profile_cli_args = late("_profile_cli_args", "hermes_cli.web_server_profiles")
  52. get_install_id = late("get_install_id")
  53. get_running_pid_cached = late("get_running_pid_cached", "gateway.status")
  54. get_runtime_status_running_pid = late("get_runtime_status_running_pid", "gateway.status")
  55. served_profile_runtime = late("served_profile_runtime", "gateway.status_served")
  56. load_config = late("load_config", "hermes_cli.config")
  57. read_runtime_status = late("read_runtime_status", "gateway.status")
  58. _open_session_db_for_profile = late("_open_session_db_for_profile", "hermes_cli.web_server_sessions")
  59. _STATUS_ACTIVE_SESSIONS_TIMEOUT = 0.75
  60. _GATEWAY_HEALTH_ROUTE_TIMEOUT = 1.0
  61. _HEALTHY_PLATFORM_STATES = {"connected", "running", "ok"}
  62. def _safe_call(mod, fn_name: str, default):
  63. try:
  64. fn = getattr(mod, fn_name, None)
  65. return fn() if callable(fn) else default
  66. except Exception:
  67. return default
  68. def _count_status_active_sessions() -> int:
  69. """Best-effort status garnish. Opens read-only (via the shared stale-schema heal) so
  70. /api/status never routinely writes to state.db while another Hermes process uses it."""
  71. from hermes_state import _default_db_path
  72. # The heal helper bootstraps a missing store; this garnish must not — on a fresh install
  73. # /api/status polls would otherwise create state.db before the user's first session.
  74. if not Path(_default_db_path()).exists():
  75. return 0
  76. db = _open_session_db_for_profile(None, read_only=True)
  77. try:
  78. sessions = db.list_sessions_rich(limit=50, compact_rows=True)
  79. now = time.time()
  80. return sum(1 for s in sessions if s.get("ended_at") is None
  81. and (now - s.get("last_active", s.get("started_at", 0))) < 300)
  82. finally:
  83. db.close()
  84. async def _status_active_sessions() -> int:
  85. try:
  86. return await asyncio.wait_for(
  87. run_in_threadpool(_count_status_active_sessions),
  88. timeout=_STATUS_ACTIVE_SESSIONS_TIMEOUT)
  89. except TimeoutError:
  90. _log.debug("/api/status active session count exceeded %.2fs; returning 0",
  91. _STATUS_ACTIVE_SESSIONS_TIMEOUT)
  92. except Exception as exc:
  93. _log.debug("/api/status active session count unavailable: %s", exc)
  94. return 0
  95. @router.get("/api/ssh/ownership")
  96. async def get_ssh_ownership(request: Request):
  97. from hermes_cli.web_server import _SSH_OWNER_NONCE
  98. _require_token(request)
  99. if not _SSH_OWNER_NONCE:
  100. raise HTTPException(status_code=404, detail="SSH ownership is not active")
  101. return {"ok": True, "sshOwnerNonce": _SSH_OWNER_NONCE, "protocolVersion": 1,
  102. "runtimeIntact": _ssh_runtime_intact()}
  103. @router.get("/api/health")
  104. async def get_health():
  105. """Lightweight process liveness for desktop/backend readiness probes.
  106. ``commit`` is the code this process BOOTED from (``get_version_info`` is cached at
  107. ``web_server`` import): Desktop refuses to attach to a backend whose commit differs from
  108. its checkout, so a serve that outlived ``hermes update`` is never re-adopted.
  109. """
  110. info = get_version_info()
  111. return {"ok": True, "version": info.base_version, "displayVersion": info.display_version,
  112. "commit": info.commit,
  113. "auth_required": bool(getattr(app.state, "auth_required", False))}
  114. @router.get("/api/host/identity")
  115. async def get_host_identity(request: Request):
  116. """Prove to an attaching `hermes serve`/`dashboard` WHO owns this port.
  117. The host rendezvous record names a (pid, port) owner, but a record cannot say whether that
  118. owner still holds the port: a graceful-shutdown window or an unrelated listener that
  119. inherited the port both look identical on disk. The attaching side dials this endpoint with
  120. the owner's 0600 token and attaches only when pid+role match. ``servesSpa`` is false for
  121. headless ``serve``, so a `hermes dashboard` user is never routed to a backend with no UI.
  122. """
  123. _require_token(request)
  124. # ``role`` is the host ROLE this process published (gateway/host_rendezvous.ROLE_SERVE, or
  125. # ROLE_DESKTOP_SERVE for a Desktop-owned child), not the launch mode: `hermes serve` and
  126. # `hermes dashboard` are one host role that differ in SPA.
  127. return {"ok": True, "protocolVersion": 1, "pid": os.getpid(),
  128. "role": getattr(app.state, "host_role", None) or "serve",
  129. "servesSpa": bool(getattr(app.state, "serves_spa", False))}
  130. @router.get("/api/health/idle")
  131. async def get_health_idle(request: Request):
  132. """Token-gated diagnostic snapshot; never a retirement permit. None means cannot prove idle."""
  133. from hermes_cli.web_server_idle_proof import idle_proof
  134. _require_token(request)
  135. return {"ok": True, **idle_proof()}
  136. @router.post("/api/health/retirement")
  137. async def post_health_retirement(request: Request):
  138. from hermes_cli.backend_retirement import retirement
  139. _require_token(request)
  140. try:
  141. body = await request.json()
  142. except ValueError:
  143. raise HTTPException(status_code=400, detail="JSON body required")
  144. if not isinstance(body, dict) or body.get("action") not in ("prepare", "commit", "cancel"):
  145. raise HTTPException(status_code=400, detail="action must be prepare, commit, or cancel")
  146. action = body["action"]
  147. if action == "prepare":
  148. return await run_in_threadpool(retirement.prepare)
  149. token = body.get("token")
  150. if not isinstance(token, str) or not token:
  151. raise HTTPException(status_code=400, detail="token required")
  152. return getattr(retirement, action)(token)
  153. # Profile segment mirrors hermes_cli.profiles._PROFILE_ID_RE. Platform segment mirrors the
  154. # Platform enum's normalized values: built-in members plus plugin directory names
  155. # (lowercased), which allow hyphens as well as underscores (e.g. ``reviewer:foo-bar``).
  156. _PROFILE_PLATFORM_STATUS_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z0-9][a-z0-9_-]{0,63}$")
  157. def _is_profile_platform_status_key(key: object) -> bool:
  158. """Accept only the runner's public ``<profile>:<platform>`` key grammar."""
  159. return isinstance(key, str) and bool(_PROFILE_PLATFORM_STATUS_KEY_RE.fullmatch(key))
  160. def _status_platform_key_allowed(key: object, configured: set[str] | None) -> bool:
  161. """Whether a runtime-status platform key may appear publicly: namespaced
  162. ``<profile>:<platform>`` keys are validated against the grammar *unconditionally* (a
  163. failed config-set load must not fail open into projecting arbitrary keys from a
  164. process-local JSON file); plain keys are checked against the configured set when it
  165. loaded, passed through when it did not."""
  166. if not isinstance(key, str):
  167. return False
  168. if ":" in key:
  169. return _is_profile_platform_status_key(key)
  170. return configured is None or key in configured
  171. # Per-entry writer-identity stamps (added by gateway.status.write_runtime_status for the
  172. # aggregation ownership check) are process recon — the same class of detail as the
  173. # auth-gated top-level ``gateway_pid`` — and must not project onto the public endpoint.
  174. _PRIVATE_PLATFORM_ENTRY_KEYS = frozenset({"writer_pid", "writer_start_time"})
  175. def _public_platform_entry(value: Any) -> Any:
  176. """Strip writer-identity stamps from a platform entry before projection."""
  177. if not isinstance(value, dict):
  178. return value
  179. return {k: v for k, v in value.items() if k not in _PRIVATE_PLATFORM_ENTRY_KEYS}
  180. def _merge_profile_gateway_platforms(gateway_platforms: dict, profile_platforms: dict) -> dict:
  181. """Merge independent per-profile gateway platform states: hosts running separate gateway
  182. services per profile (``gateway_mode == "multiple"``) persist each profile's platform
  183. failures in its own ``gateway_state.json``, invisible to the machine-level probe NAS
  184. reads unless folded in under the validated ``<profile>:<platform>`` grammar. The active
  185. profile's own map is skipped (already present); existing keys are never overwritten."""
  186. try:
  187. from hermes_cli.profiles import get_active_profile_name
  188. active = get_active_profile_name()
  189. except Exception:
  190. active = "default"
  191. merged = dict(gateway_platforms)
  192. for prof, plats in (profile_platforms or {}).items():
  193. if prof == active or not isinstance(plats, dict):
  194. continue
  195. for key, value in plats.items():
  196. if not isinstance(key, str) or ":" in key or not isinstance(value, dict):
  197. continue
  198. namespaced = f"{prof}:{key}"
  199. if not _is_profile_platform_status_key(namespaced):
  200. continue
  201. merged.setdefault(namespaced, _public_platform_entry(value))
  202. return merged
  203. # --- Gateway liveness detection --- Delegated to the single shared ladder in gateway.status so this
  204. # endpoint and /api/messaging/platforms can never disagree about whether the gateway is up (they used to:
  205. # sidebar "running" while the Channels page rendered "The gateway is not running"). When ?profile=<name> was
  206. # given, scope PID and state reads to that profile's directory — gateway identity files (PID, lock, runtime
  207. # status) are written to the per-profile home, not the process-level HERMES_HOME (see issue #69143). Plain
  208. # /api/status keeps the exact zero-arg call so its behavior (and cache signature) is unchanged. The
  209. # module-level probe references are handed to the resolver so the long-standing
  210. # `monkeypatch.setattr(gateway.status, "get_running_pid_cached", ...)` seam used across the test-suite still
  211. # intercepts them.
  212. def _bounded_health_probe():
  213. """Health probe with the route's blocking-call budget preserved. The resolver only
  214. reaches this rung when the local PID probe came up empty, so the timeout is paid at
  215. most once per request and only in the cross-container case that needs it."""
  216. with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
  217. future = pool.submit(_probe_gateway_health)
  218. try:
  219. return future.result(timeout=_GATEWAY_HEALTH_ROUTE_TIMEOUT)
  220. except concurrent.futures.TimeoutError:
  221. _log.warning("/api/status gateway health probe exceeded %.2fs; using local status",
  222. _GATEWAY_HEALTH_ROUTE_TIMEOUT)
  223. return False, None
  224. except Exception:
  225. return False, None
  226. def _project_gateway_platforms(gateway_platforms: dict, configured: set[str] | None,
  227. gateway_running: bool, gateway_state) -> dict:
  228. """Public projection of a runtime's platform map (see ``_status_platform_key_allowed``
  229. for the key rules). A cleanly stopped gateway's platform states are stale noise and are
  230. cleared so a dead process can't report "connected"; a startup_failed gateway's FATAL
  231. entries are the diagnosis (credential collisions, auth failures) that the single
  232. exit_reason string can't express, so they are kept — upstream writer-identity/freshness
  233. filtering already dropped other processes' entries."""
  234. platforms = {key: _public_platform_entry(value) for key, value in gateway_platforms.items()
  235. if _status_platform_key_allowed(key, configured)}
  236. if gateway_running:
  237. return platforms
  238. if gateway_state == "startup_failed":
  239. return {key: value for key, value in platforms.items()
  240. if isinstance(value, dict) and value.get("state") == "fatal"}
  241. return {}
  242. async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> dict[str, Any]:
  243. """Liveness + runtime-state readout (running/pid/state/platforms/exit_reason/updated_at
  244. plus the raw ``runtime`` document).
  245. Liveness is delegated to the shared ladder in gateway.status so this endpoint and
  246. /api/messaging/platforms can never disagree. With ``?profile=<name>`` PID/state reads are
  247. scoped to that profile's directory (gateway identity files live in the per-profile
  248. home); plain /api/status keeps the exact zero-arg call so its cache signature is
  249. unchanged. The module-level probe references are handed to the resolver so the
  250. ``monkeypatch.setattr(gateway.status, "get_running_pid_cached", ...)`` seam still intercepts.
  251. """
  252. local_runtime = (read_runtime_status(path=profile_dir / "gateway_state.json")
  253. if profile_dir else read_runtime_status())
  254. liveness = await run_in_threadpool(lambda: resolve_gateway_liveness(
  255. profile_dir=profile_dir, runtime=local_runtime,
  256. health_probe=_bounded_health_probe if health_url else None,
  257. pid_probe=get_running_pid_cached, runtime_reader=read_runtime_status,
  258. runtime_pid_probe=get_runtime_status_running_pid))
  259. gateway_running = liveness.running
  260. remote_health_body: dict | None = liveness.health_body
  261. try:
  262. configured = await run_in_threadpool(_load_configured_gateway_platforms)
  263. except Exception:
  264. configured = None
  265. # Prefer the detailed health endpoint response (has full state) when the local runtime
  266. # status file is absent or stale (cross-container).
  267. runtime = local_runtime
  268. if runtime is None and remote_health_body and remote_health_body.get("gateway_state"):
  269. runtime = remote_health_body
  270. # Unscoped, the profile is the process's own home (a pooled ``hermes --profile X serve``).
  271. own_home = profile_dir or get_process_hermes_home()
  272. if liveness.runtime is not None:
  273. # Served by the multiplexer: its record is this profile's runtime, with the profile's own
  274. # adapters under ``<profile>:<platform>`` re-keyed to the standalone shape.
  275. # Fold on the profile NAME, never ``profile_dir.name``: ``?profile=default`` resolves the
  276. # root itself, whose basename (``.hermes``) matched nothing and read as a named id (#123088).
  277. served_name = profile_name_for_home(own_home)
  278. runtime = {**liveness.runtime,
  279. "platforms": profile_platforms_from_multiplexer(liveness.runtime, served_name or "")}
  280. elif local_liveness_pid(liveness) is not None:
  281. # The PID rung answers before the multiplexer rung, and a served profile's gateway.lock is
  282. # held by the multiplexer itself, so the ladder returns without its record while the
  283. # profile's own gateway_state.json may be a retired standalone gateway's (#139080). The
  284. # shared rule (also the messaging card's) picks the multiplexer record only when the lock
  285. # holder IS the multiplexer and the profile's own record proves no live gateway.
  286. try:
  287. runtime = await run_in_threadpool(
  288. served_profile_runtime, own_home, local_runtime, gateway_pid=local_liveness_pid(liveness)) or runtime
  289. except Exception: # health: allow BLE001 -- status route must never 500; traceback is logged
  290. # A failed lookup keeps the profile's own record.
  291. _log.warning("/api/status multiplexer record lookup failed", exc_info=True)
  292. gateway_state = None
  293. gateway_platforms: dict = {}
  294. gateway_exit_reason = None
  295. gateway_updated_at = None
  296. gateway_heartbeat_stale_s = None
  297. if runtime:
  298. gateway_state = runtime.get("gateway_state")
  299. if not gateway_running:
  300. # Shared with /api/messaging/platforms: a durable operator stop outranks a retained
  301. # ``startup_failed`` / watchdog ``degraded`` (kept on disk for diagnostics), so the
  302. # overview does not alarm on it.
  303. gateway_state = retained_gateway_state(runtime)
  304. elif remote_health_body is not None and gateway_state in {None, "stopped"}:
  305. # The health probe confirmed the gateway is alive, but the local runtime status
  306. # file may be stale (cross-container): override so the badge is correct.
  307. gateway_state = "running"
  308. elif gateway_state in {"running", "degraded", "starting"} and runtime_status_is_stale(runtime):
  309. # Alive PID, but housekeeping stopped re-stamping the heartbeat: the loop or the
  310. # housekeeping thread wedged while the file still says 'running' (#113372). Same arm
  311. # as ``hermes gateway status`` so the sidebar strip and the CLI agree.
  312. gateway_heartbeat_stale_s = runtime_status_heartbeat_age_s(runtime)
  313. gateway_platforms = _project_gateway_platforms(
  314. runtime.get("platforms") or {}, configured, gateway_running, gateway_state)
  315. gateway_exit_reason = None if gateway_state == "stopped" else runtime.get("exit_reason")
  316. # Contract: gateway_updated_at is RFC3339 string | null, never a number. ``runtime``
  317. # may be the local gateway_state.json (legacy gateways wrote epoch floats; hand
  318. # edits can inject anything) or a remote /health/detailed body — normalize both.
  319. gateway_updated_at = normalize_updated_at(runtime.get("updated_at"))
  320. # No runtime info at all but the health probe confirmed alive (no shared volume).
  321. if gateway_running and gateway_state is None and remote_health_body is not None:
  322. gateway_state = "running"
  323. # A live multiplexer's record (the host's own, or the one serving this profile) names every
  324. # profile a restart would blip ("default, alpha, beta"); a standalone gateway's names none.
  325. shared_with = (runtime or {}).get("served_profiles") if gateway_running else None
  326. return {
  327. "runtime": runtime, "gateway_running": gateway_running, "gateway_pid": liveness.pid,
  328. "gateway_state": gateway_state, "gateway_platforms": gateway_platforms,
  329. "gateway_exit_reason": gateway_exit_reason, "gateway_updated_at": gateway_updated_at,
  330. "gateway_heartbeat_stale_s": gateway_heartbeat_stale_s,
  331. "gateway_shared_with": [str(p) for p in shared_with] if isinstance(shared_with, list) else None}
  332. def _auth_gate_status() -> dict[str, Any]:
  333. """Dashboard auth gate readout: gate engaged, registered providers, and the RFC 8252
  334. native-app capability advertisement ``auth_flows`` the desktop reads to pick the
  335. system-browser + loopback + PKCE flow over the embedded-webview cookie flow. "cookie" is
  336. always available in gated mode; "native_pkce" when at least one interactive session
  337. provider is registered (token-only credentials such as drain, and provisioned-credential
  338. session providers such as session-token, don't count). Missing
  339. "native_pkce" ⇒ older gateway ⇒ desktop falls back automatically."""
  340. auth_required = bool(getattr(app.state, "auth_required", False))
  341. auth_providers: list[str] = []
  342. auth_flows: list[str] = []
  343. try:
  344. from hermes_cli.dashboard_auth import (
  345. list_interactive_providers as _list_interactive_providers,
  346. list_providers as _list_providers)
  347. auth_providers = [p.name for p in _list_providers()]
  348. if auth_required:
  349. auth_flows.append("cookie")
  350. if _list_interactive_providers():
  351. auth_flows.append("native_pkce")
  352. except Exception:
  353. # Module not importable yet (early startup) — leave as [].
  354. pass
  355. return {"auth_required": auth_required, "auth_providers": auth_providers,
  356. "auth_flows": auth_flows}
  357. def _nous_session_validity() -> str:
  358. """Nous bootstrap-session validity for the NAS health sweep: a hosted agent whose Nous
  359. auth dies terminally looks HEALTHY to every liveness probe yet every inference turn
  360. fails, and this is the ONLY signal that surfaces it (local auth-store state, no token
  361. needed). Best-effort: never let auth classification break the probe."""
  362. try:
  363. from hermes_cli.auth import get_nous_session_validity
  364. return get_nous_session_validity()
  365. except Exception:
  366. return "unknown"
  367. async def _component_health(gateway: dict[str, Any]) -> dict[str, Any]:
  368. """Component-level health rollup: counts and status enums only (public payload — no
  369. messages, paths or other detail that could carry secrets). The storage probe reuses the
  370. gateway readiness state_db check (read-only, 1s-bounded) off-loop."""
  371. from hermes_cli.web_server import DASHBOARD_HEALTH
  372. gateway_running, gateway_state = gateway["gateway_running"], gateway["gateway_state"]
  373. gateway_platforms = gateway["gateway_platforms"]
  374. components: dict[str, Any] = {
  375. "gateway": {
  376. "status": "ok" if gateway_running and gateway_state in {"running", "draining"} else "degraded",
  377. "state": gateway_state or ("running" if gateway_running else "stopped")},
  378. "dashboard": DASHBOARD_HEALTH.snapshot()}
  379. try:
  380. from gateway.readiness import _probe_state_db
  381. storage_check = await run_in_threadpool(_probe_state_db, get_hermes_home())
  382. components["storage"] = {"status": storage_check.get("status", "degraded")}
  383. # The one reason enum consumers key off; same latch as readiness and the session lists.
  384. if storage_check.get("detail") == "corrupt":
  385. components["storage"]["reason"] = "corrupt"
  386. except Exception:
  387. components["storage"] = {"status": "degraded"}
  388. # ``disabled`` entries are platforms the multiplexer deliberately does not run for a served profile
  389. # (shared ingress owned by the default) — informational, never a degraded verdict.
  390. platform_states = [str(value.get("state") or value.get("status") or "").lower()
  391. for value in gateway_platforms.values() if isinstance(value, dict)]
  392. platform_states = [state for state in platform_states if state != "disabled"]
  393. connected = sum(1 for state in platform_states if state in _HEALTHY_PLATFORM_STATES)
  394. components["platforms"] = {"status": "ok" if connected == len(platform_states) else "degraded",
  395. "configured": len(platform_states), "connected": connected}
  396. return components
  397. async def _advisory_pressure(status: dict[str, Any], home: Path) -> None:
  398. """Memory / disk pressure rollups + deferred FTS rebuild progress (coarse numbers/enums
  399. only; public payload). Deliberately NOT folded into components/overall: pressure is
  400. advisory, not a liveness verdict, and flipping ``overall`` on it would page NAS's
  401. availability sweep for a condition the valve is already handling. Read-only, never raise.
  402. """
  403. for key, mod_name, fn_name in (("memory", "gateway.memory_status", "collect_memory_status"),
  404. ("disk", "gateway.disk_status", "collect_disk_status")):
  405. try:
  406. collect = getattr(importlib.import_module(mod_name), fn_name)
  407. status[key] = await run_in_threadpool(collect, home)
  408. except Exception:
  409. status[key] = {"pressure": "unknown"}
  410. try:
  411. # SQLite open + read: off the loop like the rest of the probe.
  412. _rebuild = await run_in_threadpool(_fts_rebuild_status)
  413. if _rebuild is not None:
  414. status["fts_rebuild"] = _rebuild
  415. except Exception:
  416. pass
  417. def _status_local_reads() -> dict[str, Any]:
  418. """The probe's synchronous local reads: the raw config.yaml stamp, the install method, the
  419. auth store and (first call only) git provenance. Run on a worker thread: on the event loop that
  420. also serves the session WebSocket they stalled every Desktop request behind the disk (#143561)."""
  421. current_ver, latest_ver = check_config_version()
  422. return {"version": get_version_info().base_version, "config_version": current_ver,
  423. "latest_config_version": latest_ver,
  424. "can_update_hermes": not _dashboard_local_update_managed_externally(),
  425. "nous_session_valid": _nous_session_validity()}
  426. def _fts_rebuild_status() -> Optional[dict[str, Any]]:
  427. """Deferred FTS rebuild progress of this home's state.db (read-only open), or None."""
  428. from hermes_state import SessionDB as _SDB
  429. from hermes_constants import get_hermes_home as _ghh
  430. _db_path = _ghh() / "state.db"
  431. if not _db_path.exists():
  432. return None
  433. _sdb = _SDB(db_path=_db_path, read_only=True)
  434. try:
  435. return _sdb.fts_rebuild_status()
  436. finally:
  437. _sdb.close()
  438. #: What the gateway's Webapp answers a caller who has not signed in: liveness, version and the
  439. #: sign-in shape (the native-app login probe and uptime checks read nothing else). Profile names,
  440. #: the install id and host figures wait for sign-in, since a Webapp on a wildcard bind is reachable
  441. #: by everyone on the network. The standalone dashboard keeps its public body: Hermes Cloud reads
  442. #: the profile list there without a session (#60585).
  443. _WEBAPP_SIGNED_OUT_STATUS_KEYS = frozenset({
  444. "version", "release_date", "gateway_running", "gateway_state", "overall", "components",
  445. "auth_required", "auth_providers", "auth_flows"})
  446. async def _webapp_caller_signed_out(request: Request) -> bool:
  447. """True when this request reached the gateway's Webapp behind its sign-in gate with no
  448. session the gate would accept. ``/api/status`` is public, so the gate never verified one."""
  449. from hermes_cli.web_server import _require_token
  450. from hermes_cli.web_server_surface import surface
  451. if request is None or surface(request.app.state) != "webapp" \
  452. or not getattr(request.app.state, "auth_required", False):
  453. return False
  454. if getattr(request.state, "session", None) is None:
  455. from hermes_cli.dashboard_auth.middleware import peek_session
  456. request.state.session = await run_in_threadpool(peek_session, request)
  457. try:
  458. _require_token(request)
  459. except HTTPException:
  460. return True
  461. return False
  462. @router.get("/api/status")
  463. async def get_status(profile: Optional[str] = None,
  464. request: Request = None): # type: ignore[assignment] # None when awaited in-process
  465. """Public machine-level liveness probe (``PUBLIC_API_PATHS``): version, gateway state,
  466. active session count and the auth-gate shape — no bodies, no session content, no secrets.
  467. ``?profile=`` (dashboard management switcher) uses the config-only contextvar scope, NOT
  468. _profile_scope: this handler awaits the remote health probe, and _profile_scope swaps
  469. process-global skills-module attributes a concurrent request would cross-restore.
  470. """
  471. from hermes_cli.web_server import _GATEWAY_HEALTH_URL
  472. signed_out = await _webapp_caller_signed_out(request)
  473. status_scope = None
  474. # A signed-out caller's ``?profile=`` is ignored: an unknown name's 404 would tell it which exist.
  475. requested_profile = "" if signed_out else (profile or "").strip()
  476. profile_dir: Optional[Path] = None
  477. if requested_profile and requested_profile.lower() != "current":
  478. profile_dir = _resolve_profile_dir(requested_profile)
  479. status_scope = _config_profile_scope(requested_profile)
  480. status_scope.__enter__()
  481. try:
  482. local = await run_in_threadpool(_status_local_reads)
  483. gateway = await _resolve_gateway_status(profile_dir, _GATEWAY_HEALTH_URL)
  484. gateway_running, gateway_state = gateway["gateway_running"], gateway["gateway_state"]
  485. # Topology (cached, TTL 10s) is fetched before the platform rollup so per-profile
  486. # gateway failures fold into the machine-level view (see
  487. # _merge_profile_gateway_platforms); a ``?profile=`` request is left unmerged.
  488. topology = await run_in_threadpool(_collect_profile_gateway_topology_cached)
  489. if not requested_profile:
  490. gateway["gateway_platforms"] = _merge_profile_gateway_platforms(
  491. gateway["gateway_platforms"], topology.get("profile_platforms") or {})
  492. active_sessions = await _status_active_sessions()
  493. # Busy/drainable (NAS lifecycle-safety gate) derive from the persisted in-flight turn
  494. # count + liveness via gateway.status. Liveness keys off gateway_running, NEVER
  495. # gateway_updated_at — a stale heartbeat is reported separately, not treated as death.
  496. active_agents = parse_active_agents((gateway["runtime"] or {}).get("active_agents", 0))
  497. # Off-loop: on a cold Windows install the first import of hermes_cli.gateway blocks
  498. # 15-30s (.pyc compilation + Defender), exceeding the desktop handshake's 15s timeout.
  499. restart_drain_timeout = await run_in_threadpool(_resolve_restart_drain_timeout)
  500. auth = _auth_gate_status()
  501. status = {
  502. "version": local["version"], "release_date": __release_date__,
  503. "config_version": local["config_version"], "latest_config_version": local["latest_config_version"],
  504. "can_update_hermes": local["can_update_hermes"],
  505. "gateway_running": gateway_running, "gateway_state": gateway_state,
  506. "gateway_platforms": gateway["gateway_platforms"],
  507. "gateway_exit_reason": gateway["gateway_exit_reason"],
  508. "gateway_updated_at": gateway["gateway_updated_at"],
  509. # Seconds since housekeeping last stamped the heartbeat, only when the PID is alive but the
  510. # stamp is past the freshness TTL (loop/housekeeping wedged, #113372); else null.
  511. "gateway_heartbeat_stale_s": gateway["gateway_heartbeat_stale_s"],
  512. # Non-null only for a profile served by the shared multiplexer: every profile that process carries.
  513. "gateway_shared_with": gateway["gateway_shared_with"],
  514. "active_agents": active_agents,
  515. "gateway_busy": derive_gateway_busy(
  516. gateway_running=gateway_running, gateway_state=gateway_state,
  517. active_agents=active_agents),
  518. "gateway_drainable": derive_gateway_drainable(
  519. gateway_running=gateway_running, gateway_state=gateway_state),
  520. "restart_drain_timeout": restart_drain_timeout, "active_sessions": active_sessions,
  521. **auth, "nous_session_valid": local["nous_session_valid"]}
  522. # Stable per-install identity (first call may touch disk). Omitted (not null) when
  523. # unpersistable so older-client behavior and the no-identity fallback stay identical.
  524. install_id = await run_in_threadpool(get_install_id)
  525. if install_id:
  526. status["install_id"] = install_id
  527. # Advisory only. Expose no paths or process identities on this public probe.
  528. from hermes_cli.shared_profile_warning import shared_profile_warning
  529. status["shared_profile_warning"] = bool(await run_in_threadpool(shared_profile_warning))
  530. components = await _component_health(gateway)
  531. status["components"] = components
  532. status["overall"] = ("ok" if all(item.get("status") == "ok" for item in components.values())
  533. else "degraded")
  534. await _advisory_pressure(status, profile_dir if profile_dir else get_hermes_home())
  535. # Profile NAMES and ``gateway_mode`` are low-sensitivity product surface (Hermes Cloud
  536. # renders the profile list over a gated bind) so they survive the auth gate; the
  537. # per-gateway ``gateways[]`` carries host ports and stays gated below.
  538. status["profiles"] = topology["profiles"]
  539. status["parked_profiles"] = topology.get("parked_profiles", [])
  540. status["gateway_mode"] = topology["gateway_mode"]
  541. status["multiplex_standalone_reason"] = topology.get("multiplex_standalone_reason")
  542. # Host paths, gateway PID, internal health URL and per-gateway ports are deployment
  543. # recon a liveness probe never needs, and on a gated bind *any* unauthenticated caller
  544. # reaches this endpoint — surface them only on a loopback / ``--insecure`` bind.
  545. if not auth["auth_required"]:
  546. status.update({
  547. "hermes_home": str(get_hermes_home()), "config_path": str(get_config_path()),
  548. "env_path": str(get_env_path()), "gateway_pid": gateway["gateway_pid"],
  549. "gateway_health_url": _GATEWAY_HEALTH_URL, "gateways": topology["gateways"]})
  550. if signed_out:
  551. return {key: value for key, value in status.items() if key in _WEBAPP_SIGNED_OUT_STATUS_KEYS}
  552. return status
  553. finally:
  554. if status_scope is not None:
  555. status_scope.__exit__(*sys.exc_info())
  556. @router.get("/api/system/stats")
  557. def get_system_stats(): # sync: FastAPI runs it on a worker (psutil.cpu_percent blocks 100 ms)
  558. """Host + process system stats for the System page (stdlib identity; psutil CPU/memory/
  559. disk/uptime when available). Non-sensitive: no env values, no paths beyond hermes home."""
  560. import platform as _platform
  561. info: dict[str, Any] = {
  562. **_display_system_platform(
  563. system=_platform.system(), release=_platform.release(), version=_platform.version(),
  564. platform_label=_platform.platform()),
  565. "arch": _platform.machine(), "hostname": _platform.node(),
  566. "python_version": _platform.python_version(),
  567. "python_impl": _platform.python_implementation(),
  568. "hermes_version": get_version_info().base_version, "cpu_count": os.cpu_count()}
  569. def _disk():
  570. du = psutil.disk_usage(str(get_hermes_home()))
  571. info["disk"] = {"total": du.total, "used": du.used, "free": du.free, "percent": du.percent}
  572. def _cpu():
  573. info["cpu_percent"] = psutil.cpu_percent(interval=0.1)
  574. la = getattr(psutil, "getloadavg", None)
  575. if la:
  576. info["load_avg"] = list(la())
  577. def _uptime():
  578. info["uptime_seconds"] = int(time.time() - psutil.boot_time())
  579. def _process():
  580. proc = psutil.Process()
  581. info["process"] = {"pid": proc.pid, "rss": proc.memory_info().rss,
  582. "create_time": int(proc.create_time()),
  583. "num_threads": proc.num_threads()}
  584. # psutil enriches the picture when present; every probe below is optional.
  585. try:
  586. import psutil # type: ignore
  587. vm = psutil.virtual_memory()
  588. info["memory"] = {"total": vm.total, "available": vm.available, "used": vm.used,
  589. "percent": vm.percent}
  590. for fill in (_disk, _cpu, _uptime, _process):
  591. try:
  592. fill()
  593. except Exception:
  594. pass
  595. info["psutil"] = True
  596. except Exception:
  597. info["psutil"] = False
  598. # stdlib-only fallbacks for load average where the kernel exposes it.
  599. try:
  600. info["load_avg"] = list(os.getloadavg())
  601. except (OSError, AttributeError):
  602. pass
  603. return info
  604. # Curator — background skill-maintenance status + the pause/resume/run-now controls.
  605. @router.get("/api/curator")
  606. async def get_curator_status(profile: Optional[str] = None):
  607. try:
  608. from agent import curator
  609. except Exception as exc:
  610. raise HTTPException(status_code=500, detail=f"Curator unavailable: {exc}")
  611. def _run():
  612. state = _safe_call(curator, "load_state", {})
  613. return {
  614. "enabled": _safe_call(curator, "is_enabled", True),
  615. "paused": _safe_call(curator, "is_paused", False),
  616. "interval_hours": _safe_call(curator, "get_interval_hours", None),
  617. "last_run_at": state.get("last_run_at"),
  618. **{key: _safe_call(curator, f"get_{key}", None)
  619. for key in ("min_idle_hours", "stale_after_days", "archive_after_days")}}
  620. return await config_scoped_to_thread(profile, _run)
  621. @router.put("/api/curator/paused")
  622. async def set_curator_paused(body: CuratorPause, profile: Optional[str] = None):
  623. from agent import curator
  624. # ``_state_file()`` is ``get_hermes_home()/skills/.curator_state`` resolved at call
  625. # time, so the request's home override is what decides which profile pauses.
  626. await config_scoped_to_thread(profile, lambda: curator.set_paused(bool(body.paused)))
  627. return {"ok": True, "paused": bool(body.paused)}
  628. def _spawn_action(argv: list, name: str, prefix: str, profile: Optional[str] = None) -> dict:
  629. """Spawn a background ``hermes -p <profile> <argv>`` action; a spawn failure is
  630. ``500 "<prefix>: <exc>"``."""
  631. try:
  632. proc = _spawn_hermes_action(_profile_cli_args(profile) + argv, name)
  633. except Exception as exc:
  634. raise HTTPException(status_code=500, detail=f"{prefix}: {exc}")
  635. return {"ok": True, "pid": proc.pid, "name": name}
  636. @router.post("/api/curator/run")
  637. async def run_curator(profile: Optional[str] = None):
  638. """Trigger a curator review now (backgrounded; tail via action status). The curator
  639. archives and rewrites skills, so an unnamed target is refused while this backend
  640. serves several profiles."""
  641. return _spawn_action(["curator", "run"], "curator-run", "Failed to run curator",
  642. destructive_profile(profile, "POST /api/curator/run"))
  643. @router.get("/api/learning/graph")
  644. async def get_learning_graph(profile: Optional[str] = None):
  645. """Learning graph for the desktop panel: profile-scoped learned skills + memory chunks."""
  646. def _run():
  647. from agent.learning_graph import build_learning_graph
  648. return build_learning_graph()
  649. try:
  650. # _profile_scope takes _SKILLS_PROFILE_LOCK and the graph build reads skills/memories
  651. # from disk — keep it off the event loop.
  652. return await scoped_to_thread(profile, _run)
  653. except HTTPException:
  654. raise # an unknown ?profile= is the scope's 404, not a graph failure
  655. except Exception:
  656. _log.exception("GET /api/learning/graph failed")
  657. raise HTTPException(status_code=500, detail="Failed to build learning graph")
  658. async def _learning_mutation(profile: Optional[str], fn, status: int, fallback: str):
  659. """Run a learning_mutations call under ``_profile_scope`` off-loop; a non-ok result
  660. becomes ``HTTPException(status, message)``."""
  661. res = await scoped_to_thread(profile, fn)
  662. if not res.get("ok"):
  663. raise HTTPException(status_code=status, detail=res.get("message", fallback))
  664. return res
  665. @router.get("/api/learning/node")
  666. async def get_learning_node(id: str, profile: Optional[str] = None):
  667. """Current content of a journey node (skill SKILL.md or memory chunk), for an edit prefill."""
  668. from agent.learning_mutations import node_detail
  669. return await _learning_mutation(profile, lambda: node_detail(id), 404, "not found")
  670. @router.delete("/api/learning/node")
  671. async def delete_learning_node(body: LearningNodeRef, profile: Optional[str] = None):
  672. """Delete a journey node — skills are archived (restorable), memories removed.
  673. ``?profile=`` is honoured too: a shared-backend Desktop scopes this call by query only, and
  674. ignoring it archived the same-named skill of the launch profile instead."""
  675. from agent.learning_mutations import delete_node
  676. return await _learning_mutation(
  677. body.profile or profile, lambda: delete_node(body.id), 400, "delete failed")
  678. @router.put("/api/learning/node")
  679. async def update_learning_node(body: LearningNodeEdit, profile: Optional[str] = None):
  680. """Rewrite a journey node's content (SKILL.md or memory chunk); profile as for DELETE."""
  681. from agent.learning_mutations import edit_node
  682. return await _learning_mutation(
  683. body.profile or profile, lambda: edit_node(body.id, body.content), 400, "edit failed")
  684. # Portal — Nous Portal auth + Tool Gateway routing status (read-only).
  685. @router.get("/api/portal")
  686. async def get_portal_status(profile: Optional[str] = None):
  687. # load_config() + auth/subscription snapshots are disk reads on a polled endpoint —
  688. # keep them off the event loop.
  689. return await config_scoped_to_thread(profile, _get_portal_status_sync)
  690. def _feature_state(feat) -> str:
  691. if getattr(feat, "managed_by_nous", False):
  692. return "via Nous Portal"
  693. if getattr(feat, "active", False):
  694. return getattr(feat, "current_provider", None) or "active"
  695. return "not configured"
  696. def _get_portal_status_sync():
  697. cfg = load_config() or {}
  698. auth: dict[str, Any] = {}
  699. try:
  700. from hermes_cli.auth import get_nous_auth_status_local
  701. # Refresh-free snapshot so polling never performs an OAuth refresh.
  702. auth = get_nous_auth_status_local() or {}
  703. except Exception:
  704. auth = {}
  705. features = []
  706. try:
  707. from hermes_cli.nous_subscription import get_nous_subscription_features
  708. feats = get_nous_subscription_features(cfg)
  709. if feats is not None:
  710. features = [{"label": getattr(feat, "label", ""), "state": _feature_state(feat)}
  711. for feat in feats.items()]
  712. except Exception:
  713. _log.exception("portal features failed")
  714. model_cfg = cfg.get("model") if isinstance(cfg.get("model"), dict) else {}
  715. return {
  716. "logged_in": bool(auth.get("logged_in")), "portal_url": auth.get("portal_base_url"),
  717. "inference_url": auth.get("inference_base_url"),
  718. "provider": str((model_cfg or {}).get("provider") or ""),
  719. # Free tier: a token exists, so logged_in stays true for callers that only ask "is there a
  720. # credential"; surfaces that render an account must branch on free_tier first.
  721. "free_tier": bool(auth.get("free_tier")), "account_tier": auth.get("account_tier"),
  722. "subscription_url": "https://portal.nousresearch.com/manage-subscription",
  723. "features": features}
  724. # Diagnostics: text-output actions spawned in the background, tailed via /api/actions/<name>.
  725. @router.post("/api/ops/prompt-size")
  726. async def run_prompt_size(profile: Optional[str] = None):
  727. return _spawn_action(["prompt-size"], "prompt-size", "Failed", profile)
  728. @router.post("/api/ops/dump")
  729. async def run_dump(profile: Optional[str] = None):
  730. return _spawn_action(["dump"], "dump", "Failed", profile)
  731. @router.post("/api/ops/config-migrate")
  732. async def run_config_migrate(profile: Optional[str] = None):
  733. return _spawn_action(["config", "migrate"], "config-migrate", "Failed", profile)
  734. @router.post("/api/ops/debug-share")
  735. async def run_debug_share_endpoint(body: DebugShareRequest | None = None,
  736. profile: Optional[str] = None):
  737. """Upload a redacted debug report + full logs and return the paste URLs. Synchronous,
  738. unlike the other diagnostics actions: the point is the shareable URLs, returned as a
  739. structured payload the dashboard renders as copyable links."""
  740. from hermes_cli.debug import build_debug_share
  741. from hermes_cli.debug_redaction import redact_debug_support_text
  742. req = body or DebugShareRequest()
  743. try:
  744. result = await config_scoped_to_thread(profile, lambda: build_debug_share(
  745. log_lines=max(1, min(int(req.lines), 5000)), redact=bool(req.redact)))
  746. except HTTPException:
  747. raise # an unknown ?profile= is the scope's 404, not a failed share
  748. except RuntimeError as exc:
  749. # Required summary-report upload failed (offline / paste service down).
  750. error = redact_debug_support_text(exc)
  751. raise HTTPException(status_code=502, detail=f"Upload failed: {error}")
  752. except Exception as exc:
  753. _log.exception("debug share failed")
  754. error = redact_debug_support_text(exc)
  755. raise HTTPException(status_code=500, detail=f"Failed: {error}")
  756. return {"ok": True, "urls": result.urls, "failures": result.failures,
  757. "redacted": result.redacted, "auto_delete_seconds": result.auto_delete_seconds}
  758. @logs_router.get("/api/logs")
  759. async def get_logs(
  760. file: str = "agent", lines: int = 100, level: Optional[str] = None,
  761. component: Optional[str] = None, search: Optional[str] = None,
  762. profile: Optional[str] = None):
  763. from hermes_cli.logs import _read_tail, LOG_FILES
  764. log_name = LOG_FILES.get(file)
  765. if not log_name:
  766. raise HTTPException(status_code=400, detail=f"Unknown log file: {file}")
  767. with _config_profile_scope(profile):
  768. log_path = get_hermes_home() / "logs" / log_name
  769. if not log_path.exists():
  770. return {"file": file, "lines": []}
  771. try:
  772. from hermes_logging import COMPONENT_PREFIXES
  773. except ImportError:
  774. COMPONENT_PREFIXES = {}
  775. # "ALL"/"all"/empty → no filter (None, not (): _matches_filters treats an empty tuple as
  776. # "must match a prefix", which silently drops every line).
  777. min_level = level if level and level.upper() != "ALL" else None
  778. comp_prefixes = None
  779. if component and component.lower() != "all":
  780. comp_prefixes = COMPONENT_PREFIXES.get(component)
  781. if comp_prefixes is None:
  782. raise HTTPException(status_code=400, detail=f"Unknown component: {component}. "
  783. f"Available: {', '.join(sorted(COMPONENT_PREFIXES))}")
  784. def _load_logs():
  785. result = _read_tail(
  786. log_path, min(lines, 500) if not search else 2000,
  787. has_filters=bool(min_level or comp_prefixes or search),
  788. min_level=min_level, component_prefixes=comp_prefixes)
  789. # _read_tail doesn't support free-text search, so post-filter (case-insensitive
  790. # substring) here and trim to the requested line count afterward.
  791. if search:
  792. needle = search.lower()
  793. result = [line for line in result if needle in line.lower()][-min(lines, 500):]
  794. return result
  795. result = await asyncio.to_thread(_load_logs)
  796. return {"file": file, "lines": result}