| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934 |
- """Status dashboard routes: health, /api/status, system stats, curator, learning graph,
- portal and diagnostics actions.
- Extracted from ``hermes_cli.web_server``; app state and helpers are late-bound through
- :mod:`hermes_cli.web_deps` (cycle-safe, monkeypatch-friendly).
- """
- import concurrent.futures
- import importlib
- import logging
- import re
- import asyncio
- import os
- import sys
- import time
- from fastapi import APIRouter
- from hermes_cli.web_deps import LateState, late
- from hermes_cli.web_server_gateway import _display_system_platform
- from starlette.concurrency import run_in_threadpool
- from fastapi import HTTPException, Request
- from gateway.status import (
- derive_gateway_busy, derive_gateway_drainable, normalize_updated_at, parse_active_agents,
- profile_platforms_from_multiplexer, resolve_gateway_liveness,
- retained_gateway_state,
- runtime_status_heartbeat_age_s, runtime_status_is_stale)
- from gateway.status_served import local_liveness_pid
- from hermes_cli import __release_date__
- from hermes_cli.config import get_config_path, get_env_path
- from hermes_cli.version_info import get_version_info
- from hermes_constants import get_process_hermes_home, profile_name_for_home
- from hermes_cli.web_models import CuratorPause, LearningNodeRef, LearningNodeEdit, DebugShareRequest
- from hermes_cli.web_routers._common import config_scoped_to_thread, destructive_profile, scoped_to_thread
- from pathlib import Path
- from typing import Any, Dict, Optional
- _log = logging.getLogger("hermes_cli.web_server")
- router = APIRouter()
- # Mounted separately by web_server so /api/logs keeps its original route-table position.
- logs_router = APIRouter()
- # Late-bound so a test's monkeypatch on the owning module wins at call time.
- _collect_profile_gateway_topology_cached = late("_collect_profile_gateway_topology_cached", "hermes_cli.web_server_gateway")
- _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
- _dashboard_local_update_managed_externally = late("_dashboard_local_update_managed_externally", "hermes_cli.web_server_files")
- _load_configured_gateway_platforms = late("_load_configured_gateway_platforms", "hermes_cli.web_server_gateway")
- _probe_gateway_health = late("_probe_gateway_health", "hermes_cli.web_server_gateway")
- _require_token = late("_require_token")
- _resolve_profile_dir = late("_resolve_profile_dir", "hermes_cli.web_server_profiles")
- _resolve_restart_drain_timeout = late("_resolve_restart_drain_timeout", "hermes_cli.web_server_lifecycle")
- _spawn_hermes_action = late("_spawn_hermes_action", "hermes_cli.web_server_gateway")
- _ssh_runtime_intact = late("_ssh_runtime_intact")
- app = LateState("app") # the FastAPI instance (app.state.*)
- check_config_version = late("check_config_version", "hermes_cli.config")
- get_hermes_home = late("get_hermes_home", "hermes_cli.config")
- _profile_cli_args = late("_profile_cli_args", "hermes_cli.web_server_profiles")
- get_install_id = late("get_install_id")
- get_running_pid_cached = late("get_running_pid_cached", "gateway.status")
- get_runtime_status_running_pid = late("get_runtime_status_running_pid", "gateway.status")
- served_profile_runtime = late("served_profile_runtime", "gateway.status_served")
- load_config = late("load_config", "hermes_cli.config")
- read_runtime_status = late("read_runtime_status", "gateway.status")
- _open_session_db_for_profile = late("_open_session_db_for_profile", "hermes_cli.web_server_sessions")
- _STATUS_ACTIVE_SESSIONS_TIMEOUT = 0.75
- _GATEWAY_HEALTH_ROUTE_TIMEOUT = 1.0
- _HEALTHY_PLATFORM_STATES = {"connected", "running", "ok"}
- def _safe_call(mod, fn_name: str, default):
- try:
- fn = getattr(mod, fn_name, None)
- return fn() if callable(fn) else default
- except Exception:
- return default
- def _count_status_active_sessions() -> int:
- """Best-effort status garnish. Opens read-only (via the shared stale-schema heal) so
- /api/status never routinely writes to state.db while another Hermes process uses it."""
- from hermes_state import _default_db_path
- # The heal helper bootstraps a missing store; this garnish must not — on a fresh install
- # /api/status polls would otherwise create state.db before the user's first session.
- if not Path(_default_db_path()).exists():
- return 0
- db = _open_session_db_for_profile(None, read_only=True)
- try:
- sessions = db.list_sessions_rich(limit=50, compact_rows=True)
- now = time.time()
- return sum(1 for s in sessions if s.get("ended_at") is None
- and (now - s.get("last_active", s.get("started_at", 0))) < 300)
- finally:
- db.close()
- async def _status_active_sessions() -> int:
- try:
- return await asyncio.wait_for(
- run_in_threadpool(_count_status_active_sessions),
- timeout=_STATUS_ACTIVE_SESSIONS_TIMEOUT)
- except TimeoutError:
- _log.debug("/api/status active session count exceeded %.2fs; returning 0",
- _STATUS_ACTIVE_SESSIONS_TIMEOUT)
- except Exception as exc:
- _log.debug("/api/status active session count unavailable: %s", exc)
- return 0
- @router.get("/api/ssh/ownership")
- async def get_ssh_ownership(request: Request):
- from hermes_cli.web_server import _SSH_OWNER_NONCE
- _require_token(request)
- if not _SSH_OWNER_NONCE:
- raise HTTPException(status_code=404, detail="SSH ownership is not active")
- return {"ok": True, "sshOwnerNonce": _SSH_OWNER_NONCE, "protocolVersion": 1,
- "runtimeIntact": _ssh_runtime_intact()}
- @router.get("/api/health")
- async def get_health():
- """Lightweight process liveness for desktop/backend readiness probes.
- ``commit`` is the code this process BOOTED from (``get_version_info`` is cached at
- ``web_server`` import): Desktop refuses to attach to a backend whose commit differs from
- its checkout, so a serve that outlived ``hermes update`` is never re-adopted.
- """
- info = get_version_info()
- return {"ok": True, "version": info.base_version, "displayVersion": info.display_version,
- "commit": info.commit,
- "auth_required": bool(getattr(app.state, "auth_required", False))}
- @router.get("/api/host/identity")
- async def get_host_identity(request: Request):
- """Prove to an attaching `hermes serve`/`dashboard` WHO owns this port.
- The host rendezvous record names a (pid, port) owner, but a record cannot say whether that
- owner still holds the port: a graceful-shutdown window or an unrelated listener that
- inherited the port both look identical on disk. The attaching side dials this endpoint with
- the owner's 0600 token and attaches only when pid+role match. ``servesSpa`` is false for
- headless ``serve``, so a `hermes dashboard` user is never routed to a backend with no UI.
- """
- _require_token(request)
- # ``role`` is the host ROLE this process published (gateway/host_rendezvous.ROLE_SERVE, or
- # ROLE_DESKTOP_SERVE for a Desktop-owned child), not the launch mode: `hermes serve` and
- # `hermes dashboard` are one host role that differ in SPA.
- return {"ok": True, "protocolVersion": 1, "pid": os.getpid(),
- "role": getattr(app.state, "host_role", None) or "serve",
- "servesSpa": bool(getattr(app.state, "serves_spa", False))}
- @router.get("/api/health/idle")
- async def get_health_idle(request: Request):
- """Token-gated diagnostic snapshot; never a retirement permit. None means cannot prove idle."""
- from hermes_cli.web_server_idle_proof import idle_proof
- _require_token(request)
- return {"ok": True, **idle_proof()}
- @router.post("/api/health/retirement")
- async def post_health_retirement(request: Request):
- from hermes_cli.backend_retirement import retirement
- _require_token(request)
- try:
- body = await request.json()
- except ValueError:
- raise HTTPException(status_code=400, detail="JSON body required")
- if not isinstance(body, dict) or body.get("action") not in ("prepare", "commit", "cancel"):
- raise HTTPException(status_code=400, detail="action must be prepare, commit, or cancel")
- action = body["action"]
- if action == "prepare":
- return await run_in_threadpool(retirement.prepare)
- token = body.get("token")
- if not isinstance(token, str) or not token:
- raise HTTPException(status_code=400, detail="token required")
- return getattr(retirement, action)(token)
- # Profile segment mirrors hermes_cli.profiles._PROFILE_ID_RE. Platform segment mirrors the
- # Platform enum's normalized values: built-in members plus plugin directory names
- # (lowercased), which allow hyphens as well as underscores (e.g. ``reviewer:foo-bar``).
- _PROFILE_PLATFORM_STATUS_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z0-9][a-z0-9_-]{0,63}$")
- def _is_profile_platform_status_key(key: object) -> bool:
- """Accept only the runner's public ``<profile>:<platform>`` key grammar."""
- return isinstance(key, str) and bool(_PROFILE_PLATFORM_STATUS_KEY_RE.fullmatch(key))
- def _status_platform_key_allowed(key: object, configured: set[str] | None) -> bool:
- """Whether a runtime-status platform key may appear publicly: namespaced
- ``<profile>:<platform>`` keys are validated against the grammar *unconditionally* (a
- failed config-set load must not fail open into projecting arbitrary keys from a
- process-local JSON file); plain keys are checked against the configured set when it
- loaded, passed through when it did not."""
- if not isinstance(key, str):
- return False
- if ":" in key:
- return _is_profile_platform_status_key(key)
- return configured is None or key in configured
- # Per-entry writer-identity stamps (added by gateway.status.write_runtime_status for the
- # aggregation ownership check) are process recon — the same class of detail as the
- # auth-gated top-level ``gateway_pid`` — and must not project onto the public endpoint.
- _PRIVATE_PLATFORM_ENTRY_KEYS = frozenset({"writer_pid", "writer_start_time"})
- def _public_platform_entry(value: Any) -> Any:
- """Strip writer-identity stamps from a platform entry before projection."""
- if not isinstance(value, dict):
- return value
- return {k: v for k, v in value.items() if k not in _PRIVATE_PLATFORM_ENTRY_KEYS}
- def _merge_profile_gateway_platforms(gateway_platforms: dict, profile_platforms: dict) -> dict:
- """Merge independent per-profile gateway platform states: hosts running separate gateway
- services per profile (``gateway_mode == "multiple"``) persist each profile's platform
- failures in its own ``gateway_state.json``, invisible to the machine-level probe NAS
- reads unless folded in under the validated ``<profile>:<platform>`` grammar. The active
- profile's own map is skipped (already present); existing keys are never overwritten."""
- try:
- from hermes_cli.profiles import get_active_profile_name
- active = get_active_profile_name()
- except Exception:
- active = "default"
- merged = dict(gateway_platforms)
- for prof, plats in (profile_platforms or {}).items():
- if prof == active or not isinstance(plats, dict):
- continue
- for key, value in plats.items():
- if not isinstance(key, str) or ":" in key or not isinstance(value, dict):
- continue
- namespaced = f"{prof}:{key}"
- if not _is_profile_platform_status_key(namespaced):
- continue
- merged.setdefault(namespaced, _public_platform_entry(value))
- return merged
- # --- Gateway liveness detection --- Delegated to the single shared ladder in gateway.status so this
- # endpoint and /api/messaging/platforms can never disagree about whether the gateway is up (they used to:
- # sidebar "running" while the Channels page rendered "The gateway is not running"). When ?profile=<name> was
- # given, scope PID and state reads to that profile's directory — gateway identity files (PID, lock, runtime
- # status) are written to the per-profile home, not the process-level HERMES_HOME (see issue #69143). Plain
- # /api/status keeps the exact zero-arg call so its behavior (and cache signature) is unchanged. The
- # module-level probe references are handed to the resolver so the long-standing
- # `monkeypatch.setattr(gateway.status, "get_running_pid_cached", ...)` seam used across the test-suite still
- # intercepts them.
- def _bounded_health_probe():
- """Health probe with the route's blocking-call budget preserved. The resolver only
- reaches this rung when the local PID probe came up empty, so the timeout is paid at
- most once per request and only in the cross-container case that needs it."""
- with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
- future = pool.submit(_probe_gateway_health)
- try:
- return future.result(timeout=_GATEWAY_HEALTH_ROUTE_TIMEOUT)
- except concurrent.futures.TimeoutError:
- _log.warning("/api/status gateway health probe exceeded %.2fs; using local status",
- _GATEWAY_HEALTH_ROUTE_TIMEOUT)
- return False, None
- except Exception:
- return False, None
- def _project_gateway_platforms(gateway_platforms: dict, configured: set[str] | None,
- gateway_running: bool, gateway_state) -> dict:
- """Public projection of a runtime's platform map (see ``_status_platform_key_allowed``
- for the key rules). A cleanly stopped gateway's platform states are stale noise and are
- cleared so a dead process can't report "connected"; a startup_failed gateway's FATAL
- entries are the diagnosis (credential collisions, auth failures) that the single
- exit_reason string can't express, so they are kept — upstream writer-identity/freshness
- filtering already dropped other processes' entries."""
- platforms = {key: _public_platform_entry(value) for key, value in gateway_platforms.items()
- if _status_platform_key_allowed(key, configured)}
- if gateway_running:
- return platforms
- if gateway_state == "startup_failed":
- return {key: value for key, value in platforms.items()
- if isinstance(value, dict) and value.get("state") == "fatal"}
- return {}
- async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> dict[str, Any]:
- """Liveness + runtime-state readout (running/pid/state/platforms/exit_reason/updated_at
- plus the raw ``runtime`` document).
- Liveness is delegated to the shared ladder in gateway.status so this endpoint and
- /api/messaging/platforms can never disagree. With ``?profile=<name>`` PID/state reads are
- scoped to that profile's directory (gateway identity files live in the per-profile
- home); plain /api/status keeps the exact zero-arg call so its cache signature is
- unchanged. The module-level probe references are handed to the resolver so the
- ``monkeypatch.setattr(gateway.status, "get_running_pid_cached", ...)`` seam still intercepts.
- """
- local_runtime = (read_runtime_status(path=profile_dir / "gateway_state.json")
- if profile_dir else read_runtime_status())
- liveness = await run_in_threadpool(lambda: resolve_gateway_liveness(
- profile_dir=profile_dir, runtime=local_runtime,
- health_probe=_bounded_health_probe if health_url else None,
- pid_probe=get_running_pid_cached, runtime_reader=read_runtime_status,
- runtime_pid_probe=get_runtime_status_running_pid))
- gateway_running = liveness.running
- remote_health_body: dict | None = liveness.health_body
- try:
- configured = await run_in_threadpool(_load_configured_gateway_platforms)
- except Exception:
- configured = None
- # Prefer the detailed health endpoint response (has full state) when the local runtime
- # status file is absent or stale (cross-container).
- runtime = local_runtime
- if runtime is None and remote_health_body and remote_health_body.get("gateway_state"):
- runtime = remote_health_body
- # Unscoped, the profile is the process's own home (a pooled ``hermes --profile X serve``).
- own_home = profile_dir or get_process_hermes_home()
- if liveness.runtime is not None:
- # Served by the multiplexer: its record is this profile's runtime, with the profile's own
- # adapters under ``<profile>:<platform>`` re-keyed to the standalone shape.
- # Fold on the profile NAME, never ``profile_dir.name``: ``?profile=default`` resolves the
- # root itself, whose basename (``.hermes``) matched nothing and read as a named id (#123088).
- served_name = profile_name_for_home(own_home)
- runtime = {**liveness.runtime,
- "platforms": profile_platforms_from_multiplexer(liveness.runtime, served_name or "")}
- elif local_liveness_pid(liveness) is not None:
- # The PID rung answers before the multiplexer rung, and a served profile's gateway.lock is
- # held by the multiplexer itself, so the ladder returns without its record while the
- # profile's own gateway_state.json may be a retired standalone gateway's (#139080). The
- # shared rule (also the messaging card's) picks the multiplexer record only when the lock
- # holder IS the multiplexer and the profile's own record proves no live gateway.
- try:
- runtime = await run_in_threadpool(
- served_profile_runtime, own_home, local_runtime, gateway_pid=local_liveness_pid(liveness)) or runtime
- except Exception: # health: allow BLE001 -- status route must never 500; traceback is logged
- # A failed lookup keeps the profile's own record.
- _log.warning("/api/status multiplexer record lookup failed", exc_info=True)
- gateway_state = None
- gateway_platforms: dict = {}
- gateway_exit_reason = None
- gateway_updated_at = None
- gateway_heartbeat_stale_s = None
- if runtime:
- gateway_state = runtime.get("gateway_state")
- if not gateway_running:
- # Shared with /api/messaging/platforms: a durable operator stop outranks a retained
- # ``startup_failed`` / watchdog ``degraded`` (kept on disk for diagnostics), so the
- # overview does not alarm on it.
- gateway_state = retained_gateway_state(runtime)
- elif remote_health_body is not None and gateway_state in {None, "stopped"}:
- # The health probe confirmed the gateway is alive, but the local runtime status
- # file may be stale (cross-container): override so the badge is correct.
- gateway_state = "running"
- elif gateway_state in {"running", "degraded", "starting"} and runtime_status_is_stale(runtime):
- # Alive PID, but housekeeping stopped re-stamping the heartbeat: the loop or the
- # housekeeping thread wedged while the file still says 'running' (#113372). Same arm
- # as ``hermes gateway status`` so the sidebar strip and the CLI agree.
- gateway_heartbeat_stale_s = runtime_status_heartbeat_age_s(runtime)
- gateway_platforms = _project_gateway_platforms(
- runtime.get("platforms") or {}, configured, gateway_running, gateway_state)
- gateway_exit_reason = None if gateway_state == "stopped" else runtime.get("exit_reason")
- # Contract: gateway_updated_at is RFC3339 string | null, never a number. ``runtime``
- # may be the local gateway_state.json (legacy gateways wrote epoch floats; hand
- # edits can inject anything) or a remote /health/detailed body — normalize both.
- gateway_updated_at = normalize_updated_at(runtime.get("updated_at"))
- # No runtime info at all but the health probe confirmed alive (no shared volume).
- if gateway_running and gateway_state is None and remote_health_body is not None:
- gateway_state = "running"
- # A live multiplexer's record (the host's own, or the one serving this profile) names every
- # profile a restart would blip ("default, alpha, beta"); a standalone gateway's names none.
- shared_with = (runtime or {}).get("served_profiles") if gateway_running else None
- return {
- "runtime": runtime, "gateway_running": gateway_running, "gateway_pid": liveness.pid,
- "gateway_state": gateway_state, "gateway_platforms": gateway_platforms,
- "gateway_exit_reason": gateway_exit_reason, "gateway_updated_at": gateway_updated_at,
- "gateway_heartbeat_stale_s": gateway_heartbeat_stale_s,
- "gateway_shared_with": [str(p) for p in shared_with] if isinstance(shared_with, list) else None}
- def _auth_gate_status() -> dict[str, Any]:
- """Dashboard auth gate readout: gate engaged, registered providers, and the RFC 8252
- native-app capability advertisement ``auth_flows`` the desktop reads to pick the
- system-browser + loopback + PKCE flow over the embedded-webview cookie flow. "cookie" is
- always available in gated mode; "native_pkce" when at least one interactive session
- provider is registered (token-only credentials such as drain, and provisioned-credential
- session providers such as session-token, don't count). Missing
- "native_pkce" ⇒ older gateway ⇒ desktop falls back automatically."""
- auth_required = bool(getattr(app.state, "auth_required", False))
- auth_providers: list[str] = []
- auth_flows: list[str] = []
- try:
- from hermes_cli.dashboard_auth import (
- list_interactive_providers as _list_interactive_providers,
- list_providers as _list_providers)
- auth_providers = [p.name for p in _list_providers()]
- if auth_required:
- auth_flows.append("cookie")
- if _list_interactive_providers():
- auth_flows.append("native_pkce")
- except Exception:
- # Module not importable yet (early startup) — leave as [].
- pass
- return {"auth_required": auth_required, "auth_providers": auth_providers,
- "auth_flows": auth_flows}
- def _nous_session_validity() -> str:
- """Nous bootstrap-session validity for the NAS health sweep: a hosted agent whose Nous
- auth dies terminally looks HEALTHY to every liveness probe yet every inference turn
- fails, and this is the ONLY signal that surfaces it (local auth-store state, no token
- needed). Best-effort: never let auth classification break the probe."""
- try:
- from hermes_cli.auth import get_nous_session_validity
- return get_nous_session_validity()
- except Exception:
- return "unknown"
- async def _component_health(gateway: dict[str, Any]) -> dict[str, Any]:
- """Component-level health rollup: counts and status enums only (public payload — no
- messages, paths or other detail that could carry secrets). The storage probe reuses the
- gateway readiness state_db check (read-only, 1s-bounded) off-loop."""
- from hermes_cli.web_server import DASHBOARD_HEALTH
- gateway_running, gateway_state = gateway["gateway_running"], gateway["gateway_state"]
- gateway_platforms = gateway["gateway_platforms"]
- components: dict[str, Any] = {
- "gateway": {
- "status": "ok" if gateway_running and gateway_state in {"running", "draining"} else "degraded",
- "state": gateway_state or ("running" if gateway_running else "stopped")},
- "dashboard": DASHBOARD_HEALTH.snapshot()}
- try:
- from gateway.readiness import _probe_state_db
- storage_check = await run_in_threadpool(_probe_state_db, get_hermes_home())
- components["storage"] = {"status": storage_check.get("status", "degraded")}
- # The one reason enum consumers key off; same latch as readiness and the session lists.
- if storage_check.get("detail") == "corrupt":
- components["storage"]["reason"] = "corrupt"
- except Exception:
- components["storage"] = {"status": "degraded"}
- # ``disabled`` entries are platforms the multiplexer deliberately does not run for a served profile
- # (shared ingress owned by the default) — informational, never a degraded verdict.
- platform_states = [str(value.get("state") or value.get("status") or "").lower()
- for value in gateway_platforms.values() if isinstance(value, dict)]
- platform_states = [state for state in platform_states if state != "disabled"]
- connected = sum(1 for state in platform_states if state in _HEALTHY_PLATFORM_STATES)
- components["platforms"] = {"status": "ok" if connected == len(platform_states) else "degraded",
- "configured": len(platform_states), "connected": connected}
- return components
- async def _advisory_pressure(status: dict[str, Any], home: Path) -> None:
- """Memory / disk pressure rollups + deferred FTS rebuild progress (coarse numbers/enums
- only; public payload). Deliberately NOT folded into components/overall: pressure is
- advisory, not a liveness verdict, and flipping ``overall`` on it would page NAS's
- availability sweep for a condition the valve is already handling. Read-only, never raise.
- """
- for key, mod_name, fn_name in (("memory", "gateway.memory_status", "collect_memory_status"),
- ("disk", "gateway.disk_status", "collect_disk_status")):
- try:
- collect = getattr(importlib.import_module(mod_name), fn_name)
- status[key] = await run_in_threadpool(collect, home)
- except Exception:
- status[key] = {"pressure": "unknown"}
- try:
- # SQLite open + read: off the loop like the rest of the probe.
- _rebuild = await run_in_threadpool(_fts_rebuild_status)
- if _rebuild is not None:
- status["fts_rebuild"] = _rebuild
- except Exception:
- pass
- def _status_local_reads() -> dict[str, Any]:
- """The probe's synchronous local reads: the raw config.yaml stamp, the install method, the
- auth store and (first call only) git provenance. Run on a worker thread: on the event loop that
- also serves the session WebSocket they stalled every Desktop request behind the disk (#143561)."""
- current_ver, latest_ver = check_config_version()
- return {"version": get_version_info().base_version, "config_version": current_ver,
- "latest_config_version": latest_ver,
- "can_update_hermes": not _dashboard_local_update_managed_externally(),
- "nous_session_valid": _nous_session_validity()}
- def _fts_rebuild_status() -> Optional[dict[str, Any]]:
- """Deferred FTS rebuild progress of this home's state.db (read-only open), or None."""
- from hermes_state import SessionDB as _SDB
- from hermes_constants import get_hermes_home as _ghh
- _db_path = _ghh() / "state.db"
- if not _db_path.exists():
- return None
- _sdb = _SDB(db_path=_db_path, read_only=True)
- try:
- return _sdb.fts_rebuild_status()
- finally:
- _sdb.close()
- #: What the gateway's Webapp answers a caller who has not signed in: liveness, version and the
- #: sign-in shape (the native-app login probe and uptime checks read nothing else). Profile names,
- #: the install id and host figures wait for sign-in, since a Webapp on a wildcard bind is reachable
- #: by everyone on the network. The standalone dashboard keeps its public body: Hermes Cloud reads
- #: the profile list there without a session (#60585).
- _WEBAPP_SIGNED_OUT_STATUS_KEYS = frozenset({
- "version", "release_date", "gateway_running", "gateway_state", "overall", "components",
- "auth_required", "auth_providers", "auth_flows"})
- async def _webapp_caller_signed_out(request: Request) -> bool:
- """True when this request reached the gateway's Webapp behind its sign-in gate with no
- session the gate would accept. ``/api/status`` is public, so the gate never verified one."""
- from hermes_cli.web_server import _require_token
- from hermes_cli.web_server_surface import surface
- if request is None or surface(request.app.state) != "webapp" \
- or not getattr(request.app.state, "auth_required", False):
- return False
- if getattr(request.state, "session", None) is None:
- from hermes_cli.dashboard_auth.middleware import peek_session
- request.state.session = await run_in_threadpool(peek_session, request)
- try:
- _require_token(request)
- except HTTPException:
- return True
- return False
- @router.get("/api/status")
- async def get_status(profile: Optional[str] = None,
- request: Request = None): # type: ignore[assignment] # None when awaited in-process
- """Public machine-level liveness probe (``PUBLIC_API_PATHS``): version, gateway state,
- active session count and the auth-gate shape — no bodies, no session content, no secrets.
- ``?profile=`` (dashboard management switcher) uses the config-only contextvar scope, NOT
- _profile_scope: this handler awaits the remote health probe, and _profile_scope swaps
- process-global skills-module attributes a concurrent request would cross-restore.
- """
- from hermes_cli.web_server import _GATEWAY_HEALTH_URL
- signed_out = await _webapp_caller_signed_out(request)
- status_scope = None
- # A signed-out caller's ``?profile=`` is ignored: an unknown name's 404 would tell it which exist.
- requested_profile = "" if signed_out else (profile or "").strip()
- profile_dir: Optional[Path] = None
- if requested_profile and requested_profile.lower() != "current":
- profile_dir = _resolve_profile_dir(requested_profile)
- status_scope = _config_profile_scope(requested_profile)
- status_scope.__enter__()
- try:
- local = await run_in_threadpool(_status_local_reads)
- gateway = await _resolve_gateway_status(profile_dir, _GATEWAY_HEALTH_URL)
- gateway_running, gateway_state = gateway["gateway_running"], gateway["gateway_state"]
- # Topology (cached, TTL 10s) is fetched before the platform rollup so per-profile
- # gateway failures fold into the machine-level view (see
- # _merge_profile_gateway_platforms); a ``?profile=`` request is left unmerged.
- topology = await run_in_threadpool(_collect_profile_gateway_topology_cached)
- if not requested_profile:
- gateway["gateway_platforms"] = _merge_profile_gateway_platforms(
- gateway["gateway_platforms"], topology.get("profile_platforms") or {})
- active_sessions = await _status_active_sessions()
- # Busy/drainable (NAS lifecycle-safety gate) derive from the persisted in-flight turn
- # count + liveness via gateway.status. Liveness keys off gateway_running, NEVER
- # gateway_updated_at — a stale heartbeat is reported separately, not treated as death.
- active_agents = parse_active_agents((gateway["runtime"] or {}).get("active_agents", 0))
- # Off-loop: on a cold Windows install the first import of hermes_cli.gateway blocks
- # 15-30s (.pyc compilation + Defender), exceeding the desktop handshake's 15s timeout.
- restart_drain_timeout = await run_in_threadpool(_resolve_restart_drain_timeout)
- auth = _auth_gate_status()
- status = {
- "version": local["version"], "release_date": __release_date__,
- "config_version": local["config_version"], "latest_config_version": local["latest_config_version"],
- "can_update_hermes": local["can_update_hermes"],
- "gateway_running": gateway_running, "gateway_state": gateway_state,
- "gateway_platforms": gateway["gateway_platforms"],
- "gateway_exit_reason": gateway["gateway_exit_reason"],
- "gateway_updated_at": gateway["gateway_updated_at"],
- # Seconds since housekeeping last stamped the heartbeat, only when the PID is alive but the
- # stamp is past the freshness TTL (loop/housekeeping wedged, #113372); else null.
- "gateway_heartbeat_stale_s": gateway["gateway_heartbeat_stale_s"],
- # Non-null only for a profile served by the shared multiplexer: every profile that process carries.
- "gateway_shared_with": gateway["gateway_shared_with"],
- "active_agents": active_agents,
- "gateway_busy": derive_gateway_busy(
- gateway_running=gateway_running, gateway_state=gateway_state,
- active_agents=active_agents),
- "gateway_drainable": derive_gateway_drainable(
- gateway_running=gateway_running, gateway_state=gateway_state),
- "restart_drain_timeout": restart_drain_timeout, "active_sessions": active_sessions,
- **auth, "nous_session_valid": local["nous_session_valid"]}
- # Stable per-install identity (first call may touch disk). Omitted (not null) when
- # unpersistable so older-client behavior and the no-identity fallback stay identical.
- install_id = await run_in_threadpool(get_install_id)
- if install_id:
- status["install_id"] = install_id
- # Advisory only. Expose no paths or process identities on this public probe.
- from hermes_cli.shared_profile_warning import shared_profile_warning
- status["shared_profile_warning"] = bool(await run_in_threadpool(shared_profile_warning))
- components = await _component_health(gateway)
- status["components"] = components
- status["overall"] = ("ok" if all(item.get("status") == "ok" for item in components.values())
- else "degraded")
- await _advisory_pressure(status, profile_dir if profile_dir else get_hermes_home())
- # Profile NAMES and ``gateway_mode`` are low-sensitivity product surface (Hermes Cloud
- # renders the profile list over a gated bind) so they survive the auth gate; the
- # per-gateway ``gateways[]`` carries host ports and stays gated below.
- status["profiles"] = topology["profiles"]
- status["parked_profiles"] = topology.get("parked_profiles", [])
- status["gateway_mode"] = topology["gateway_mode"]
- status["multiplex_standalone_reason"] = topology.get("multiplex_standalone_reason")
- # Host paths, gateway PID, internal health URL and per-gateway ports are deployment
- # recon a liveness probe never needs, and on a gated bind *any* unauthenticated caller
- # reaches this endpoint — surface them only on a loopback / ``--insecure`` bind.
- if not auth["auth_required"]:
- status.update({
- "hermes_home": str(get_hermes_home()), "config_path": str(get_config_path()),
- "env_path": str(get_env_path()), "gateway_pid": gateway["gateway_pid"],
- "gateway_health_url": _GATEWAY_HEALTH_URL, "gateways": topology["gateways"]})
- if signed_out:
- return {key: value for key, value in status.items() if key in _WEBAPP_SIGNED_OUT_STATUS_KEYS}
- return status
- finally:
- if status_scope is not None:
- status_scope.__exit__(*sys.exc_info())
- @router.get("/api/system/stats")
- def get_system_stats(): # sync: FastAPI runs it on a worker (psutil.cpu_percent blocks 100 ms)
- """Host + process system stats for the System page (stdlib identity; psutil CPU/memory/
- disk/uptime when available). Non-sensitive: no env values, no paths beyond hermes home."""
- import platform as _platform
- info: dict[str, Any] = {
- **_display_system_platform(
- system=_platform.system(), release=_platform.release(), version=_platform.version(),
- platform_label=_platform.platform()),
- "arch": _platform.machine(), "hostname": _platform.node(),
- "python_version": _platform.python_version(),
- "python_impl": _platform.python_implementation(),
- "hermes_version": get_version_info().base_version, "cpu_count": os.cpu_count()}
- def _disk():
- du = psutil.disk_usage(str(get_hermes_home()))
- info["disk"] = {"total": du.total, "used": du.used, "free": du.free, "percent": du.percent}
- def _cpu():
- info["cpu_percent"] = psutil.cpu_percent(interval=0.1)
- la = getattr(psutil, "getloadavg", None)
- if la:
- info["load_avg"] = list(la())
- def _uptime():
- info["uptime_seconds"] = int(time.time() - psutil.boot_time())
- def _process():
- proc = psutil.Process()
- info["process"] = {"pid": proc.pid, "rss": proc.memory_info().rss,
- "create_time": int(proc.create_time()),
- "num_threads": proc.num_threads()}
- # psutil enriches the picture when present; every probe below is optional.
- try:
- import psutil # type: ignore
- vm = psutil.virtual_memory()
- info["memory"] = {"total": vm.total, "available": vm.available, "used": vm.used,
- "percent": vm.percent}
- for fill in (_disk, _cpu, _uptime, _process):
- try:
- fill()
- except Exception:
- pass
- info["psutil"] = True
- except Exception:
- info["psutil"] = False
- # stdlib-only fallbacks for load average where the kernel exposes it.
- try:
- info["load_avg"] = list(os.getloadavg())
- except (OSError, AttributeError):
- pass
- return info
- # Curator — background skill-maintenance status + the pause/resume/run-now controls.
- @router.get("/api/curator")
- async def get_curator_status(profile: Optional[str] = None):
- try:
- from agent import curator
- except Exception as exc:
- raise HTTPException(status_code=500, detail=f"Curator unavailable: {exc}")
- def _run():
- state = _safe_call(curator, "load_state", {})
- return {
- "enabled": _safe_call(curator, "is_enabled", True),
- "paused": _safe_call(curator, "is_paused", False),
- "interval_hours": _safe_call(curator, "get_interval_hours", None),
- "last_run_at": state.get("last_run_at"),
- **{key: _safe_call(curator, f"get_{key}", None)
- for key in ("min_idle_hours", "stale_after_days", "archive_after_days")}}
- return await config_scoped_to_thread(profile, _run)
- @router.put("/api/curator/paused")
- async def set_curator_paused(body: CuratorPause, profile: Optional[str] = None):
- from agent import curator
- # ``_state_file()`` is ``get_hermes_home()/skills/.curator_state`` resolved at call
- # time, so the request's home override is what decides which profile pauses.
- await config_scoped_to_thread(profile, lambda: curator.set_paused(bool(body.paused)))
- return {"ok": True, "paused": bool(body.paused)}
- def _spawn_action(argv: list, name: str, prefix: str, profile: Optional[str] = None) -> dict:
- """Spawn a background ``hermes -p <profile> <argv>`` action; a spawn failure is
- ``500 "<prefix>: <exc>"``."""
- try:
- proc = _spawn_hermes_action(_profile_cli_args(profile) + argv, name)
- except Exception as exc:
- raise HTTPException(status_code=500, detail=f"{prefix}: {exc}")
- return {"ok": True, "pid": proc.pid, "name": name}
- @router.post("/api/curator/run")
- async def run_curator(profile: Optional[str] = None):
- """Trigger a curator review now (backgrounded; tail via action status). The curator
- archives and rewrites skills, so an unnamed target is refused while this backend
- serves several profiles."""
- return _spawn_action(["curator", "run"], "curator-run", "Failed to run curator",
- destructive_profile(profile, "POST /api/curator/run"))
- @router.get("/api/learning/graph")
- async def get_learning_graph(profile: Optional[str] = None):
- """Learning graph for the desktop panel: profile-scoped learned skills + memory chunks."""
- def _run():
- from agent.learning_graph import build_learning_graph
- return build_learning_graph()
- try:
- # _profile_scope takes _SKILLS_PROFILE_LOCK and the graph build reads skills/memories
- # from disk — keep it off the event loop.
- return await scoped_to_thread(profile, _run)
- except HTTPException:
- raise # an unknown ?profile= is the scope's 404, not a graph failure
- except Exception:
- _log.exception("GET /api/learning/graph failed")
- raise HTTPException(status_code=500, detail="Failed to build learning graph")
- async def _learning_mutation(profile: Optional[str], fn, status: int, fallback: str):
- """Run a learning_mutations call under ``_profile_scope`` off-loop; a non-ok result
- becomes ``HTTPException(status, message)``."""
- res = await scoped_to_thread(profile, fn)
- if not res.get("ok"):
- raise HTTPException(status_code=status, detail=res.get("message", fallback))
- return res
- @router.get("/api/learning/node")
- async def get_learning_node(id: str, profile: Optional[str] = None):
- """Current content of a journey node (skill SKILL.md or memory chunk), for an edit prefill."""
- from agent.learning_mutations import node_detail
- return await _learning_mutation(profile, lambda: node_detail(id), 404, "not found")
- @router.delete("/api/learning/node")
- async def delete_learning_node(body: LearningNodeRef, profile: Optional[str] = None):
- """Delete a journey node — skills are archived (restorable), memories removed.
- ``?profile=`` is honoured too: a shared-backend Desktop scopes this call by query only, and
- ignoring it archived the same-named skill of the launch profile instead."""
- from agent.learning_mutations import delete_node
- return await _learning_mutation(
- body.profile or profile, lambda: delete_node(body.id), 400, "delete failed")
- @router.put("/api/learning/node")
- async def update_learning_node(body: LearningNodeEdit, profile: Optional[str] = None):
- """Rewrite a journey node's content (SKILL.md or memory chunk); profile as for DELETE."""
- from agent.learning_mutations import edit_node
- return await _learning_mutation(
- body.profile or profile, lambda: edit_node(body.id, body.content), 400, "edit failed")
- # Portal — Nous Portal auth + Tool Gateway routing status (read-only).
- @router.get("/api/portal")
- async def get_portal_status(profile: Optional[str] = None):
- # load_config() + auth/subscription snapshots are disk reads on a polled endpoint —
- # keep them off the event loop.
- return await config_scoped_to_thread(profile, _get_portal_status_sync)
- def _feature_state(feat) -> str:
- if getattr(feat, "managed_by_nous", False):
- return "via Nous Portal"
- if getattr(feat, "active", False):
- return getattr(feat, "current_provider", None) or "active"
- return "not configured"
- def _get_portal_status_sync():
- cfg = load_config() or {}
- auth: dict[str, Any] = {}
- try:
- from hermes_cli.auth import get_nous_auth_status_local
- # Refresh-free snapshot so polling never performs an OAuth refresh.
- auth = get_nous_auth_status_local() or {}
- except Exception:
- auth = {}
- features = []
- try:
- from hermes_cli.nous_subscription import get_nous_subscription_features
- feats = get_nous_subscription_features(cfg)
- if feats is not None:
- features = [{"label": getattr(feat, "label", ""), "state": _feature_state(feat)}
- for feat in feats.items()]
- except Exception:
- _log.exception("portal features failed")
- model_cfg = cfg.get("model") if isinstance(cfg.get("model"), dict) else {}
- return {
- "logged_in": bool(auth.get("logged_in")), "portal_url": auth.get("portal_base_url"),
- "inference_url": auth.get("inference_base_url"),
- "provider": str((model_cfg or {}).get("provider") or ""),
- # Free tier: a token exists, so logged_in stays true for callers that only ask "is there a
- # credential"; surfaces that render an account must branch on free_tier first.
- "free_tier": bool(auth.get("free_tier")), "account_tier": auth.get("account_tier"),
- "subscription_url": "https://portal.nousresearch.com/manage-subscription",
- "features": features}
- # Diagnostics: text-output actions spawned in the background, tailed via /api/actions/<name>.
- @router.post("/api/ops/prompt-size")
- async def run_prompt_size(profile: Optional[str] = None):
- return _spawn_action(["prompt-size"], "prompt-size", "Failed", profile)
- @router.post("/api/ops/dump")
- async def run_dump(profile: Optional[str] = None):
- return _spawn_action(["dump"], "dump", "Failed", profile)
- @router.post("/api/ops/config-migrate")
- async def run_config_migrate(profile: Optional[str] = None):
- return _spawn_action(["config", "migrate"], "config-migrate", "Failed", profile)
- @router.post("/api/ops/debug-share")
- async def run_debug_share_endpoint(body: DebugShareRequest | None = None,
- profile: Optional[str] = None):
- """Upload a redacted debug report + full logs and return the paste URLs. Synchronous,
- unlike the other diagnostics actions: the point is the shareable URLs, returned as a
- structured payload the dashboard renders as copyable links."""
- from hermes_cli.debug import build_debug_share
- from hermes_cli.debug_redaction import redact_debug_support_text
- req = body or DebugShareRequest()
- try:
- result = await config_scoped_to_thread(profile, lambda: build_debug_share(
- log_lines=max(1, min(int(req.lines), 5000)), redact=bool(req.redact)))
- except HTTPException:
- raise # an unknown ?profile= is the scope's 404, not a failed share
- except RuntimeError as exc:
- # Required summary-report upload failed (offline / paste service down).
- error = redact_debug_support_text(exc)
- raise HTTPException(status_code=502, detail=f"Upload failed: {error}")
- except Exception as exc:
- _log.exception("debug share failed")
- error = redact_debug_support_text(exc)
- raise HTTPException(status_code=500, detail=f"Failed: {error}")
- return {"ok": True, "urls": result.urls, "failures": result.failures,
- "redacted": result.redacted, "auto_delete_seconds": result.auto_delete_seconds}
- @logs_router.get("/api/logs")
- async def get_logs(
- file: str = "agent", lines: int = 100, level: Optional[str] = None,
- component: Optional[str] = None, search: Optional[str] = None,
- profile: Optional[str] = None):
- from hermes_cli.logs import _read_tail, LOG_FILES
- log_name = LOG_FILES.get(file)
- if not log_name:
- raise HTTPException(status_code=400, detail=f"Unknown log file: {file}")
- with _config_profile_scope(profile):
- log_path = get_hermes_home() / "logs" / log_name
- if not log_path.exists():
- return {"file": file, "lines": []}
- try:
- from hermes_logging import COMPONENT_PREFIXES
- except ImportError:
- COMPONENT_PREFIXES = {}
- # "ALL"/"all"/empty → no filter (None, not (): _matches_filters treats an empty tuple as
- # "must match a prefix", which silently drops every line).
- min_level = level if level and level.upper() != "ALL" else None
- comp_prefixes = None
- if component and component.lower() != "all":
- comp_prefixes = COMPONENT_PREFIXES.get(component)
- if comp_prefixes is None:
- raise HTTPException(status_code=400, detail=f"Unknown component: {component}. "
- f"Available: {', '.join(sorted(COMPONENT_PREFIXES))}")
- def _load_logs():
- result = _read_tail(
- log_path, min(lines, 500) if not search else 2000,
- has_filters=bool(min_level or comp_prefixes or search),
- min_level=min_level, component_prefixes=comp_prefixes)
- # _read_tail doesn't support free-text search, so post-filter (case-insensitive
- # substring) here and trim to the requested line count afterward.
- if search:
- needle = search.lower()
- result = [line for line in result if needle in line.lower()][-min(lines, 500):]
- return result
- result = await asyncio.to_thread(_load_logs)
- return {"file": file, "lines": result}
|