| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125 |
- """Bounded, non-destructive readiness probes for authenticated health surfaces."""
- from __future__ import annotations
- import shutil
- import sqlite3
- from contextlib import closing
- from pathlib import Path
- from typing import Any
- from hermes_constants import get_hermes_home
- _DISK_DEGRADED_PERCENT = 90.0
- _CONNECTED_STATES = {"connected", "running", "ok"}
- def _check(status: str, detail: str | None = None, **extra: Any) -> dict[str, Any]:
- return {"status": status, **({"detail": detail} if detail else {}), **extra}
- def _probe_state_db(home: Path) -> dict[str, Any]:
- """Read-only schema probe plus the process-wide corruption latch (``hermes_state_health``).
- The schema read only catches an unreadable header or schema; damage deeper in the file
- surfaces when a reader or writer touches it, and those publish into the latch. Reporting
- the latch here is what makes readiness and ``/api/status`` agree with the session list
- (#72046). ``detail="corrupt"`` is the one reason string consumers key off."""
- from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state
- path = home / "state.db"
- if not path.exists():
- return _check("ok", "not initialized")
- if storage_state(path) == STORAGE_CORRUPT:
- return _check("degraded", STORAGE_CORRUPT)
- try:
- # Read-only schema query: catches unreadable/corrupt DBs without competing with
- # writers. ``closing`` is required — sqlite3's context manager only commits/rolls
- # back, never closes, so a bare ``with connect()`` leaks a connection per poll.
- with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True, timeout=1.0)) as conn:
- # A readiness probe must never compete with normal state writers. See #69567, #69678.
- conn.execute("PRAGMA query_only = ON")
- conn.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
- return _check("ok")
- except Exception as exc:
- if note_storage_error(path, exc):
- return _check("degraded", STORAGE_CORRUPT)
- return _check("degraded", type(exc).__name__)
- def _probe_config(home: Path) -> dict[str, Any]:
- from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists, read_config_doc_readonly
- path = home / "config.yaml"
- try:
- if not config_exists(path):
- return _check("ok", "using defaults")
- raw = read_config_doc_readonly(path)
- except ConfigBackendUnavailable:
- # A SystemExit on purpose (readers must not serve defaults); this probe exists to report it.
- return _check("degraded", "config backend unavailable")
- except Exception as exc:
- return _check("degraded", f"invalid config ({type(exc).__name__})")
- return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")
- def _probe_disk(home: Path) -> dict[str, Any]:
- try:
- usage = shutil.disk_usage(home)
- except Exception as exc:
- return _check("degraded", type(exc).__name__)
- used_pct = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
- return _check("degraded" if used_pct >= _DISK_DEGRADED_PERCENT else "ok", used_percent=used_pct, free_bytes=usage.free)
- def _probe_gateway(runtime_status: dict[str, Any]) -> dict[str, Any]:
- state = str(runtime_status.get("gateway_state") or "unknown")
- platforms = runtime_status.get("platforms")
- platforms = platforms if isinstance(platforms, dict) else {}
- connected = sum(
- isinstance(v, dict) and str(v.get("state") or v.get("status") or "").lower() in _CONNECTED_STATES
- for v in platforms.values()
- )
- return _check("ok" if state in {"running", "draining"} else "degraded", state=state,
- connected_platforms=connected, platforms=len(platforms))
- def _probe_session_store(runtime_status: dict[str, Any], state_db_probe: dict[str, Any]) -> dict[str, Any]:
- """Report the running gateway cache state, not an independent reopen. A corrupt store is
- unavailable whatever the cache says: an open handle on a damaged file is not a working one."""
- if state_db_probe.get("detail") == "corrupt":
- return _check("unavailable", "corrupt")
- runtime_store = runtime_status.get("session_store")
- state = str(runtime_store.get("status") or "unknown") if isinstance(runtime_store, dict) else ""
- if state in {"ok", "unavailable", "retrying"}:
- return _check(state)
- # Older gateways publish no cache state: fall back to the state_db probe.
- return _check("ok" if state_db_probe.get("status") == "ok" else "unavailable")
- def collect_runtime_readiness(
- *, configured_model: str, runtime_status: dict[str, Any] | None, active_api_runs: int = 0,
- process_completion_queue_depth: int = 0, active_delegations: int = 0,
- ) -> dict[str, Any]:
- """Bounded readiness diagnostics, no runtime mutation. Even authenticated, probes
- expose status and counts only: never config values, credentials, paths, payloads."""
- home = get_hermes_home()
- runtime = runtime_status if isinstance(runtime_status, dict) else {}
- state_db_probe = _probe_state_db(home)
- checks = {
- "state_db": state_db_probe,
- "session_store": _probe_session_store(runtime, state_db_probe),
- "config": _probe_config(home),
- "model": _check("ok" if str(configured_model or "").strip() else "degraded"),
- "disk": _probe_disk(home),
- "gateway": _probe_gateway(runtime),
- "background_queues": _check(
- "ok", active_api_runs=max(0, int(active_api_runs)),
- process_completions=max(0, int(process_completion_queue_depth)),
- active_delegations=max(0, int(active_delegations)),
- ),
- }
- return {"status": "ok" if all(c.get("status") == "ok" for c in checks.values()) else "degraded", "checks": checks}
- __all__ = ["collect_runtime_readiness"]
|