readiness.py 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125
  1. """Bounded, non-destructive readiness probes for authenticated health surfaces."""
  2. from __future__ import annotations
  3. import shutil
  4. import sqlite3
  5. from contextlib import closing
  6. from pathlib import Path
  7. from typing import Any
  8. from hermes_constants import get_hermes_home
  9. _DISK_DEGRADED_PERCENT = 90.0
  10. _CONNECTED_STATES = {"connected", "running", "ok"}
  11. def _check(status: str, detail: str | None = None, **extra: Any) -> dict[str, Any]:
  12. return {"status": status, **({"detail": detail} if detail else {}), **extra}
  13. def _probe_state_db(home: Path) -> dict[str, Any]:
  14. """Read-only schema probe plus the process-wide corruption latch (``hermes_state_health``).
  15. The schema read only catches an unreadable header or schema; damage deeper in the file
  16. surfaces when a reader or writer touches it, and those publish into the latch. Reporting
  17. the latch here is what makes readiness and ``/api/status`` agree with the session list
  18. (#72046). ``detail="corrupt"`` is the one reason string consumers key off."""
  19. from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state
  20. path = home / "state.db"
  21. if not path.exists():
  22. return _check("ok", "not initialized")
  23. if storage_state(path) == STORAGE_CORRUPT:
  24. return _check("degraded", STORAGE_CORRUPT)
  25. try:
  26. # Read-only schema query: catches unreadable/corrupt DBs without competing with
  27. # writers. ``closing`` is required — sqlite3's context manager only commits/rolls
  28. # back, never closes, so a bare ``with connect()`` leaks a connection per poll.
  29. with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True, timeout=1.0)) as conn:
  30. # A readiness probe must never compete with normal state writers. See #69567, #69678.
  31. conn.execute("PRAGMA query_only = ON")
  32. conn.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
  33. return _check("ok")
  34. except Exception as exc:
  35. if note_storage_error(path, exc):
  36. return _check("degraded", STORAGE_CORRUPT)
  37. return _check("degraded", type(exc).__name__)
  38. def _probe_config(home: Path) -> dict[str, Any]:
  39. from hermes_cli.config_backend import ConfigBackendUnavailable, config_exists, read_config_doc_readonly
  40. path = home / "config.yaml"
  41. try:
  42. if not config_exists(path):
  43. return _check("ok", "using defaults")
  44. raw = read_config_doc_readonly(path)
  45. except ConfigBackendUnavailable:
  46. # A SystemExit on purpose (readers must not serve defaults); this probe exists to report it.
  47. return _check("degraded", "config backend unavailable")
  48. except Exception as exc:
  49. return _check("degraded", f"invalid config ({type(exc).__name__})")
  50. return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")
  51. def _probe_disk(home: Path) -> dict[str, Any]:
  52. try:
  53. usage = shutil.disk_usage(home)
  54. except Exception as exc:
  55. return _check("degraded", type(exc).__name__)
  56. used_pct = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
  57. return _check("degraded" if used_pct >= _DISK_DEGRADED_PERCENT else "ok", used_percent=used_pct, free_bytes=usage.free)
  58. def _probe_gateway(runtime_status: dict[str, Any]) -> dict[str, Any]:
  59. state = str(runtime_status.get("gateway_state") or "unknown")
  60. platforms = runtime_status.get("platforms")
  61. platforms = platforms if isinstance(platforms, dict) else {}
  62. connected = sum(
  63. isinstance(v, dict) and str(v.get("state") or v.get("status") or "").lower() in _CONNECTED_STATES
  64. for v in platforms.values()
  65. )
  66. return _check("ok" if state in {"running", "draining"} else "degraded", state=state,
  67. connected_platforms=connected, platforms=len(platforms))
  68. def _probe_session_store(runtime_status: dict[str, Any], state_db_probe: dict[str, Any]) -> dict[str, Any]:
  69. """Report the running gateway cache state, not an independent reopen. A corrupt store is
  70. unavailable whatever the cache says: an open handle on a damaged file is not a working one."""
  71. if state_db_probe.get("detail") == "corrupt":
  72. return _check("unavailable", "corrupt")
  73. runtime_store = runtime_status.get("session_store")
  74. state = str(runtime_store.get("status") or "unknown") if isinstance(runtime_store, dict) else ""
  75. if state in {"ok", "unavailable", "retrying"}:
  76. return _check(state)
  77. # Older gateways publish no cache state: fall back to the state_db probe.
  78. return _check("ok" if state_db_probe.get("status") == "ok" else "unavailable")
  79. def collect_runtime_readiness(
  80. *, configured_model: str, runtime_status: dict[str, Any] | None, active_api_runs: int = 0,
  81. process_completion_queue_depth: int = 0, active_delegations: int = 0,
  82. ) -> dict[str, Any]:
  83. """Bounded readiness diagnostics, no runtime mutation. Even authenticated, probes
  84. expose status and counts only: never config values, credentials, paths, payloads."""
  85. home = get_hermes_home()
  86. runtime = runtime_status if isinstance(runtime_status, dict) else {}
  87. state_db_probe = _probe_state_db(home)
  88. checks = {
  89. "state_db": state_db_probe,
  90. "session_store": _probe_session_store(runtime, state_db_probe),
  91. "config": _probe_config(home),
  92. "model": _check("ok" if str(configured_model or "").strip() else "degraded"),
  93. "disk": _probe_disk(home),
  94. "gateway": _probe_gateway(runtime),
  95. "background_queues": _check(
  96. "ok", active_api_runs=max(0, int(active_api_runs)),
  97. process_completions=max(0, int(process_completion_queue_depth)),
  98. active_delegations=max(0, int(active_delegations)),
  99. ),
  100. }
  101. return {"status": "ok" if all(c.get("status") == "ok" for c in checks.values()) else "degraded", "checks": checks}
  102. __all__ = ["collect_runtime_readiness"]