gateway_health.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. """Gateway health and diagnostics signal producer.
  2. Keeps the plane narrow: service health plus redacted operational diagnostics,
  3. derived from the existing gateway runtime-status contract. No prompts,
  4. messages, tool args, session history, audit records, or product analytics.
  5. """
  6. from __future__ import annotations
  7. import hashlib
  8. import logging
  9. import re
  10. from dataclasses import dataclass
  11. from typing import Any, Callable, Dict, List, Optional
  12. from agent.monitoring import emitter
  13. from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
  14. from agent.monitoring.redaction import redact_bounded
  15. logger = logging.getLogger(__name__)
  16. @dataclass(frozen=True, slots=True)
  17. class GatewayMetric:
  18. name: str
  19. value: int | float
  20. attributes: dict[str, str]
  21. @dataclass(frozen=True, slots=True)
  22. class GatewayHealthSnapshot:
  23. metrics: list[GatewayMetric]
  24. events: list[GatewayHealthEvent | GatewayDiagnosticEvent]
  25. _RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
  26. _FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
  27. _KNOWN_GATEWAY_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"}
  28. _KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"}
  29. _SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
  30. _SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
  31. def source_logger_for_export(name: Any) -> Optional[str]:
  32. """Return a bounded source-controlled gateway logger name for OTLP scope."""
  33. value = str(name or "")
  34. return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None
  35. def _contains_any(*needles: str) -> Callable[[str], bool]:
  36. return lambda text: any(needle in text for needle in needles)
  37. # Ordered (predicate, class) rules; first match wins, so auth outranks rate-limit etc.
  38. _GATEWAY_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = (
  39. (_contains_any("auth", "token", "unauthorized", "forbidden", "401", "403"), "auth_failed"),
  40. (lambda s: "rate" in s and "limit" in s, "rate_limited"),
  41. (_contains_any("timeout", "timed out"), "timeout"),
  42. (_contains_any("network", "connection", "dns", "socket", "connect call failed", "failed to connect", "cannot connect",
  43. "unreachable", "name resolution"), "network_error"),
  44. (_contains_any("config", "missing", "invalid"), "invalid_config"),
  45. (_contains_any("startup"), "startup_failed"),
  46. (_contains_any("fatal"), "platform_fatal"),
  47. )
  48. def classify_gateway_error(raw: Any) -> str:
  49. s = str(raw or "").lower()
  50. return next((label for match, label in _GATEWAY_ERROR_RULES if match(s)), "unknown")
  51. def classify_exit_reason(raw: Any, *, state: Any, restart_requested: bool) -> Optional[str]:
  52. """Reduce free-form shutdown text to a bounded operational class."""
  53. if restart_requested:
  54. return "restart_requested"
  55. state_name = str(state or "").lower()
  56. if raw is None and state_name != "startup_failed":
  57. return None
  58. classified = classify_gateway_error(raw)
  59. if state_name == "startup_failed":
  60. return classified if classified != "unknown" else "startup_failed"
  61. text = str(raw or "").lower()
  62. if "signal" in text or "sigterm" in text or "sigint" in text:
  63. return "signal"
  64. if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")):
  65. return "planned_stop"
  66. return classified
  67. def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
  68. state = str(raw or "unknown").lower()
  69. return state if state in allowed else "unknown"
  70. def _optional_state(raw: Any, *, allowed: set[str]) -> Optional[str]:
  71. """``_bounded_state`` that preserves "absent" (None) instead of coercing to unknown."""
  72. return None if raw is None else _bounded_state(raw, allowed=allowed)
  73. def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
  74. return redact_bounded(raw, limit=limit, empty="unknown", unavailable="unknown")
  75. def _safe_instance_id(raw: Any) -> str:
  76. """Return a stable opaque instance key without exporting the source ID."""
  77. value = str(raw or "unknown").encode("utf-8", errors="replace")
  78. return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
  79. def subsystem_for_logger(logger_name: str) -> str:
  80. parts = logger_name.split(".")
  81. if parts[:2] == ["gateway", "relay"]:
  82. return "platform.relay"
  83. if parts[:2] == ["gateway", "platforms"] and len(parts) >= 3 and parts[2]:
  84. return f"platform.{parts[2]}"
  85. return "platform" if logger_name.startswith("gateway.platforms") else "gateway"
  86. def platform_for_subsystem(subsystem: str) -> Optional[str]:
  87. return (subsystem.split(".", 1)[1] or None) if subsystem.startswith("platform.") else None
  88. def _coerce_pid(raw: Any) -> Optional[int]:
  89. try:
  90. pid = int(raw)
  91. except (TypeError, ValueError):
  92. return None
  93. return pid if pid > 0 else None
  94. def _gateway_status(name: str, fallback: Callable[[], Any], /, **kwargs: Any) -> Any:
  95. """Prefer ``gateway.status.<name>`` (the runtime-status contract); fall back to the local approximation."""
  96. try:
  97. from gateway import status
  98. return getattr(status, name)(**kwargs)
  99. except Exception:
  100. return fallback()
  101. def _parse_active_agents(raw: Any) -> int:
  102. def fallback() -> int:
  103. try:
  104. return max(0, int(raw))
  105. except (TypeError, ValueError):
  106. return 0
  107. return _gateway_status("parse_active_agents", fallback, raw=raw)
  108. def _dict_or_empty(raw: Any) -> dict[str, Any]:
  109. return raw if isinstance(raw, dict) else {}
  110. def _platforms_of(runtime: Optional[dict[str, Any]]) -> dict[str, Any]:
  111. return _dict_or_empty((runtime or {}).get("platforms"))
  112. def build_gateway_health_snapshot(
  113. runtime: Optional[dict[str, Any]], *, gateway_running: bool, profile: str, install_id: str, version: str,
  114. supervision_mode: str = "unknown",
  115. ) -> GatewayHealthSnapshot:
  116. """Convert gateway_state.json-compatible runtime state into P0 signals."""
  117. runtime = runtime or {}
  118. gateway_state = _bounded_state(runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
  119. active_agents = _parse_active_agents(runtime.get("active_agents", 0))
  120. running = gateway_running and gateway_state == "running"
  121. busy = _gateway_status(
  122. "derive_gateway_busy", lambda: bool(running and _parse_active_agents(active_agents) > 0),
  123. gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents,
  124. )
  125. drainable = _gateway_status(
  126. "derive_gateway_drainable", lambda: bool(running), gateway_running=gateway_running, gateway_state=gateway_state
  127. )
  128. platforms = _platforms_of(runtime)
  129. mode = str(supervision_mode or "unknown").lower()
  130. base = {
  131. "service.instance.id": _safe_instance_id(install_id),
  132. "service.version": _safe_metric_value(version, limit=64),
  133. "hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown",
  134. }
  135. def metric(name: str, value: float, **extra: str) -> GatewayMetric:
  136. attrs = dict(base)
  137. for key, val in extra.items():
  138. if val is not None:
  139. attrs[key] = _safe_metric_value(val)
  140. return GatewayMetric(name=name, value=value, attributes=attrs)
  141. metrics: list[GatewayMetric] = [
  142. metric("hermes.gateway.up", int(bool(gateway_running))),
  143. metric("hermes.gateway.active_agents", active_agents),
  144. metric("hermes.gateway.busy", int(bool(busy))),
  145. metric("hermes.gateway.drainable", int(bool(drainable))),
  146. metric("hermes.gateway.restart_requested", int(bool(runtime.get("restart_requested")))),
  147. metric("hermes.gateway.state", 1, **{"hermes.gateway.state": gateway_state}),
  148. ]
  149. fatal_count = 0
  150. events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
  151. # classify_* is idempotent on its own labels, so error_class == error_code downstream.
  152. for platform, pdata in platforms.items():
  153. pdata = _dict_or_empty(pdata)
  154. state = _bounded_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
  155. error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
  156. is_degraded = state in _FATAL_PLATFORM_STATES
  157. fatal_count += is_degraded
  158. pattrs = {"hermes.platform": str(platform), "hermes.platform.state": state}
  159. metrics.append(metric("hermes.platform.up", int(state in _RUNNING_PLATFORM_STATES), **pattrs))
  160. metrics.append(metric("hermes.platform.degraded", int(is_degraded), **pattrs, **{"hermes.error_code": error_code}))
  161. if is_degraded:
  162. events.append(GatewayDiagnosticEvent(
  163. name="platform.fatal", subsystem=f"platform.{platform}", platform=str(platform),
  164. error_code=error_code, error_class=error_code, profile=profile, version=version,
  165. severity="error" if state == "fatal" else "warning",
  166. ))
  167. events.insert(0, GatewayHealthEvent(
  168. name="gateway.health_snapshot", gateway_state=gateway_state, active_agents=active_agents, gateway_busy=busy,
  169. gateway_drainable=drainable, platform_count=len(platforms), fatal_platform_count=fatal_count, profile=profile,
  170. install_id=install_id, version=version, supervision_mode=supervision_mode, pid=_coerce_pid(runtime.get("pid")),
  171. ))
  172. return GatewayHealthSnapshot(metrics=metrics, events=events)
  173. def _safe_profile() -> str:
  174. try:
  175. from hermes_cli.profiles import get_active_profile_name
  176. return str(get_active_profile_name() or "default")
  177. except Exception:
  178. return "default"
  179. def _safe_version() -> str:
  180. try:
  181. from hermes_cli.version_info import get_version_info
  182. return get_version_info().base_version
  183. except Exception:
  184. return "unknown"
  185. def _lifecycle_events(
  186. previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
  187. ) -> list[GatewayHealthEvent | GatewayDiagnosticEvent]:
  188. """Gateway-level transition events: lifecycle, startup_failed diagnostic, exit."""
  189. old_state = _optional_state((previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
  190. new_state = _optional_state(current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
  191. if old_state == new_state or not new_state:
  192. return []
  193. restart_requested = bool(current.get("restart_requested"))
  194. def health(name: str) -> GatewayHealthEvent:
  195. return GatewayHealthEvent(
  196. name=name, gateway_state=new_state, old_state=old_state, new_state=new_state,
  197. exit_reason=classify_exit_reason(current.get("exit_reason"), state=new_state, restart_requested=restart_requested),
  198. restart_requested=restart_requested, active_agents=_parse_active_agents(current.get("active_agents", 0)),
  199. profile=profile, version=version, pid=_coerce_pid(current.get("pid")),
  200. )
  201. out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [health("gateway.lifecycle")]
  202. if new_state == "startup_failed":
  203. error_class = classify_gateway_error(current.get("exit_reason") or "startup_failed")
  204. out.append(GatewayDiagnosticEvent(
  205. name="gateway.startup_failed", subsystem="gateway", error_class=error_class, error_code=error_class,
  206. profile=profile, version=version, severity="error",
  207. ))
  208. if new_state == "stopped":
  209. out.append(health("gateway.exit"))
  210. return out
  211. def _platform_events(
  212. previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
  213. ) -> list[GatewayDiagnosticEvent]:
  214. """Per-platform state_change diagnostics, plus platform.fatal when the new state is fatal."""
  215. old_platforms = _platforms_of(previous)
  216. out: list[GatewayDiagnosticEvent] = []
  217. for platform, pdata in _platforms_of(current).items():
  218. pdata = _dict_or_empty(pdata)
  219. prev = _dict_or_empty(old_platforms.get(platform, {}))
  220. old_state = _optional_state(prev.get("state"), allowed=_KNOWN_PLATFORM_STATES)
  221. new_state = _optional_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
  222. if old_state == new_state or not new_state:
  223. continue
  224. error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
  225. common: dict[str, Any] = dict(
  226. subsystem=f"platform.{platform}", platform=str(platform), error_code=error_code, error_class=error_code,
  227. profile=profile, version=version, severity="error" if new_state in {"fatal", "failed", "error"} else "warning",
  228. )
  229. out.append(GatewayDiagnosticEvent(name="platform.state_change", old_state=old_state, new_state=new_state, **common))
  230. if new_state in _FATAL_PLATFORM_STATES:
  231. out.append(GatewayDiagnosticEvent(name="platform.fatal", **common))
  232. return out
  233. def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
  234. """Emit immediate content-free gateway events for runtime status changes. Called by
  235. gateway.status.write_runtime_status after persisting; fully fail-open."""
  236. try:
  237. ctx = dict(profile=_safe_profile(), version=_safe_version())
  238. for ev in _lifecycle_events(previous, current, **ctx) + _platform_events(previous, current, **ctx):
  239. emitter.emit(ev)
  240. except Exception:
  241. logger.debug("gateway runtime status transition emit failed", exc_info=True)
  242. class GatewayDiagnosticLogHandler(logging.Handler):
  243. """Allowlisted warning/error bridge for gateway-owned diagnostics."""
  244. def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
  245. super().__init__(level=logging.WARNING)
  246. self.profile = profile
  247. self.version = version
  248. def emit(self, record: logging.LogRecord) -> None:
  249. try:
  250. if record.levelno < logging.WARNING or not (record.name == "gateway" or record.name.startswith("gateway.")):
  251. return
  252. subsystem = subsystem_for_logger(record.name)
  253. error_class = classify_gateway_error(record.getMessage())
  254. emitter.get_emitter().emit(GatewayDiagnosticEvent(
  255. name=f"gateway.log.{record.levelname.lower()}", subsystem=subsystem,
  256. source_logger=source_logger_for_export(record.name), platform=platform_for_subsystem(subsystem),
  257. error_class=error_class, error_code=error_class, profile=self.profile, version=self.version,
  258. severity=record.levelname.lower(),
  259. ))
  260. except Exception:
  261. logger.debug("gateway diagnostic emit failed", exc_info=True)
  262. __all__ = [
  263. "GatewayDiagnosticLogHandler",
  264. "GatewayHealthSnapshot",
  265. "GatewayMetric",
  266. "build_gateway_health_snapshot",
  267. "classify_gateway_error",
  268. "source_logger_for_export",
  269. ]