doctor_live.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193
  1. """``hermes doctor --live`` — opt-in bounded real-call tool-backend probes.
  2. Opt-in only: these probes make real (cheap, metadata/read-only) network calls and may spend a
  3. trivial amount of quota. They run ONLY when the user passes ``hermes doctor --live``.
  4. """
  5. from __future__ import annotations
  6. import os
  7. from dataclasses import dataclass
  8. from typing import Callable, List, Optional
  9. from hermes_cli.browser_runtime import chromium_executable
  10. from hermes_cli.doctor import _section, check_info
  11. from hermes_cli.doctor_report import check_fail, check_ok, check_warn
  12. from utils import normalize_proxy_env_vars
  13. DEFAULT_PROBE_TIMEOUT = 10.0
  14. # Metadata-only endpoints (none spend generation credits): name -> (url, env var, auth scheme).
  15. _KEYED_PROBES = {
  16. "Firecrawl": ("https://api.firecrawl.dev/v2/team/credit-usage", "FIRECRAWL_API_KEY", "Bearer"),
  17. "FAL": ("https://fal.ai/api/models?page=1", "FAL_KEY", "Key"),
  18. }
  19. # TTS/STT providers that never touch the network (nothing to probe).
  20. _LOCAL_AUDIO_PROVIDERS = {"", "local", "edge", "neutts", "kittentts", "piper"}
  21. _AUDIO_PROBES = {
  22. "openai": ("https://api.openai.com/v1/models", "OPENAI_API_KEY", "Bearer"),
  23. "groq": ("https://api.groq.com/openai/v1/models", "GROQ_API_KEY", "Bearer"),
  24. "elevenlabs": ("https://api.elevenlabs.io/v1/voices", "ELEVENLABS_API_KEY", "xi"),
  25. }
  26. @dataclass
  27. class ProbeResult:
  28. """Outcome of one backend probe."""
  29. name: str
  30. status: str # "pass" | "warn" | "fail" | "skip"
  31. detail: str = ""
  32. # ── Small seams (monkeypatchable in tests, and single points of control) ──
  33. def _http_get(url: str, headers: Optional[dict] = None, timeout: Optional[float] = None):
  34. """Single HTTP GET seam for all metadata probes."""
  35. import httpx
  36. # Sanitize at the seam (idempotent, no-op on a clean env): a bracketed-IPv6 NO_PROXY entry
  37. # ([::1], Clash Verge/mihomo) makes this bare trust_env client raise InvalidURL at
  38. # construction (#118159), which _run_one's catch-all would dress up as backend downtime.
  39. normalize_proxy_env_vars()
  40. return httpx.get(url, headers=headers or {}, timeout=timeout)
  41. def _browser_available() -> bool:
  42. """Is there a Chromium for browser_exec's local launcher?"""
  43. return chromium_executable() is not None
  44. def _launch_browser_probe(timeout: float) -> tuple:
  45. """Launch Chromium through browser_exec's own launcher, wait for its CDP endpoint, stop it.
  46. Returns (ok, detail); the probe owns the whole lifecycle and always cleans up."""
  47. from tools.browser_launcher import launch_chromium, stop_browser
  48. key = "doctor-live-probe"
  49. try:
  50. browser, error = launch_chromium(key, ephemeral=True, timeout=timeout)
  51. finally:
  52. stop_browser(key)
  53. if browser is None:
  54. return (False, error or "launch failed")
  55. return (True, "launched + CDP ready + stopped")
  56. def _probe_mcp_server(name: str, config: dict, timeout: float):
  57. """initialize + tools/list against one configured MCP server."""
  58. from hermes_cli.mcp_config import _probe_single_server
  59. return _probe_single_server(name, config, connect_timeout=timeout)
  60. # ── Per-backend probes. Each returns a ProbeResult; _run_one's catch-all handles crashes. ──
  61. def _classify_http(name: str, resp, key_hint: str) -> ProbeResult:
  62. code = getattr(resp, "status_code", None)
  63. if code is not None and 200 <= code < 300:
  64. return ProbeResult(name, "pass", f"(HTTP {code})")
  65. return ProbeResult(name, "fail", f"(HTTP {code} — check {key_hint})" if code in (401, 403) else f"(HTTP {code})")
  66. def _keyed_probe(name: str, url: str, env_var: str, scheme: str, timeout: float) -> ProbeResult:
  67. """Metadata GET authenticated by one env var (never a generation call)."""
  68. key = os.getenv(env_var, "").strip()
  69. if not key:
  70. return ProbeResult(name, "skip", "(not configured)")
  71. resp = _http_get(url, headers={"Authorization": f"{scheme} {key}"}, timeout=timeout)
  72. return _classify_http(name, resp, env_var)
  73. def _probe_browser(timeout: float) -> ProbeResult:
  74. if not _browser_available():
  75. return ProbeResult("Browser", "skip", "(not configured)")
  76. ok, detail = _launch_browser_probe(timeout)
  77. return ProbeResult("Browser", "pass" if ok else "fail", f"({detail})")
  78. def _probe_audio(kind: str, config: dict, timeout: float) -> ProbeResult:
  79. """Shared TTS/STT metadata probe (voices/models list GET only)."""
  80. name = kind.upper()
  81. provider = (((config.get(kind) or {}).get("provider")) or "").strip().lower()
  82. if provider in _LOCAL_AUDIO_PROVIDERS:
  83. return ProbeResult(name, "skip", f"(provider '{provider or 'local'}' — no remote backend to probe)")
  84. if provider not in _AUDIO_PROBES:
  85. return ProbeResult(name, "skip", f"(provider '{provider}' — no live probe implemented)")
  86. url, env_var, scheme = _AUDIO_PROBES[provider]
  87. key = os.getenv(env_var, "").strip()
  88. if not key:
  89. return ProbeResult(name, "warn", f"(provider '{provider}' configured but {env_var} is not set)")
  90. headers = {"xi-api-key": key} if scheme == "xi" else {"Authorization": f"Bearer {key}"}
  91. result = _classify_http(name, _http_get(url, headers=headers, timeout=timeout), env_var)
  92. result.detail = f"({provider}) {result.detail}"
  93. return result
  94. # ── Orchestration ──
  95. _REPORTERS = {"pass": check_ok, "warn": check_warn, "fail": check_fail}
  96. def _report(result: ProbeResult, issues: list[str]) -> None:
  97. reporter = _REPORTERS.get(result.status)
  98. if reporter is None: # skip
  99. check_info(f"{result.name} {result.detail} — skipped")
  100. return
  101. reporter(result.name, result.detail)
  102. if result.status == "fail":
  103. issues.append(f"Live probe failed: {result.name} {result.detail}")
  104. def _run_one(name: str, fn: Callable[[], ProbeResult], issues: list[str]) -> ProbeResult:
  105. """Run one probe with a catch-all so a crash never kills doctor."""
  106. try:
  107. result = fn()
  108. except TimeoutError as exc:
  109. result = ProbeResult(name, "fail", f"(timed out: {exc})")
  110. except Exception as exc:
  111. msg = str(exc) or exc.__class__.__name__
  112. result = ProbeResult(name, "fail", f"(timed out: {msg})" if "time" in msg.lower() else f"({msg})")
  113. _report(result, issues)
  114. return result
  115. def run_live_checks(issues: list[str]) -> list[ProbeResult]:
  116. """Run one bounded, read-only probe per configured tool backend — sequential by design (predictable output
  117. ordering). Appends a remediation line to ``issues`` per failed probe; skipped backends never append."""
  118. from hermes_cli.config import load_config_readonly
  119. config = load_config_readonly()
  120. try:
  121. timeout = float((config.get("doctor") or {}).get("live_probe_timeout", DEFAULT_PROBE_TIMEOUT))
  122. except (TypeError, ValueError):
  123. timeout = DEFAULT_PROBE_TIMEOUT
  124. timeout = max(1.0, timeout)
  125. _section("Live Backend Probes (opt-in, real calls)")
  126. results: list[ProbeResult] = [
  127. _run_one(name, lambda n=name, spec=spec: _keyed_probe(n, *spec, timeout), issues)
  128. for name, spec in _KEYED_PROBES.items()
  129. ]
  130. results.append(_run_one("Browser", lambda: _probe_browser(timeout), issues))
  131. servers = config.get("mcp_servers") or {}
  132. if isinstance(servers, dict) and servers:
  133. for name in sorted(servers):
  134. def _probe(n=name, e=servers[name]) -> ProbeResult:
  135. if not isinstance(e, dict):
  136. return ProbeResult(f"MCP: {n}", "skip", "(malformed config entry)")
  137. return ProbeResult(f"MCP: {n}", "pass", f"({len(_probe_mcp_server(n, e, timeout))} tool(s))")
  138. results.append(_run_one(f"MCP: {name}", _probe, issues))
  139. else:
  140. results.append(ProbeResult("MCP", "skip", "(no servers configured)"))
  141. _report(results[-1], issues)
  142. for kind in ("tts", "stt"):
  143. results.append(_run_one(kind.upper(), lambda k=kind: _probe_audio(k, config, timeout), issues))
  144. return results
  145. def maybe_run_live_checks(args, issues: list[str]):
  146. """Called from ``run_doctor`` after the static checks; no-op (None) unless ``--live`` was passed.
  147. A crash anywhere in the live subsystem must never break doctor."""
  148. if not getattr(args, "live", False):
  149. return None
  150. try:
  151. return run_live_checks(issues)
  152. except Exception as exc: # catch-all: doctor must survive
  153. check_warn("Live backend probes crashed", f"({exc})")
  154. return None