| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193 |
- """``hermes doctor --live`` — opt-in bounded real-call tool-backend probes.
- Opt-in only: these probes make real (cheap, metadata/read-only) network calls and may spend a
- trivial amount of quota. They run ONLY when the user passes ``hermes doctor --live``.
- """
- from __future__ import annotations
- import os
- from dataclasses import dataclass
- from typing import Callable, List, Optional
- from hermes_cli.browser_runtime import chromium_executable
- from hermes_cli.doctor import _section, check_info
- from hermes_cli.doctor_report import check_fail, check_ok, check_warn
- from utils import normalize_proxy_env_vars
- DEFAULT_PROBE_TIMEOUT = 10.0
- # Metadata-only endpoints (none spend generation credits): name -> (url, env var, auth scheme).
- _KEYED_PROBES = {
- "Firecrawl": ("https://api.firecrawl.dev/v2/team/credit-usage", "FIRECRAWL_API_KEY", "Bearer"),
- "FAL": ("https://fal.ai/api/models?page=1", "FAL_KEY", "Key"),
- }
- # TTS/STT providers that never touch the network (nothing to probe).
- _LOCAL_AUDIO_PROVIDERS = {"", "local", "edge", "neutts", "kittentts", "piper"}
- _AUDIO_PROBES = {
- "openai": ("https://api.openai.com/v1/models", "OPENAI_API_KEY", "Bearer"),
- "groq": ("https://api.groq.com/openai/v1/models", "GROQ_API_KEY", "Bearer"),
- "elevenlabs": ("https://api.elevenlabs.io/v1/voices", "ELEVENLABS_API_KEY", "xi"),
- }
- @dataclass
- class ProbeResult:
- """Outcome of one backend probe."""
- name: str
- status: str # "pass" | "warn" | "fail" | "skip"
- detail: str = ""
- # ── Small seams (monkeypatchable in tests, and single points of control) ──
- def _http_get(url: str, headers: Optional[dict] = None, timeout: Optional[float] = None):
- """Single HTTP GET seam for all metadata probes."""
- import httpx
- # Sanitize at the seam (idempotent, no-op on a clean env): a bracketed-IPv6 NO_PROXY entry
- # ([::1], Clash Verge/mihomo) makes this bare trust_env client raise InvalidURL at
- # construction (#118159), which _run_one's catch-all would dress up as backend downtime.
- normalize_proxy_env_vars()
- return httpx.get(url, headers=headers or {}, timeout=timeout)
- def _browser_available() -> bool:
- """Is there a Chromium for browser_exec's local launcher?"""
- return chromium_executable() is not None
- def _launch_browser_probe(timeout: float) -> tuple:
- """Launch Chromium through browser_exec's own launcher, wait for its CDP endpoint, stop it.
- Returns (ok, detail); the probe owns the whole lifecycle and always cleans up."""
- from tools.browser_launcher import launch_chromium, stop_browser
- key = "doctor-live-probe"
- try:
- browser, error = launch_chromium(key, ephemeral=True, timeout=timeout)
- finally:
- stop_browser(key)
- if browser is None:
- return (False, error or "launch failed")
- return (True, "launched + CDP ready + stopped")
- def _probe_mcp_server(name: str, config: dict, timeout: float):
- """initialize + tools/list against one configured MCP server."""
- from hermes_cli.mcp_config import _probe_single_server
- return _probe_single_server(name, config, connect_timeout=timeout)
- # ── Per-backend probes. Each returns a ProbeResult; _run_one's catch-all handles crashes. ──
- def _classify_http(name: str, resp, key_hint: str) -> ProbeResult:
- code = getattr(resp, "status_code", None)
- if code is not None and 200 <= code < 300:
- return ProbeResult(name, "pass", f"(HTTP {code})")
- return ProbeResult(name, "fail", f"(HTTP {code} — check {key_hint})" if code in (401, 403) else f"(HTTP {code})")
- def _keyed_probe(name: str, url: str, env_var: str, scheme: str, timeout: float) -> ProbeResult:
- """Metadata GET authenticated by one env var (never a generation call)."""
- key = os.getenv(env_var, "").strip()
- if not key:
- return ProbeResult(name, "skip", "(not configured)")
- resp = _http_get(url, headers={"Authorization": f"{scheme} {key}"}, timeout=timeout)
- return _classify_http(name, resp, env_var)
- def _probe_browser(timeout: float) -> ProbeResult:
- if not _browser_available():
- return ProbeResult("Browser", "skip", "(not configured)")
- ok, detail = _launch_browser_probe(timeout)
- return ProbeResult("Browser", "pass" if ok else "fail", f"({detail})")
- def _probe_audio(kind: str, config: dict, timeout: float) -> ProbeResult:
- """Shared TTS/STT metadata probe (voices/models list GET only)."""
- name = kind.upper()
- provider = (((config.get(kind) or {}).get("provider")) or "").strip().lower()
- if provider in _LOCAL_AUDIO_PROVIDERS:
- return ProbeResult(name, "skip", f"(provider '{provider or 'local'}' — no remote backend to probe)")
- if provider not in _AUDIO_PROBES:
- return ProbeResult(name, "skip", f"(provider '{provider}' — no live probe implemented)")
- url, env_var, scheme = _AUDIO_PROBES[provider]
- key = os.getenv(env_var, "").strip()
- if not key:
- return ProbeResult(name, "warn", f"(provider '{provider}' configured but {env_var} is not set)")
- headers = {"xi-api-key": key} if scheme == "xi" else {"Authorization": f"Bearer {key}"}
- result = _classify_http(name, _http_get(url, headers=headers, timeout=timeout), env_var)
- result.detail = f"({provider}) {result.detail}"
- return result
- # ── Orchestration ──
- _REPORTERS = {"pass": check_ok, "warn": check_warn, "fail": check_fail}
- def _report(result: ProbeResult, issues: list[str]) -> None:
- reporter = _REPORTERS.get(result.status)
- if reporter is None: # skip
- check_info(f"{result.name} {result.detail} — skipped")
- return
- reporter(result.name, result.detail)
- if result.status == "fail":
- issues.append(f"Live probe failed: {result.name} {result.detail}")
- def _run_one(name: str, fn: Callable[[], ProbeResult], issues: list[str]) -> ProbeResult:
- """Run one probe with a catch-all so a crash never kills doctor."""
- try:
- result = fn()
- except TimeoutError as exc:
- result = ProbeResult(name, "fail", f"(timed out: {exc})")
- except Exception as exc:
- msg = str(exc) or exc.__class__.__name__
- result = ProbeResult(name, "fail", f"(timed out: {msg})" if "time" in msg.lower() else f"({msg})")
- _report(result, issues)
- return result
- def run_live_checks(issues: list[str]) -> list[ProbeResult]:
- """Run one bounded, read-only probe per configured tool backend — sequential by design (predictable output
- ordering). Appends a remediation line to ``issues`` per failed probe; skipped backends never append."""
- from hermes_cli.config import load_config_readonly
- config = load_config_readonly()
- try:
- timeout = float((config.get("doctor") or {}).get("live_probe_timeout", DEFAULT_PROBE_TIMEOUT))
- except (TypeError, ValueError):
- timeout = DEFAULT_PROBE_TIMEOUT
- timeout = max(1.0, timeout)
- _section("Live Backend Probes (opt-in, real calls)")
- results: list[ProbeResult] = [
- _run_one(name, lambda n=name, spec=spec: _keyed_probe(n, *spec, timeout), issues)
- for name, spec in _KEYED_PROBES.items()
- ]
- results.append(_run_one("Browser", lambda: _probe_browser(timeout), issues))
- servers = config.get("mcp_servers") or {}
- if isinstance(servers, dict) and servers:
- for name in sorted(servers):
- def _probe(n=name, e=servers[name]) -> ProbeResult:
- if not isinstance(e, dict):
- return ProbeResult(f"MCP: {n}", "skip", "(malformed config entry)")
- return ProbeResult(f"MCP: {n}", "pass", f"({len(_probe_mcp_server(n, e, timeout))} tool(s))")
- results.append(_run_one(f"MCP: {name}", _probe, issues))
- else:
- results.append(ProbeResult("MCP", "skip", "(no servers configured)"))
- _report(results[-1], issues)
- for kind in ("tts", "stt"):
- results.append(_run_one(kind.upper(), lambda k=kind: _probe_audio(k, config, timeout), issues))
- return results
- def maybe_run_live_checks(args, issues: list[str]):
- """Called from ``run_doctor`` after the static checks; no-op (None) unless ``--live`` was passed.
- A crash anywhere in the live subsystem must never break doctor."""
- if not getattr(args, "live", False):
- return None
- try:
- return run_live_checks(issues)
- except Exception as exc: # catch-all: doctor must survive
- check_warn("Live backend probes crashed", f"({exc})")
- return None
|