doctor_connectivity.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. """API connectivity probes for ``hermes doctor`` (split out of ``doctor.py``).
  2. Every probe is a pure function: one HTTP/SDK call returning a ``ProbeResult`` with the row(s) to
  3. print and issue strings to append. No printing inside workers — the caller prints in submission order.
  4. """
  5. from __future__ import annotations
  6. from pm import install_hint
  7. import concurrent.futures
  8. import errno
  9. import functools
  10. import os
  11. import socket
  12. from typing import NamedTuple
  13. from urllib.parse import urlsplit
  14. from hermes_cli.colors import Colors, color
  15. from hermes_cli.version_info import hermes_user_agent
  16. from hermes_constants import OPENROUTER_MODELS_URL
  17. from utils import base_url_host_matches, normalize_proxy_env_vars
  18. _APIKEY_PROVIDERS_CACHE: list | None = None
  19. class ProbeResult(NamedTuple):
  20. label: str
  21. lines: list # [(glyph, label, detail)]
  22. issues: list
  23. _GLYPH = {"ok": ("✓", Colors.GREEN), "warn": ("⚠", Colors.YELLOW), "fail": ("✗", Colors.RED)}
  24. def _row(name: str, status: str, detail: str = "", issues: list | None = None, label: str | None = None) -> ProbeResult:
  25. glyph, col = _GLYPH[status]
  26. return ProbeResult(name, [(color(glyph, col), name if label is None else label, color(detail, Colors.DIM) if detail else "")],
  27. list(issues or []))
  28. def _skip(name: str) -> ProbeResult:
  29. return ProbeResult(name, [], [])
  30. def _has_healthy_oauth_fallback_for_apikey_provider(provider_label: str) -> bool:
  31. """True when a failed direct API-key probe is non-blocking because the same provider family's OAuth
  32. runtime path is already healthy: the failed row is still shown, but not promoted into the summary."""
  33. getter = {"minimax": "get_minimax_oauth_auth_status", "xai": "get_xai_oauth_auth_status"}.get((provider_label or "").strip().lower())
  34. if not getter:
  35. return False
  36. try:
  37. from hermes_cli import auth
  38. return bool((getattr(auth, getter)() or {}).get("logged_in"))
  39. except Exception:
  40. return False
  41. def _build_apikey_providers_list() -> list:
  42. """Build the API-key provider health-check list once and cache it.
  43. Tuple format: (name, env_vars, default_url, base_env, supports_models_endpoint). Base list augmented
  44. with any ProviderProfile with auth_type="api_key" not already present — adding
  45. plugins/model-providers/<name>/ is sufficient to get into doctor.
  46. """
  47. _static = [
  48. ("Z.AI / GLM", ("GLM_API_KEY", "ZAI_API_KEY", "Z_AI_API_KEY"), "https://api.z.ai/api/paas/v4/models", "GLM_BASE_URL", True),
  49. ("Kimi / Moonshot", ("KIMI_API_KEY",), "https://api.moonshot.ai/v1/models", "KIMI_BASE_URL", True),
  50. ("StepFun Step Plan", ("STEPFUN_API_KEY",), "https://api.stepfun.ai/step_plan/v1/models", "STEPFUN_BASE_URL", True),
  51. ("Kimi / Moonshot (China)", ("KIMI_CN_API_KEY",), "https://api.moonshot.cn/v1/models", None, True),
  52. ("Arcee AI", ("ARCEEAI_API_KEY",), "https://api.arcee.ai/api/v1/models", "ARCEE_BASE_URL", True),
  53. ("GMI Cloud", ("GMI_API_KEY",), "https://api.gmi-serving.com/v1/models", "GMI_BASE_URL", True),
  54. ("DeepSeek", ("DEEPSEEK_API_KEY",), "https://api.deepseek.com/v1/models", "DEEPSEEK_BASE_URL", True),
  55. ("Hugging Face", ("HF_TOKEN",), "https://router.huggingface.co/v1/models", "HF_BASE_URL", True),
  56. ("NVIDIA NIM", ("NVIDIA_API_KEY",), "https://integrate.api.nvidia.com/v1/models", "NVIDIA_BASE_URL", True),
  57. ("Alibaba/DashScope", ("DASHSCOPE_API_KEY",), "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/models", "DASHSCOPE_BASE_URL", True),
  58. # MiniMax global: /v1 endpoint supports /models.
  59. ("MiniMax", ("MINIMAX_API_KEY",), "https://api.minimax.io/v1/models", "MINIMAX_BASE_URL", True),
  60. # MiniMax CN: /v1 endpoint does NOT support /models (returns 404).
  61. ("MiniMax (China)", ("MINIMAX_CN_API_KEY",), "https://api.minimaxi.com/v1/models", "MINIMAX_CN_BASE_URL", False),
  62. ("Vercel AI Gateway", ("AI_GATEWAY_API_KEY",), "https://ai-gateway.vercel.sh/v1/models", "AI_GATEWAY_BASE_URL", True),
  63. ("Kilo Code", ("KILOCODE_API_KEY",), "https://api.kilo.ai/api/gateway/models", "KILOCODE_BASE_URL", True),
  64. ("OpenCode Zen", ("OPENCODE_ZEN_API_KEY",), "https://opencode.ai/zen/v1/models", "OPENCODE_ZEN_BASE_URL", True),
  65. # OpenCode Go has no shared /models endpoint; skip the health check.
  66. ("OpenCode Go", ("OPENCODE_GO_API_KEY",), None, "OPENCODE_GO_BASE_URL", False),
  67. ]
  68. _known_names = {t[0] for t in _static}
  69. # Providers with a dedicated health check (custom headers/auth): skip their pluggable profiles so
  70. # the generic Bearer loop doesn't run a duplicate, broken check (Anthropic needs x-api-key).
  71. _dedicated_canonical = {"anthropic", "openrouter", "bedrock"}
  72. # Canonical profile names of the static rows, so profiles without a display_name don't duplicate.
  73. _known_canonical = {
  74. "zai", "kimi-coding", "stepfun", "kimi-coding-cn", "arcee", "gmi", "deepseek", "huggingface", "nvidia",
  75. "alibaba", "minimax", "minimax-cn", "ai-gateway", "kilocode", "opencode-zen", "opencode-go",
  76. } | _dedicated_canonical
  77. try:
  78. from providers import list_providers
  79. from providers.base import ProviderProfile as _PP
  80. try:
  81. from hermes_cli.providers import normalize_provider as _normalize_provider
  82. except Exception: # pragma: no cover - normalization is best-effort
  83. def _normalize_provider(_name: str) -> str:
  84. return (_name or "").strip().lower()
  85. for _pp in list_providers():
  86. if not isinstance(_pp, _PP) or _pp.auth_type != "api_key" or not _pp.env_vars:
  87. continue
  88. _label = _pp.display_name or _pp.name
  89. if _label in _known_names or _pp.name in _known_canonical:
  90. continue
  91. if {_normalize_provider(a) for a in (_pp.name, *(_pp.aliases or ()))} & _dedicated_canonical:
  92. continue
  93. # Key vars vs base-URL vars: the first found value goes out as Authorization: Bearer, never a URL.
  94. _is_url = lambda v: v.endswith("_BASE_URL") or v.endswith("_URL")
  95. _key_vars = tuple(v for v in _pp.env_vars if not _is_url(v))
  96. if not _key_vars:
  97. continue
  98. _base_var = next((v for v in _pp.env_vars if _is_url(v)), None)
  99. _models_url = (_pp.models_url or (_pp.base_url.rstrip("/") + "/models")) if _pp.base_url else None
  100. _static.append((_label, _key_vars, _models_url, _base_var, getattr(_pp, "supports_health_check", True)))
  101. except Exception:
  102. pass
  103. return _static
  104. # HTTP status -> (detail, issue) for the OpenRouter probe; anything else is a generic HTTP failure.
  105. _OPENROUTER_STATUS = {
  106. 401: ("(invalid API key)", "Check OPENROUTER_API_KEY in .env"),
  107. 402: ("(out of credits — payment required)",
  108. "OpenRouter account has insufficient credits. "
  109. "Fix: run 'hermes config set model.provider <provider>' "
  110. "to switch providers, or fund your OpenRouter account "
  111. "at https://openrouter.ai/settings/credits"),
  112. 429: ("(rate limited)", "OpenRouter rate limit hit — consider switching to a different provider or waiting"),
  113. }
  114. def _probe_openrouter() -> ProbeResult:
  115. name = "OpenRouter API"
  116. key = os.getenv("OPENROUTER_API_KEY")
  117. if not key:
  118. return _row(name, "warn", "(not configured)")
  119. try:
  120. import httpx
  121. r = httpx.get(OPENROUTER_MODELS_URL, headers={"Authorization": f"Bearer {key}"}, timeout=10)
  122. except Exception as e:
  123. return _row(name, "fail", f"({e})", ["Check network connectivity"])
  124. if r.status_code == 200:
  125. return _row(name, "ok")
  126. detail, issue = _OPENROUTER_STATUS.get(r.status_code, (f"(HTTP {r.status_code})", None))
  127. return _row(name, "fail", detail, [issue] if issue else None)
  128. def _probe_anthropic() -> ProbeResult:
  129. name = "Anthropic API"
  130. from hermes_cli.auth import get_anthropic_key
  131. key = get_anthropic_key()
  132. if not key:
  133. return _skip(name)
  134. try:
  135. import httpx
  136. from agent.anthropic_adapter import _COMMON_BETAS, _OAUTH_ONLY_BETAS, _CONTEXT_1M_BETA
  137. from agent.anthropic_credentials import _is_oauth_token
  138. is_oauth = _is_oauth_token(key)
  139. headers = {"anthropic-version": "2023-06-01", **({"Authorization": f"Bearer {key}", "anthropic-beta": ",".join(_COMMON_BETAS + _OAUTH_ONLY_BETAS)}
  140. if is_oauth else {"x-api-key": key})}
  141. url = "https://api.anthropic.com/v1/models"
  142. r = httpx.get(url, headers=headers, timeout=10)
  143. # OAuth subscriptions without 1M context reject with 400 "long context beta is not yet available";
  144. # retry once with that beta stripped so doctor doesn't falsely report Anthropic as unreachable.
  145. if is_oauth and r.status_code == 400 and "long context beta" in r.text.lower() and "not yet available" in r.text.lower():
  146. headers["anthropic-beta"] = ",".join([b for b in _COMMON_BETAS if b != _CONTEXT_1M_BETA] + list(_OAUTH_ONLY_BETAS))
  147. r = httpx.get(url, headers=headers, timeout=10)
  148. except Exception as e:
  149. return _row(name, "warn", f"({e})")
  150. return _row(name, *{200: ("ok",), 401: ("fail", "(invalid API key)")}.get(r.status_code, ("warn", "(couldn't verify)")))
  151. def _probe_apikey_provider(pname, env_vars, default_url, base_env, supports_health_check) -> ProbeResult:
  152. key = next((k for k in (os.getenv(ev, "") for ev in env_vars) if k), "")
  153. if not key:
  154. return _skip(pname)
  155. label = pname.ljust(20)
  156. if not supports_health_check:
  157. return _row(pname, "ok", "(key configured)", label=label)
  158. try:
  159. import httpx
  160. base, url, headers = _apikey_request(key, base_env, default_url)
  161. if base.rstrip("/").endswith("/anthropic"):
  162. # Anthropic-only gateway (no OpenAI-compat sibling, so no /models): probe the route the runtime uses.
  163. r = _anthropic_messages_probe(base, key)
  164. if r.status_code == 400: # Anthropic-shaped 400 still proves route + auth (#66756)
  165. return _row(pname, "ok", label=label)
  166. if r.status_code == 403:
  167. return _row(pname, "fail", "(access denied)", [f"Check {env_vars[0]} in .env"], label=label)
  168. else:
  169. r = httpx.get(url, headers=headers, timeout=10)
  170. if pname == "Alibaba/DashScope" and not base and r.status_code == 401:
  171. r = httpx.get("https://dashscope.aliyuncs.com/compatible-mode/v1/models", headers=headers, timeout=10)
  172. except Exception as e:
  173. return _row(pname, "warn", f"({e})", label=label)
  174. if r.status_code == 401:
  175. return _row(pname, "fail", "(invalid API key)", [f"Check {env_vars[0]} in .env"], label=label)
  176. return _row(pname, "ok", label=label) if r.status_code == 200 else _row(pname, "warn", f"(HTTP {r.status_code})", label=label)
  177. def _anthropic_messages_probe(base: str, key: str):
  178. """POST ``<base>/v1/messages`` with ``max_tokens=1`` exactly as the Anthropic adapter would: same
  179. auth family (Bearer for Azure Foundry, else x-api-key) and the same ``api-version`` query. Azure
  180. Foundry's ``/anthropic`` route 404s on ``GET /models`` even when chat works (#66756)."""
  181. import httpx
  182. from agent.anthropic_adapter import _base_client_kwargs
  183. from agent.anthropic_endpoints import _requires_bearer_auth
  184. normalized, kwargs = _base_client_kwargs(base, None)
  185. auth = {"Authorization": f"Bearer {key}"} if _requires_bearer_auth(normalized) else {"x-api-key": key}
  186. headers = {"anthropic-version": "2023-06-01", "User-Agent": hermes_user_agent(), **auth}
  187. model = str(_model_cfg().get("default") or "").strip() or "claude-sonnet-4-5"
  188. body = {"model": model, "max_tokens": 1, "messages": [{"role": "user", "content": "ping"}]}
  189. return httpx.post(normalized + "/v1/messages", headers=headers, params=kwargs.get("default_query"), json=body, timeout=10)
  190. def _model_cfg() -> dict:
  191. try:
  192. from hermes_cli.config import load_config_readonly
  193. model_cfg = (load_config_readonly() or {}).get("model")
  194. except Exception:
  195. return {}
  196. return model_cfg if isinstance(model_cfg, dict) else {}
  197. def _apikey_request(key: str, base_env, default_url) -> tuple:
  198. """(effective base, models URL, headers) for a generic Bearer-auth probe, with the per-vendor rewrites."""
  199. base = os.getenv(base_env, "") if base_env else ""
  200. # Azure Foundry's base URL is per-resource and normally lives in config (model.base_url), not the env var.
  201. if not base and base_env == "AZURE_FOUNDRY_BASE_URL" and str(_model_cfg().get("provider") or "").strip().lower() == "azure-foundry":
  202. base = str(_model_cfg().get("base_url") or "").strip()
  203. # Kimi Code keys (sk-kimi-) → api.kimi.com/coding/v1 (OpenAI-compat surface exposing /models).
  204. if not base and key.startswith("sk-kimi-"):
  205. base = "https://api.kimi.com/coding/v1"
  206. # Anthropic-compat endpoints (/anthropic, api.kimi.com/coding sans /v1) lack /models — use the OpenAI-compat /v1 surface.
  207. if base and base.rstrip("/").endswith("/anthropic"):
  208. from agent.auxiliary_client import _to_openai_base_url
  209. base = _to_openai_base_url(base)
  210. if base_url_host_matches(base, "api.kimi.com") and base.rstrip("/").endswith("/coding"):
  211. base = base.rstrip("/") + "/v1"
  212. url = (base.rstrip("/") + "/models") if base else default_url
  213. headers = {"Authorization": f"Bearer {key}", "User-Agent": hermes_user_agent()}
  214. if base_url_host_matches(base, "api.kimi.com"):
  215. headers["User-Agent"] = "claude-code/0.1.0"
  216. # Google's Generative Language API rejects ``Authorization: Bearer <api-key>`` with 401
  217. # ACCESS_TOKEN_TYPE_UNSUPPORTED (reserved for OAuth 2 tokens); plain keys use ``x-goog-api-key``.
  218. if url and (base_url_host_matches(url, "generativelanguage.googleapis.com")
  219. or base_url_host_matches(url, "aiplatform.googleapis.com")):
  220. from agent.gemini_native_adapter import is_vertex_express_base_url, normalize_gemini_base_url
  221. root = url.rsplit("/models", 1)[0]
  222. if base_url_host_matches(url, "generativelanguage.googleapis.com") or is_vertex_express_base_url(root):
  223. # Normalize guarantees the version segment and completes an explicitly configured express
  224. # aiplatform base to the publishers form; the key itself never decides the surface — AQ.
  225. # keys exist for both AI Studio and Vertex express mode (#115306). The OAuth Vertex
  226. # ``…/endpoints/openapi`` base is OpenAI-compatible and stays exactly as configured.
  227. url = normalize_gemini_base_url(root) + "/models"
  228. headers.pop("Authorization", None)
  229. headers["x-goog-api-key"] = key
  230. return base, url, headers
  231. def _probe_bedrock() -> ProbeResult:
  232. name = "AWS Bedrock"
  233. try:
  234. from agent.bedrock_adapter import has_aws_credentials, resolve_aws_auth_env_var, resolve_bedrock_region
  235. except ImportError:
  236. return _skip(name)
  237. if not has_aws_credentials():
  238. return _skip(name)
  239. auth_var, region, label = resolve_aws_auth_env_var(), resolve_bedrock_region(), name.ljust(20)
  240. try:
  241. import boto3
  242. from botocore.config import Config as _BotoConfig
  243. # Trim retries so a transient failure doesn't pad the doctor run by 30+ seconds.
  244. client = boto3.client("bedrock", region_name=region, config=_BotoConfig(connect_timeout=5, read_timeout=10, retries={"max_attempts": 1}))
  245. n = len(client.list_foundation_models().get("modelSummaries", []))
  246. return _row(name, "ok", f"({auth_var}, {region}, {n} models)", label=label)
  247. except ImportError:
  248. hint = ("From the Hermes environment, run: "
  249. f"{install_hint('bedrock')}. "
  250. "Then restart Hermes.")
  251. return _row(name, "warn", "(boto3 not installed)", [hint], label=label)
  252. except Exception as e:
  253. err_name = type(e).__name__
  254. return _row(name, "warn", f"({err_name}: {e})", [f"AWS Bedrock: {err_name} — check IAM permissions for bedrock:ListFoundationModels"], label=label)
  255. def _probe_azure_entra() -> ProbeResult:
  256. """Probe Azure Foundry Entra ID auth, parallel to ``_probe_bedrock``.
  257. Skipped unless the active config has ``model.provider: azure-foundry`` AND ``model.auth_mode: entra_id``
  258. — we don't probe the token-service / CLI chain for plain API-key Azure. Bounded by a 10s timeout.
  259. """
  260. name = "Azure Foundry (Entra ID)"
  261. label = name.ljust(28)
  262. try:
  263. from hermes_cli.config import load_config
  264. cfg = load_config()
  265. model_cfg = cfg.get("model") if isinstance(cfg, dict) else {}
  266. if not isinstance(model_cfg, dict):
  267. return _skip(name)
  268. if [str(model_cfg.get(k) or "").strip().lower() for k in ("provider", "auth_mode")] != ["azure-foundry", "entra_id"]:
  269. return _skip(name)
  270. except Exception:
  271. return _skip(name)
  272. try:
  273. from agent.azure_identity_adapter import (
  274. EntraIdentityConfig, SCOPE_AI_AZURE_DEFAULT, describe_active_credential, has_azure_identity_installed,
  275. )
  276. except Exception as exc:
  277. return _row(name, "warn", f"(adapter import failed: {exc})", [f"Azure Foundry adapter import failed: {exc}"], label=label)
  278. if not has_azure_identity_installed():
  279. return _row(name, "warn", "(azure-identity not installed)", ["From the Hermes environment, run: "
  280. f"{install_hint('azure-identity')}. "
  281. "Then restart Hermes."], label=label)
  282. entra_cfg = model_cfg.get("entra") or {}
  283. scope = (str(entra_cfg.get("scope") or "").strip() if isinstance(entra_cfg, dict) else "") or SCOPE_AI_AZURE_DEFAULT
  284. info = describe_active_credential(config=EntraIdentityConfig(scope=scope), timeout_seconds=10.0)
  285. if info.get("ok"):
  286. tag = ", ".join(info.get("env_sources") or []) or "default credential chain"
  287. return _row(name, "ok", f"({tag}, scope={scope})", label=label)
  288. err = info.get("error") or "credential chain exhausted"
  289. hint = info.get("hint") or "Run `az login`, set AZURE_TENANT_ID/AZURE_CLIENT_ID/AZURE_CLIENT_SECRET, or attach a managed identity to this VM."
  290. return _row(name, "warn", f"({err})", [f"Azure Foundry Entra: {err}. {hint}"], label=label)
  291. def _load_network_config() -> dict:
  292. try:
  293. from hermes_cli.config import load_config_readonly
  294. net = (load_config_readonly() or {}).get("network")
  295. except Exception:
  296. return {}
  297. return net if isinstance(net, dict) else {}
  298. def _tcp_connect(sockaddr, timeout: float) -> None:
  299. """Open + close one IPv6 TCP connection; raises OSError (TimeoutError on a dead route)."""
  300. with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock:
  301. sock.settimeout(timeout)
  302. sock.connect(sockaddr)
  303. _IPV6_PROBE_TIMEOUT = 2.0
  304. def _probe_ipv6_path() -> ProbeResult:
  305. """Dead-IPv6-route detector (#114265): an advertised AAAA path that only times out makes every
  306. serial connect burn its full timeout before IPv4 answers. Name the remedy instead of stalling."""
  307. name = "IPv6 route"
  308. if _load_network_config().get("force_ipv4"):
  309. return _skip(name) # IPv6 is never dialled
  310. host = urlsplit(OPENROUTER_MODELS_URL).hostname
  311. try:
  312. infos = socket.getaddrinfo(host, 443, socket.AF_INET6, socket.SOCK_STREAM)
  313. except OSError:
  314. infos = []
  315. if not infos:
  316. return _skip(name) # no AAAA record / no IPv6 resolver: nothing to test
  317. try:
  318. _tcp_connect(infos[0][4], _IPV6_PROBE_TIMEOUT)
  319. except TimeoutError:
  320. remedy = "set `network.force_ipv4: true` in config.yaml (or fix the IPv6 route)"
  321. return _row(name, "warn", f"(IPv6 route to {host} advertised but dead: connect timed out after "
  322. f"{_IPV6_PROBE_TIMEOUT:g}s — {remedy})",
  323. [f"Dead IPv6 route: every IPv6-first connect stalls before IPv4 answers. Fix: {remedy}"])
  324. except OSError as e:
  325. if e.errno in (errno.ENETUNREACH, errno.EHOSTUNREACH, errno.EADDRNOTAVAIL):
  326. return _row(name, "ok", "(no IPv6 route — IPv4 only)") # fails fast, so no stall
  327. return _row(name, "ok", f"(IPv6 path to {host} reachable)") # refused/reset also prove a live path
  328. # /rate_limit is reachable by EVERY token type and does not count against the quota. /user answers
  329. # 403 "Resource not accessible by integration" for App installation tokens (the GITHUB_TOKEN every
  330. # Actions job exports), which would paint a valid token red.
  331. GITHUB_API_PROBE_URL = "https://api.github.com/rate_limit"
  332. def _probe_github_token() -> ProbeResult:
  333. """Validate a configured ``GITHUB_TOKEN``/``GH_TOKEN`` against api.github.com (#115257).
  334. A dead PAT in ``.env`` used to fail every git-auth clone with a message that never named the
  335. token; the resolver now falls through to the gh CLI, and this row tells the user WHICH file
  336. still carries the stale token so they can remove it.
  337. """
  338. name = "GitHub token"
  339. from hermes_cli.config import get_env_value, load_env
  340. var = next((v for v in ("GITHUB_TOKEN", "GH_TOKEN") if get_env_value(v)), None)
  341. if var is None:
  342. return _skip(name) # the Skills Hub section already reports gh-CLI / no-token state
  343. from hermes_cli.doctor import _DHH
  344. where = f"{_DHH}/.env" if var in load_env() else "the environment"
  345. try:
  346. import httpx
  347. r = httpx.get(GITHUB_API_PROBE_URL, timeout=10, headers={
  348. "Authorization": f"Bearer {get_env_value(var)}", "User-Agent": hermes_user_agent(),
  349. "Accept": "application/vnd.github+json"})
  350. except Exception as e:
  351. return _row(name, "fail", f"({e})", ["Check network connectivity"])
  352. if r.status_code == 200:
  353. return _row(name, "ok", f"({var} from {where} accepted by api.github.com)")
  354. if r.status_code == 401:
  355. return _row(name, "fail", f"({var} in {where} rejected by api.github.com — expired or revoked)",
  356. [f"{var} in {where} is expired or revoked: remove it (gh CLI login is used instead) or paste a fresh token"])
  357. return _row(name, "fail", f"(HTTP {r.status_code} from api.github.com)")
  358. def build_probes() -> list:
  359. """(label, callable) pairs in display order."""
  360. global _APIKEY_PROVIDERS_CACHE
  361. if _APIKEY_PROVIDERS_CACHE is None:
  362. _APIKEY_PROVIDERS_CACHE = _build_apikey_providers_list()
  363. return [
  364. ("IPv6 route", _probe_ipv6_path),
  365. ("OpenRouter API", _probe_openrouter), ("Anthropic API", _probe_anthropic),
  366. # functools.partial binds each row's args so every callable keeps its own provider.
  367. *((row[0], functools.partial(_probe_apikey_provider, *row)) for row in _APIKEY_PROVIDERS_CACHE),
  368. ("AWS Bedrock", _probe_bedrock), ("Azure Foundry (Entra ID)", _probe_azure_entra),
  369. ("GitHub token", _probe_github_token),
  370. ]
  371. def run_probes(probes: list) -> list:
  372. """Run every probe in a thread pool; results in submission order.
  373. Probes are independent HTTP calls (series cost ~5s wall, 2s of it boto3's IMDS lookup);
  374. parallel collapses the section to roughly the slowest single probe without changing the output.
  375. """
  376. # Disable boto3's EC2 instance-metadata probe (169.254.169.254, multi-second timeout off-EC2). Set on the
  377. # parent thread before submitting so it never races a worker; has_aws_credentials() already gates on real creds.
  378. normalize_proxy_env_vars() # a bracketed-IPv6 NO_PROXY entry ([::1], Clash Verge/mihomo) makes every bare
  379. # trust_env client a probe builds raise InvalidURL at construction (#118159) — sanitize once, before any
  380. # worker runs, so no probe ever reports "(Invalid port: :1])" as provider downtime.
  381. _imds_prev = os.environ.get("AWS_EC2_METADATA_DISABLED")
  382. os.environ["AWS_EC2_METADATA_DISABLED"] = "true"
  383. try:
  384. # 8 workers is plenty — each probe is one HTTP call plus a TLS handshake.
  385. with concurrent.futures.ThreadPoolExecutor(max_workers=8, thread_name_prefix="doctor-probe") as ex:
  386. return [f.result() for f in [ex.submit(fn) for _, fn in probes]]
  387. finally:
  388. if _imds_prev is None:
  389. os.environ.pop("AWS_EC2_METADATA_DISABLED", None)
  390. else:
  391. os.environ["AWS_EC2_METADATA_DISABLED"] = _imds_prev