auxiliary_unavailable.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. """Why an auxiliary client could not be built — and the Nous credential failure behind it.
  2. ``_resolve_nous_runtime_api`` must swallow the resolver's ``AuthError`` and return None so the
  3. ladder can still fall back (stored token, fallback_chain). Swallowing it at DEBUG left the goal
  4. loop reporting ``judge error: RuntimeError`` while the real cause was an ``invalid_grant`` refresh
  5. (#42177). This module remembers the latest failure so the ladder's raise and the goal judge can
  6. name it, and warns ONCE per distinct message so operators see it without debug logging.
  7. """
  8. import contextlib
  9. import logging
  10. import threading
  11. import time
  12. from datetime import datetime
  13. from typing import Any, Optional
  14. from hermes_time import safe_strftime
  15. logger = logging.getLogger(__name__)
  16. class AuxiliaryClientUnavailable(RuntimeError):
  17. """No auxiliary client could be built for the task (missing credentials / provider)."""
  18. _lock = threading.Lock()
  19. _last_nous_detail: Optional[str] = None
  20. _warned_nous_details: set[str] = set()
  21. def _quarantined_nous_error(exc: BaseException) -> BaseException:
  22. """Prefer the persisted terminal quarantine marker over *exc*; sentence-terminate either message.
  23. The pool rung swallows the real ``invalid_grant`` and wipes the dead tokens, so by the time the
  24. auth-store resolver runs it can only say "No access token found"; the marker it wrote still
  25. carries the message and code the user needs (#42177). ``format_auth_error`` appends the
  26. remediation sentence with a space, so an unterminated message reads "Invalid refresh token Run …".
  27. """
  28. from hermes_cli.auth import AuthError, get_provider_auth_state
  29. from hermes_cli.auth_nous import _terminal_quarantine_marker
  30. with contextlib.suppress(Exception):
  31. marker = _terminal_quarantine_marker(get_provider_auth_state("nous") or {})
  32. if marker and marker.get("message"):
  33. return AuthError(_sentence(marker["message"]), provider="nous", code=marker.get("code"),
  34. relogin_required=True)
  35. if isinstance(exc, AuthError):
  36. return AuthError(_sentence(exc), provider=exc.provider, code=exc.code,
  37. relogin_required=exc.relogin_required, retry_after=exc.retry_after,
  38. retryable=exc.retryable)
  39. return exc
  40. def _sentence(text: object) -> str:
  41. return str(text).strip().rstrip(".") + "."
  42. def pool_cooldown_message(provider_id: str) -> Optional[str]:
  43. """The "all N credentials … are cooling down" error when the provider's pool is fully benched.
  44. ``resolve_provider_client()`` returns ``None`` both when no credential exists and when every
  45. pool entry sits in a 429/quota cooldown, so the raise sites could only say "no credentials
  46. were found. Run hermes auth add …" — wrong on both counts for a valid OAuth grant that is
  47. merely rate-limited (#56810). Read the persisted pool state (no seeding, no writes) and name
  48. the cooldown and its reset time instead; ``None`` when the pool is empty or a credential is
  49. usable (the caller keeps the missing-credential diagnostic).
  50. """
  51. from agent.credential_pool import STATUS_DEAD, PooledCredential, _exhausted_until
  52. from hermes_cli.auth import read_credential_pool
  53. entries = []
  54. with contextlib.suppress(Exception):
  55. entries = [PooledCredential.from_dict(provider_id, e)
  56. for e in read_credential_pool(provider_id) if isinstance(e, dict)]
  57. live = [e for e in entries if e.last_status != STATUS_DEAD]
  58. if not live:
  59. return None
  60. now = time.time()
  61. resets = [until for e in live
  62. for until in (_exhausted_until(e, sole_credential=len(live) == 1),)
  63. if until is not None and until > now]
  64. if len(resets) != len(live):
  65. return None
  66. when = safe_strftime(datetime.fromtimestamp(min(resets)).astimezone(), "%Y-%m-%d %H:%M %Z")
  67. which = ("its only credential is" if len(live) == 1
  68. else f"all {len(live)} credentials are")
  69. return (f"Provider '{provider_id}' is set in config.yaml but {which} cooling down after a "
  70. f"rate limit / quota error (429); the next one resets at {when}. Wait for the reset, "
  71. f"add another credential with `hermes auth add {provider_id}`, or switch to a "
  72. "different provider with `hermes model`.")
  73. class ProviderCredentialsExhaustedError(RuntimeError):
  74. """An explicit provider's credential pool is fully exhausted (402 out of credits, quota bench).
  75. Subclasses ``RuntimeError`` so every existing ``except RuntimeError`` keeps working; the distinct
  76. type lets a caller (gateway, Desktop) route a billing/quota failure to billing guidance instead
  77. of the provider-setup flow a plain "No LLM provider configured" would trigger (#94785).
  78. """
  79. def __init__(self, message: str, *, provider: Optional[str] = None):
  80. super().__init__(message)
  81. self.provider = provider
  82. def pool_billing_message(
  83. provider_id: str,
  84. *,
  85. model: Optional[str] = None,
  86. pool: Optional[Any] = None,
  87. ) -> Optional[str]:
  88. """The "Provider 'X' … is out of usable credentials (402)" error for a billing-burned pool.
  89. ``resolve_provider_client()`` returns ``None`` both when no credential exists and when every
  90. pool entry has been marked exhausted — a 402 out-of-credits account, a 429/quota bench. The
  91. ``openrouter`` / ``custom`` branch of ``agent_init`` has no provider-specific missing-credentials
  92. message, so a burned pool there fell through to the generic "No LLM provider configured" setup
  93. message: actively misleading when a *different* provider is configured and working and only the
  94. session's override is out of credits (#94785). Name the provider, the session's model, the last
  95. error code and the provider's own message instead.
  96. Only a genuine billing verdict — a 402, or the classifier's ``billing`` reason on a 403 spending
  97. limit — is named here; a plain 429/quota bench keeps the existing cooldown wording (with its
  98. reset time, #56810), which the caller falls back to when this returns ``None``.
  99. *pool* is an already-loaded ``CredentialPool`` (preferred — it carries the live in-memory
  100. exhaustion state); otherwise the persisted rows are read (no seeding, no writes, mirroring
  101. ``pool_cooldown_message``). ``None`` when the pool is empty, a credential is usable, no entry is
  102. currently benched, or none carries a billing verdict.
  103. """
  104. from agent.credential_pool import (
  105. FAILURE_REASON_BILLING,
  106. STATUS_DEAD,
  107. STATUS_EXHAUSTED,
  108. PooledCredential,
  109. _exhausted_until,
  110. )
  111. entries: list = []
  112. with contextlib.suppress(Exception):
  113. if pool is not None:
  114. entries = list(pool.entries())
  115. else:
  116. from hermes_cli.auth import read_credential_pool
  117. entries = [PooledCredential.from_dict(provider_id, e)
  118. for e in read_credential_pool(provider_id) if isinstance(e, dict)]
  119. live = [e for e in entries if getattr(e, "last_status", None) != STATUS_DEAD]
  120. if not live:
  121. return None
  122. now = time.time()
  123. benched = [
  124. e for e in live
  125. if getattr(e, "last_status", None) == STATUS_EXHAUSTED
  126. and (_exhausted_until(e, sole_credential=len(live) == 1) or 0) > now
  127. ]
  128. if len(benched) != len(live):
  129. # A credential is still usable (or merely cooling down on one model): not our case.
  130. return None
  131. billing = [
  132. e for e in benched
  133. if getattr(e, "last_error_code", None) == 402
  134. or getattr(e, "failure_reason", None) == FAILURE_REASON_BILLING
  135. ]
  136. if not billing:
  137. return None
  138. pick = billing[0]
  139. code = getattr(pick, "last_error_code", None)
  140. detail = str(getattr(pick, "last_error_message", None) or "").strip()
  141. which = "its only credential is" if len(live) == 1 else f"all {len(live)} credentials are"
  142. scope = f" for model '{model}'" if model else ""
  143. code_text = f" (last error {code}{': ' + detail if detail else ''})" if code else ""
  144. return (
  145. f"Provider '{provider_id}'{scope} is out of usable credentials: {which} exhausted"
  146. f"{code_text}. Add credits or update billing with that provider, then retry, or switch "
  147. f"to a different provider with `hermes model`."
  148. )
  149. class ProviderNotConfiguredError(RuntimeError):
  150. """Agent init found no usable inference provider: nothing is configured, or the explicitly
  151. configured one has no credentials.
  152. A distinct type because the remedy is setup, not a retry: clients route it to their provider
  153. setup / onboarding flow. The wording of the two messages has changed more than once, so a
  154. client that recognises this failure by matching their text silently stops routing (#119232's
  155. sibling); the type and the gateway's ``code`` survive rewording.
  156. """
  157. def missing_provider_credentials_message(provider_id: str) -> str:
  158. """The "Provider 'X' is set in config.yaml but …" error for an explicit provider with no credentials.
  159. The remedy comes from the registry, never from the provider id: ``f"{id.upper()}_API_KEY"``
  160. invents names nothing reads (alibaba → ALIBABA_API_KEY instead of DASHSCOPE_API_KEY, and the
  161. unsettable MINIMAX-OAUTH_API_KEY for OAuth ids, #114405 / #78996). OAuth providers have no key
  162. env var at all, so they are pointed at the sign-in command instead. A pool whose every
  163. credential is cooling down is not "missing" — that case names the cooldown (#56810).
  164. """
  165. cooldown = pool_cooldown_message(provider_id)
  166. if cooldown:
  167. return cooldown
  168. pconfig = None
  169. with contextlib.suppress(Exception):
  170. from hermes_cli.auth import PROVIDER_REGISTRY
  171. pconfig = PROVIDER_REGISTRY.get(provider_id)
  172. env_vars = tuple(getattr(pconfig, "api_key_env_vars", None) or ())
  173. problem, remedy = "no API key was found", ""
  174. from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES
  175. if provider_id in LLAMACPP_ALIASES:
  176. problem, remedy = "no local llama.cpp server is running", "Start it in Settings → Providers → Local models"
  177. elif env_vars:
  178. remedy = f"Set the {env_vars[0]} environment variable"
  179. elif pconfig is None:
  180. remedy = f"Set the {provider_id.upper().replace('-', '_')}_API_KEY environment variable"
  181. elif str(pconfig.auth_type).startswith("oauth"):
  182. problem, remedy = "no credentials were found", f"Run `hermes auth add {provider_id}` to sign in"
  183. else:
  184. problem = "no credentials were found"
  185. switch = "switch to a different provider with `hermes model`."
  186. return (f"Provider '{provider_id}' is set in config.yaml but {problem}. "
  187. + (f"{remedy}, or {switch}" if remedy else switch.capitalize()))
  188. def _nous_credential_present(exc: BaseException) -> bool:
  189. """True when a Nous credential exists that failed — a coded error (``invalid_grant``, a quarantine
  190. marker, ``refresh_failed``) or persisted Nous auth state. The uncoded "not logged into Nous Portal"
  191. with no stored state is the normal condition for users who never chose Nous; the auto-route walk
  192. hits it on every discovery pass and must not warn (the ladder already logs its own summary).
  193. """
  194. if getattr(exc, "code", None):
  195. return True
  196. from hermes_cli.auth import get_provider_auth_state
  197. with contextlib.suppress(Exception):
  198. return bool(get_provider_auth_state("nous"))
  199. return False
  200. def record_nous_credential_failure(exc: BaseException) -> str:
  201. """Remember *exc* as the latest Nous credential failure.
  202. Logged once per distinct message: WARNING when a real credential failed, DEBUG when Hermes was
  203. simply never logged into Nous.
  204. """
  205. from hermes_cli.auth import format_auth_error
  206. exc = _quarantined_nous_error(exc)
  207. message = format_auth_error(exc) if isinstance(exc, Exception) else str(exc)
  208. message = message.strip() or type(exc).__name__
  209. code = getattr(exc, "code", None)
  210. if code and str(code) not in message:
  211. message = f"{message} (code: {code})"
  212. detail = f"Nous Portal runtime credentials unavailable: {message}"
  213. global _last_nous_detail
  214. with _lock:
  215. _last_nous_detail = detail
  216. first_time = detail not in _warned_nous_details
  217. _warned_nous_details.add(detail)
  218. if first_time:
  219. level = logging.WARNING if _nous_credential_present(exc) else logging.DEBUG
  220. logger.log(level, "Auxiliary Nous client unavailable: %s", detail)
  221. return detail
  222. def clear_nous_credential_failure() -> None:
  223. global _last_nous_detail
  224. with _lock:
  225. _last_nous_detail = None
  226. def nous_credential_failure_detail() -> Optional[str]:
  227. """The latest recorded Nous credential failure, or None when the last resolution succeeded."""
  228. with _lock:
  229. return _last_nous_detail