| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275 |
- """Why an auxiliary client could not be built — and the Nous credential failure behind it.
- ``_resolve_nous_runtime_api`` must swallow the resolver's ``AuthError`` and return None so the
- ladder can still fall back (stored token, fallback_chain). Swallowing it at DEBUG left the goal
- loop reporting ``judge error: RuntimeError`` while the real cause was an ``invalid_grant`` refresh
- (#42177). This module remembers the latest failure so the ladder's raise and the goal judge can
- name it, and warns ONCE per distinct message so operators see it without debug logging.
- """
- import contextlib
- import logging
- import threading
- import time
- from datetime import datetime
- from typing import Any, Optional
- from hermes_time import safe_strftime
- logger = logging.getLogger(__name__)
- class AuxiliaryClientUnavailable(RuntimeError):
- """No auxiliary client could be built for the task (missing credentials / provider)."""
- _lock = threading.Lock()
- _last_nous_detail: Optional[str] = None
- _warned_nous_details: set[str] = set()
- def _quarantined_nous_error(exc: BaseException) -> BaseException:
- """Prefer the persisted terminal quarantine marker over *exc*; sentence-terminate either message.
- The pool rung swallows the real ``invalid_grant`` and wipes the dead tokens, so by the time the
- auth-store resolver runs it can only say "No access token found"; the marker it wrote still
- carries the message and code the user needs (#42177). ``format_auth_error`` appends the
- remediation sentence with a space, so an unterminated message reads "Invalid refresh token Run …".
- """
- from hermes_cli.auth import AuthError, get_provider_auth_state
- from hermes_cli.auth_nous import _terminal_quarantine_marker
- with contextlib.suppress(Exception):
- marker = _terminal_quarantine_marker(get_provider_auth_state("nous") or {})
- if marker and marker.get("message"):
- return AuthError(_sentence(marker["message"]), provider="nous", code=marker.get("code"),
- relogin_required=True)
- if isinstance(exc, AuthError):
- return AuthError(_sentence(exc), provider=exc.provider, code=exc.code,
- relogin_required=exc.relogin_required, retry_after=exc.retry_after,
- retryable=exc.retryable)
- return exc
- def _sentence(text: object) -> str:
- return str(text).strip().rstrip(".") + "."
- def pool_cooldown_message(provider_id: str) -> Optional[str]:
- """The "all N credentials … are cooling down" error when the provider's pool is fully benched.
- ``resolve_provider_client()`` returns ``None`` both when no credential exists and when every
- pool entry sits in a 429/quota cooldown, so the raise sites could only say "no credentials
- were found. Run hermes auth add …" — wrong on both counts for a valid OAuth grant that is
- merely rate-limited (#56810). Read the persisted pool state (no seeding, no writes) and name
- the cooldown and its reset time instead; ``None`` when the pool is empty or a credential is
- usable (the caller keeps the missing-credential diagnostic).
- """
- from agent.credential_pool import STATUS_DEAD, PooledCredential, _exhausted_until
- from hermes_cli.auth import read_credential_pool
- entries = []
- with contextlib.suppress(Exception):
- entries = [PooledCredential.from_dict(provider_id, e)
- for e in read_credential_pool(provider_id) if isinstance(e, dict)]
- live = [e for e in entries if e.last_status != STATUS_DEAD]
- if not live:
- return None
- now = time.time()
- resets = [until for e in live
- for until in (_exhausted_until(e, sole_credential=len(live) == 1),)
- if until is not None and until > now]
- if len(resets) != len(live):
- return None
- when = safe_strftime(datetime.fromtimestamp(min(resets)).astimezone(), "%Y-%m-%d %H:%M %Z")
- which = ("its only credential is" if len(live) == 1
- else f"all {len(live)} credentials are")
- return (f"Provider '{provider_id}' is set in config.yaml but {which} cooling down after a "
- f"rate limit / quota error (429); the next one resets at {when}. Wait for the reset, "
- f"add another credential with `hermes auth add {provider_id}`, or switch to a "
- "different provider with `hermes model`.")
- class ProviderCredentialsExhaustedError(RuntimeError):
- """An explicit provider's credential pool is fully exhausted (402 out of credits, quota bench).
- Subclasses ``RuntimeError`` so every existing ``except RuntimeError`` keeps working; the distinct
- type lets a caller (gateway, Desktop) route a billing/quota failure to billing guidance instead
- of the provider-setup flow a plain "No LLM provider configured" would trigger (#94785).
- """
- def __init__(self, message: str, *, provider: Optional[str] = None):
- super().__init__(message)
- self.provider = provider
- def pool_billing_message(
- provider_id: str,
- *,
- model: Optional[str] = None,
- pool: Optional[Any] = None,
- ) -> Optional[str]:
- """The "Provider 'X' … is out of usable credentials (402)" error for a billing-burned pool.
- ``resolve_provider_client()`` returns ``None`` both when no credential exists and when every
- pool entry has been marked exhausted — a 402 out-of-credits account, a 429/quota bench. The
- ``openrouter`` / ``custom`` branch of ``agent_init`` has no provider-specific missing-credentials
- message, so a burned pool there fell through to the generic "No LLM provider configured" setup
- message: actively misleading when a *different* provider is configured and working and only the
- session's override is out of credits (#94785). Name the provider, the session's model, the last
- error code and the provider's own message instead.
- Only a genuine billing verdict — a 402, or the classifier's ``billing`` reason on a 403 spending
- limit — is named here; a plain 429/quota bench keeps the existing cooldown wording (with its
- reset time, #56810), which the caller falls back to when this returns ``None``.
- *pool* is an already-loaded ``CredentialPool`` (preferred — it carries the live in-memory
- exhaustion state); otherwise the persisted rows are read (no seeding, no writes, mirroring
- ``pool_cooldown_message``). ``None`` when the pool is empty, a credential is usable, no entry is
- currently benched, or none carries a billing verdict.
- """
- from agent.credential_pool import (
- FAILURE_REASON_BILLING,
- STATUS_DEAD,
- STATUS_EXHAUSTED,
- PooledCredential,
- _exhausted_until,
- )
- entries: list = []
- with contextlib.suppress(Exception):
- if pool is not None:
- entries = list(pool.entries())
- else:
- from hermes_cli.auth import read_credential_pool
- entries = [PooledCredential.from_dict(provider_id, e)
- for e in read_credential_pool(provider_id) if isinstance(e, dict)]
- live = [e for e in entries if getattr(e, "last_status", None) != STATUS_DEAD]
- if not live:
- return None
- now = time.time()
- benched = [
- e for e in live
- if getattr(e, "last_status", None) == STATUS_EXHAUSTED
- and (_exhausted_until(e, sole_credential=len(live) == 1) or 0) > now
- ]
- if len(benched) != len(live):
- # A credential is still usable (or merely cooling down on one model): not our case.
- return None
- billing = [
- e for e in benched
- if getattr(e, "last_error_code", None) == 402
- or getattr(e, "failure_reason", None) == FAILURE_REASON_BILLING
- ]
- if not billing:
- return None
- pick = billing[0]
- code = getattr(pick, "last_error_code", None)
- detail = str(getattr(pick, "last_error_message", None) or "").strip()
- which = "its only credential is" if len(live) == 1 else f"all {len(live)} credentials are"
- scope = f" for model '{model}'" if model else ""
- code_text = f" (last error {code}{': ' + detail if detail else ''})" if code else ""
- return (
- f"Provider '{provider_id}'{scope} is out of usable credentials: {which} exhausted"
- f"{code_text}. Add credits or update billing with that provider, then retry, or switch "
- f"to a different provider with `hermes model`."
- )
- class ProviderNotConfiguredError(RuntimeError):
- """Agent init found no usable inference provider: nothing is configured, or the explicitly
- configured one has no credentials.
- A distinct type because the remedy is setup, not a retry: clients route it to their provider
- setup / onboarding flow. The wording of the two messages has changed more than once, so a
- client that recognises this failure by matching their text silently stops routing (#119232's
- sibling); the type and the gateway's ``code`` survive rewording.
- """
- def missing_provider_credentials_message(provider_id: str) -> str:
- """The "Provider 'X' is set in config.yaml but …" error for an explicit provider with no credentials.
- The remedy comes from the registry, never from the provider id: ``f"{id.upper()}_API_KEY"``
- invents names nothing reads (alibaba → ALIBABA_API_KEY instead of DASHSCOPE_API_KEY, and the
- unsettable MINIMAX-OAUTH_API_KEY for OAuth ids, #114405 / #78996). OAuth providers have no key
- env var at all, so they are pointed at the sign-in command instead. A pool whose every
- credential is cooling down is not "missing" — that case names the cooldown (#56810).
- """
- cooldown = pool_cooldown_message(provider_id)
- if cooldown:
- return cooldown
- pconfig = None
- with contextlib.suppress(Exception):
- from hermes_cli.auth import PROVIDER_REGISTRY
- pconfig = PROVIDER_REGISTRY.get(provider_id)
- env_vars = tuple(getattr(pconfig, "api_key_env_vars", None) or ())
- problem, remedy = "no API key was found", ""
- from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES
- if provider_id in LLAMACPP_ALIASES:
- problem, remedy = "no local llama.cpp server is running", "Start it in Settings → Providers → Local models"
- elif env_vars:
- remedy = f"Set the {env_vars[0]} environment variable"
- elif pconfig is None:
- remedy = f"Set the {provider_id.upper().replace('-', '_')}_API_KEY environment variable"
- elif str(pconfig.auth_type).startswith("oauth"):
- problem, remedy = "no credentials were found", f"Run `hermes auth add {provider_id}` to sign in"
- else:
- problem = "no credentials were found"
- switch = "switch to a different provider with `hermes model`."
- return (f"Provider '{provider_id}' is set in config.yaml but {problem}. "
- + (f"{remedy}, or {switch}" if remedy else switch.capitalize()))
- def _nous_credential_present(exc: BaseException) -> bool:
- """True when a Nous credential exists that failed — a coded error (``invalid_grant``, a quarantine
- marker, ``refresh_failed``) or persisted Nous auth state. The uncoded "not logged into Nous Portal"
- with no stored state is the normal condition for users who never chose Nous; the auto-route walk
- hits it on every discovery pass and must not warn (the ladder already logs its own summary).
- """
- if getattr(exc, "code", None):
- return True
- from hermes_cli.auth import get_provider_auth_state
- with contextlib.suppress(Exception):
- return bool(get_provider_auth_state("nous"))
- return False
- def record_nous_credential_failure(exc: BaseException) -> str:
- """Remember *exc* as the latest Nous credential failure.
- Logged once per distinct message: WARNING when a real credential failed, DEBUG when Hermes was
- simply never logged into Nous.
- """
- from hermes_cli.auth import format_auth_error
- exc = _quarantined_nous_error(exc)
- message = format_auth_error(exc) if isinstance(exc, Exception) else str(exc)
- message = message.strip() or type(exc).__name__
- code = getattr(exc, "code", None)
- if code and str(code) not in message:
- message = f"{message} (code: {code})"
- detail = f"Nous Portal runtime credentials unavailable: {message}"
- global _last_nous_detail
- with _lock:
- _last_nous_detail = detail
- first_time = detail not in _warned_nous_details
- _warned_nous_details.add(detail)
- if first_time:
- level = logging.WARNING if _nous_credential_present(exc) else logging.DEBUG
- logger.log(level, "Auxiliary Nous client unavailable: %s", detail)
- return detail
- def clear_nous_credential_failure() -> None:
- global _last_nous_detail
- with _lock:
- _last_nous_detail = None
- def nous_credential_failure_detail() -> Optional[str]:
- """The latest recorded Nous credential failure, or None when the last resolution succeeded."""
- with _lock:
- return _last_nous_detail
|