| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249 |
- """Structured error-surface descriptors for UI clients (Desktop/TUI).
- Maps the internal failure taxonomy (``FailoverReason`` values carried in turn
- results as ``failure_reason``, or raw exceptions from the turn dispatcher)
- onto a small, stable wire descriptor ``{"layer", "code", "retryable"}``.
- Layers: provider (model API rejected/failed), endpoint (user-configured
- custom/local endpoint transport failure), streaming (SSE dropped mid-turn),
- auth, billing (fallback signal; clients usually have a richer
- ``billing_block``), gateway (local runtime errored), disk (disk full).
- Dependency-light and NEVER raises: surfacing diagnostics must not break the
- error path it describes. Descriptors are advisory — clients fall back to
- string sniffing when absent or partial.
- """
- from __future__ import annotations
- import logging
- from typing import Any, Optional
- logger = logging.getLogger(__name__)
- LAYER_PROVIDER = "provider"
- LAYER_ENDPOINT = "endpoint"
- LAYER_STREAMING = "streaming"
- LAYER_AUTH = "auth"
- LAYER_BILLING = "billing"
- LAYER_GATEWAY = "gateway"
- LAYER_DISK = "disk"
- # failure_reason → UI layer. Unlisted reasons fall back to LAYER_PROVIDER:
- # every FailoverReason comes from classifying a provider call. Loop-site codes
- # (agent/turn_failure_copy.py::SITE_FAILURE_CODES) are listed explicitly: the
- # ones that are not provider verdicts map to the gateway layer so the client
- # does not offer "Switch provider"; the ones the model/provider caused
- # (cut-off output, empty or broken reply) stay on the provider layer, where the
- # clients' per-code copy names the real fix (`continue`, smaller steps, /retry).
- _REASON_TO_LAYER = {
- "auth": LAYER_AUTH, "auth_permanent": LAYER_AUTH, "billing": LAYER_BILLING, "billing_unverified": LAYER_BILLING,
- "loop_error": LAYER_GATEWAY, "interpreter_shutdown": LAYER_GATEWAY, "session_busy": LAYER_GATEWAY,
- "gateway_connection": LAYER_GATEWAY, "host_error": LAYER_GATEWAY, "workspace_unavailable": LAYER_GATEWAY,
- "launch_credentials_unavailable": LAYER_GATEWAY,
- "local_runtime_unavailable": LAYER_ENDPOINT, # the managed local model server is down
- "truncated": LAYER_PROVIDER, "empty_response": LAYER_PROVIDER, "invalid_response": LAYER_PROVIDER,
- "context_overflow": LAYER_PROVIDER, # a bigger-window model IS the fix, so Switch provider applies
- }
- # Failures between us and the base_url (not a provider verdict); on a
- # custom/local endpoint they point at the user's endpoint config.
- _TRANSPORT_REASONS = {"timeout", "ssl_cert_verification"}
- # Free-tier kinds where a later send can succeed on its own (a wait, an outage clearing); the
- # rest need a sign-in or another provider.
- _FREE_TIER_RETRYABLE_KINDS = {"rate_limited", "at_capacity", "outage"}
- # Deterministic for the request — a bare "Retry" repeats the failure. Fallback
- # only: current backends stamp the classifier's verdict in ``failure_retryable``.
- # Kept in sync with ``classify_api_error``'s retryable=False verdicts.
- _NON_RETRYABLE_REASONS = {
- "auth", "auth_permanent", "billing", "billing_unverified", "content_policy_blocked",
- "provider_policy_blocked", "data_policy_denied", "model_not_found", "format_error", "ssl_cert_verification",
- "context_overflow", "interpreter_shutdown", "upstream_blocked",
- }
- # Providers whose base_url is user-supplied rather than a known vendor.
- _CUSTOM_ENDPOINT_PROVIDERS = {"custom", "local", "llama.cpp", "llamacpp", "ollama", "lmstudio", "vllm"}
- # Mid-stream drop markers. Deliberately narrow: our own retry-exhaustion
- # summaries plus the OpenAI SDK's stream-abort errors.
- _STREAM_DROP_FRAGMENTS = (
- "stream connection", "peer closed connection", "incomplete chunked read",
- "connection broken", "stream ended prematurely", "sse", "mid-stream",
- )
- # Exception top-level modules that mean "API/transport call failed" (vs. a bug
- # in our dispatcher = gateway layer): every SDK family our adapters raise from
- # plus raw transports.
- _API_EXC_MODULE_PREFIXES = (
- "openai", "httpx", "httpcore", "anthropic", "botocore", "boto3", "google",
- "grpc", "requests", "aiohttp", "ssl", "socket", "urllib",
- )
- def _is_custom_endpoint(provider: Optional[str]) -> bool:
- p = (provider or "").strip().lower()
- return p in _CUSTOM_ENDPOINT_PROVIDERS or p.startswith("custom:")
- def _looks_like_stream_drop(message: str) -> bool:
- return any(fragment in message.lower() for fragment in _STREAM_DROP_FRAGMENTS)
- def _surface(layer: str, code: str, retryable: bool, provider: str = "", model: str = "") -> dict:
- # Identity captured at classification time, so clients report the session
- # that actually failed — not whatever the composer points at later.
- identity = {k: v for k, v in (("provider", provider), ("model", model)) if v}
- surface = {"layer": layer, "code": code, "retryable": bool(retryable), **identity}
- if provider:
- # Clients name the failing provider in the card copy ("OpenCode Go did
- # not answer…"), not by its config slug.
- surface["provider_label"] = _provider_label(provider)
- if layer == LAYER_AUTH and provider:
- # OAuth providers are fixed by signing in again; API-key providers by
- # replacing the key. The client's one-click recovery needs to know which.
- surface["auth_kind"] = auth_kind(provider)
- if surface["auth_kind"] == "api_key" and (env_var := _api_key_env(provider)):
- surface["api_key_env"] = env_var
- return surface
- def _api_key_env(provider: str) -> str:
- """The env var holding ``provider``'s API key, so the client can open that row."""
- try:
- from hermes_cli.provider_catalog import provider_catalog_by_slug
- descriptor = provider_catalog_by_slug().get(provider.strip().lower())
- return descriptor.api_key_env_vars[0] if descriptor and descriptor.api_key_env_vars else ""
- except Exception: # pragma: no cover — advisory only
- return ""
- def _provider_label(provider: str) -> str:
- try:
- from hermes_cli.models import provider_label
- return provider_label(provider)
- except Exception: # pragma: no cover — advisory only
- return provider
- def auth_kind(provider: Optional[str]) -> str:
- """``"oauth"`` for providers whose credential is an OAuth/subscription grant
- (desktop Accounts tab), ``"api_key"`` for everything else."""
- try:
- from hermes_cli.provider_catalog import provider_catalog_by_slug
- descriptor = provider_catalog_by_slug().get((provider or "").strip().lower())
- return "oauth" if descriptor is not None and descriptor.tab == "accounts" else "api_key"
- except Exception: # pragma: no cover — advisory only
- return "api_key"
- def _disk_full(candidate: Any) -> bool:
- try:
- from hermes_state_errors import is_disk_full_error
- return bool(is_disk_full_error(candidate))
- except Exception: # pragma: no cover - defensive import guard
- return False
- def _result_layer(reason: str, error_text: str, provider: str) -> str:
- if reason in _REASON_TO_LAYER:
- return _REASON_TO_LAYER[reason]
- if reason in _TRANSPORT_REASONS and _is_custom_endpoint(provider):
- return LAYER_ENDPOINT
- return LAYER_STREAMING if _looks_like_stream_drop(error_text) else LAYER_PROVIDER
- def build_error_surface_from_result(result: Any, provider: str = "", model: str = "") -> Optional[dict]:
- """Descriptor for a returned-error turn result (``failed=True`` dicts).
- Uses the stamped ``failure_reason`` plus error text. None when the result
- carries no failure signal.
- """
- try:
- if not isinstance(result, dict):
- return None
- error_text = str(result.get("error") or "")
- reason = str(result.get("failure_reason") or "").strip()
- if not error_text and not reason:
- return None
- # Disk-full wins outright: the fix (free space) is unrelated to the
- # provider stack; hermes_state owns the pattern list.
- if error_text and _disk_full(error_text):
- return _surface(LAYER_DISK, "disk_full", False, provider, model)
- if result.get("billing_block") or reason in ("billing", "billing_unverified"):
- if window_message := str(result.get("usage_window_message") or ""):
- # A window plan's limit, not an empty account: the card shows the stop-line sentence.
- # Retry is the natural next click once Keep going / a top-up is on, and at the reset.
- surface = _surface(LAYER_BILLING, "usage_window", True, provider, model)
- surface["message"] = window_message
- if isinstance(resets_at := result.get("failure_resets_at"), (int, float)) and not isinstance(resets_at, bool):
- surface["resets_at"] = float(resets_at)
- if isinstance(limit := result.get("usage_limit"), dict):
- # What ran out, the plan's recommended models that still work, and the credits
- # option: the card's one-click buttons (agent/plan_limits.py).
- surface["usage_limit"] = limit
- return surface
- return _surface(LAYER_BILLING, reason or "billing", False, provider, model)
- # The Nous free tier refused or could not serve the turn (``agent/turn_recovery.py``
- # stamps ``free_tier``): its own code, so a client offers the free sign-in rather than an
- # OAuth re-login, and the chat sentence rides along as the card body.
- if isinstance(free_tier := result.get("free_tier"), dict) and free_tier.get("kind"):
- kind = str(free_tier["kind"])
- surface = _surface(LAYER_PROVIDER, f"free_tier_{kind}", kind in _FREE_TIER_RETRYABLE_KINDS, provider, model)
- if message := str(free_tier.get("message") or ""):
- surface["message"] = message
- return surface
- if not reason: # failed result without a classified reason (legacy paths)
- drop = _looks_like_stream_drop(error_text)
- return _surface(LAYER_STREAMING if drop else LAYER_PROVIDER, "stream_drop" if drop else "unknown", True, provider, model)
- # Prefer the classifier's own verdict (``failure_retryable``); the
- # reason-set fallback covers older results.
- retryable = result.get("failure_retryable")
- if not isinstance(retryable, bool):
- retryable = reason not in _NON_RETRYABLE_REASONS
- surface = _surface(_result_layer(reason, error_text, provider), reason, retryable, provider, model)
- # When the provider named the moment its limit lifts (Retry-After / ``resets_at``,
- # ``agent/turn_recovery.py::_stamp_limit_reset``) the card can say "Limit resets at HH:mm"
- # next to Retry instead of leaving the user to guess (#98852). Epoch seconds.
- if isinstance(resets_at := result.get("failure_resets_at"), (int, float)) and not isinstance(resets_at, bool):
- surface["resets_at"] = float(resets_at)
- return surface
- except Exception: # pragma: no cover — never break the error path
- logger.debug("error_surface: result classification failed", exc_info=True)
- return None
- def build_error_surface_from_exception(
- exc: BaseException, provider: str = "", model: str = "", api_key: Any = None,
- ) -> Optional[dict]:
- """Descriptor for an exception that escaped the turn dispatcher.
- API/transport exceptions go through ``classify_api_error`` (same taxonomy
- as the retry loop); anything else is a gateway-layer failure.
- """
- try:
- message = str(exc) or type(exc).__name__
- if _disk_full(exc):
- return _surface(LAYER_DISK, "disk_full", False, provider, model)
- api_like = (type(exc).__module__ or "").split(".")[0] in _API_EXC_MODULE_PREFIXES or hasattr(exc, "status_code")
- if not api_like or not isinstance(exc, Exception):
- return _surface(LAYER_GATEWAY, type(exc).__name__, True, provider, model)
- from agent.error_classifier import classify_api_error
- classified = classify_api_error(exc, provider=provider, model=model, api_key=api_key)
- synthetic = {"error": classified.message or message, "failure_reason": classified.reason.value}
- from agent.agent_runtime_helpers import extract_api_error_context
- from agent.credential_pool import _parse_absolute_timestamp
- if (resets_at := _parse_absolute_timestamp(extract_api_error_context(exc).get("reset_at"))) is not None:
- synthetic["failure_resets_at"] = resets_at
- surface = build_error_surface_from_result(synthetic, provider=provider, model=model)
- if surface is not None:
- surface["retryable"] = bool(classified.retryable)
- return surface
- except Exception: # pragma: no cover — never break the error path
- logger.debug("error_surface: exception classification failed", exc_info=True)
- return None
|