error_surface.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249
  1. """Structured error-surface descriptors for UI clients (Desktop/TUI).
  2. Maps the internal failure taxonomy (``FailoverReason`` values carried in turn
  3. results as ``failure_reason``, or raw exceptions from the turn dispatcher)
  4. onto a small, stable wire descriptor ``{"layer", "code", "retryable"}``.
  5. Layers: provider (model API rejected/failed), endpoint (user-configured
  6. custom/local endpoint transport failure), streaming (SSE dropped mid-turn),
  7. auth, billing (fallback signal; clients usually have a richer
  8. ``billing_block``), gateway (local runtime errored), disk (disk full).
  9. Dependency-light and NEVER raises: surfacing diagnostics must not break the
  10. error path it describes. Descriptors are advisory — clients fall back to
  11. string sniffing when absent or partial.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. from typing import Any, Optional
  16. logger = logging.getLogger(__name__)
  17. LAYER_PROVIDER = "provider"
  18. LAYER_ENDPOINT = "endpoint"
  19. LAYER_STREAMING = "streaming"
  20. LAYER_AUTH = "auth"
  21. LAYER_BILLING = "billing"
  22. LAYER_GATEWAY = "gateway"
  23. LAYER_DISK = "disk"
  24. # failure_reason → UI layer. Unlisted reasons fall back to LAYER_PROVIDER:
  25. # every FailoverReason comes from classifying a provider call. Loop-site codes
  26. # (agent/turn_failure_copy.py::SITE_FAILURE_CODES) are listed explicitly: the
  27. # ones that are not provider verdicts map to the gateway layer so the client
  28. # does not offer "Switch provider"; the ones the model/provider caused
  29. # (cut-off output, empty or broken reply) stay on the provider layer, where the
  30. # clients' per-code copy names the real fix (`continue`, smaller steps, /retry).
  31. _REASON_TO_LAYER = {
  32. "auth": LAYER_AUTH, "auth_permanent": LAYER_AUTH, "billing": LAYER_BILLING, "billing_unverified": LAYER_BILLING,
  33. "loop_error": LAYER_GATEWAY, "interpreter_shutdown": LAYER_GATEWAY, "session_busy": LAYER_GATEWAY,
  34. "gateway_connection": LAYER_GATEWAY, "host_error": LAYER_GATEWAY, "workspace_unavailable": LAYER_GATEWAY,
  35. "launch_credentials_unavailable": LAYER_GATEWAY,
  36. "local_runtime_unavailable": LAYER_ENDPOINT, # the managed local model server is down
  37. "truncated": LAYER_PROVIDER, "empty_response": LAYER_PROVIDER, "invalid_response": LAYER_PROVIDER,
  38. "context_overflow": LAYER_PROVIDER, # a bigger-window model IS the fix, so Switch provider applies
  39. }
  40. # Failures between us and the base_url (not a provider verdict); on a
  41. # custom/local endpoint they point at the user's endpoint config.
  42. _TRANSPORT_REASONS = {"timeout", "ssl_cert_verification"}
  43. # Free-tier kinds where a later send can succeed on its own (a wait, an outage clearing); the
  44. # rest need a sign-in or another provider.
  45. _FREE_TIER_RETRYABLE_KINDS = {"rate_limited", "at_capacity", "outage"}
  46. # Deterministic for the request — a bare "Retry" repeats the failure. Fallback
  47. # only: current backends stamp the classifier's verdict in ``failure_retryable``.
  48. # Kept in sync with ``classify_api_error``'s retryable=False verdicts.
  49. _NON_RETRYABLE_REASONS = {
  50. "auth", "auth_permanent", "billing", "billing_unverified", "content_policy_blocked",
  51. "provider_policy_blocked", "data_policy_denied", "model_not_found", "format_error", "ssl_cert_verification",
  52. "context_overflow", "interpreter_shutdown", "upstream_blocked",
  53. }
  54. # Providers whose base_url is user-supplied rather than a known vendor.
  55. _CUSTOM_ENDPOINT_PROVIDERS = {"custom", "local", "llama.cpp", "llamacpp", "ollama", "lmstudio", "vllm"}
  56. # Mid-stream drop markers. Deliberately narrow: our own retry-exhaustion
  57. # summaries plus the OpenAI SDK's stream-abort errors.
  58. _STREAM_DROP_FRAGMENTS = (
  59. "stream connection", "peer closed connection", "incomplete chunked read",
  60. "connection broken", "stream ended prematurely", "sse", "mid-stream",
  61. )
  62. # Exception top-level modules that mean "API/transport call failed" (vs. a bug
  63. # in our dispatcher = gateway layer): every SDK family our adapters raise from
  64. # plus raw transports.
  65. _API_EXC_MODULE_PREFIXES = (
  66. "openai", "httpx", "httpcore", "anthropic", "botocore", "boto3", "google",
  67. "grpc", "requests", "aiohttp", "ssl", "socket", "urllib",
  68. )
  69. def _is_custom_endpoint(provider: Optional[str]) -> bool:
  70. p = (provider or "").strip().lower()
  71. return p in _CUSTOM_ENDPOINT_PROVIDERS or p.startswith("custom:")
  72. def _looks_like_stream_drop(message: str) -> bool:
  73. return any(fragment in message.lower() for fragment in _STREAM_DROP_FRAGMENTS)
  74. def _surface(layer: str, code: str, retryable: bool, provider: str = "", model: str = "") -> dict:
  75. # Identity captured at classification time, so clients report the session
  76. # that actually failed — not whatever the composer points at later.
  77. identity = {k: v for k, v in (("provider", provider), ("model", model)) if v}
  78. surface = {"layer": layer, "code": code, "retryable": bool(retryable), **identity}
  79. if provider:
  80. # Clients name the failing provider in the card copy ("OpenCode Go did
  81. # not answer…"), not by its config slug.
  82. surface["provider_label"] = _provider_label(provider)
  83. if layer == LAYER_AUTH and provider:
  84. # OAuth providers are fixed by signing in again; API-key providers by
  85. # replacing the key. The client's one-click recovery needs to know which.
  86. surface["auth_kind"] = auth_kind(provider)
  87. if surface["auth_kind"] == "api_key" and (env_var := _api_key_env(provider)):
  88. surface["api_key_env"] = env_var
  89. return surface
  90. def _api_key_env(provider: str) -> str:
  91. """The env var holding ``provider``'s API key, so the client can open that row."""
  92. try:
  93. from hermes_cli.provider_catalog import provider_catalog_by_slug
  94. descriptor = provider_catalog_by_slug().get(provider.strip().lower())
  95. return descriptor.api_key_env_vars[0] if descriptor and descriptor.api_key_env_vars else ""
  96. except Exception: # pragma: no cover — advisory only
  97. return ""
  98. def _provider_label(provider: str) -> str:
  99. try:
  100. from hermes_cli.models import provider_label
  101. return provider_label(provider)
  102. except Exception: # pragma: no cover — advisory only
  103. return provider
  104. def auth_kind(provider: Optional[str]) -> str:
  105. """``"oauth"`` for providers whose credential is an OAuth/subscription grant
  106. (desktop Accounts tab), ``"api_key"`` for everything else."""
  107. try:
  108. from hermes_cli.provider_catalog import provider_catalog_by_slug
  109. descriptor = provider_catalog_by_slug().get((provider or "").strip().lower())
  110. return "oauth" if descriptor is not None and descriptor.tab == "accounts" else "api_key"
  111. except Exception: # pragma: no cover — advisory only
  112. return "api_key"
  113. def _disk_full(candidate: Any) -> bool:
  114. try:
  115. from hermes_state_errors import is_disk_full_error
  116. return bool(is_disk_full_error(candidate))
  117. except Exception: # pragma: no cover - defensive import guard
  118. return False
  119. def _result_layer(reason: str, error_text: str, provider: str) -> str:
  120. if reason in _REASON_TO_LAYER:
  121. return _REASON_TO_LAYER[reason]
  122. if reason in _TRANSPORT_REASONS and _is_custom_endpoint(provider):
  123. return LAYER_ENDPOINT
  124. return LAYER_STREAMING if _looks_like_stream_drop(error_text) else LAYER_PROVIDER
  125. def build_error_surface_from_result(result: Any, provider: str = "", model: str = "") -> Optional[dict]:
  126. """Descriptor for a returned-error turn result (``failed=True`` dicts).
  127. Uses the stamped ``failure_reason`` plus error text. None when the result
  128. carries no failure signal.
  129. """
  130. try:
  131. if not isinstance(result, dict):
  132. return None
  133. error_text = str(result.get("error") or "")
  134. reason = str(result.get("failure_reason") or "").strip()
  135. if not error_text and not reason:
  136. return None
  137. # Disk-full wins outright: the fix (free space) is unrelated to the
  138. # provider stack; hermes_state owns the pattern list.
  139. if error_text and _disk_full(error_text):
  140. return _surface(LAYER_DISK, "disk_full", False, provider, model)
  141. if result.get("billing_block") or reason in ("billing", "billing_unverified"):
  142. if window_message := str(result.get("usage_window_message") or ""):
  143. # A window plan's limit, not an empty account: the card shows the stop-line sentence.
  144. # Retry is the natural next click once Keep going / a top-up is on, and at the reset.
  145. surface = _surface(LAYER_BILLING, "usage_window", True, provider, model)
  146. surface["message"] = window_message
  147. if isinstance(resets_at := result.get("failure_resets_at"), (int, float)) and not isinstance(resets_at, bool):
  148. surface["resets_at"] = float(resets_at)
  149. if isinstance(limit := result.get("usage_limit"), dict):
  150. # What ran out, the plan's recommended models that still work, and the credits
  151. # option: the card's one-click buttons (agent/plan_limits.py).
  152. surface["usage_limit"] = limit
  153. return surface
  154. return _surface(LAYER_BILLING, reason or "billing", False, provider, model)
  155. # The Nous free tier refused or could not serve the turn (``agent/turn_recovery.py``
  156. # stamps ``free_tier``): its own code, so a client offers the free sign-in rather than an
  157. # OAuth re-login, and the chat sentence rides along as the card body.
  158. if isinstance(free_tier := result.get("free_tier"), dict) and free_tier.get("kind"):
  159. kind = str(free_tier["kind"])
  160. surface = _surface(LAYER_PROVIDER, f"free_tier_{kind}", kind in _FREE_TIER_RETRYABLE_KINDS, provider, model)
  161. if message := str(free_tier.get("message") or ""):
  162. surface["message"] = message
  163. return surface
  164. if not reason: # failed result without a classified reason (legacy paths)
  165. drop = _looks_like_stream_drop(error_text)
  166. return _surface(LAYER_STREAMING if drop else LAYER_PROVIDER, "stream_drop" if drop else "unknown", True, provider, model)
  167. # Prefer the classifier's own verdict (``failure_retryable``); the
  168. # reason-set fallback covers older results.
  169. retryable = result.get("failure_retryable")
  170. if not isinstance(retryable, bool):
  171. retryable = reason not in _NON_RETRYABLE_REASONS
  172. surface = _surface(_result_layer(reason, error_text, provider), reason, retryable, provider, model)
  173. # When the provider named the moment its limit lifts (Retry-After / ``resets_at``,
  174. # ``agent/turn_recovery.py::_stamp_limit_reset``) the card can say "Limit resets at HH:mm"
  175. # next to Retry instead of leaving the user to guess (#98852). Epoch seconds.
  176. if isinstance(resets_at := result.get("failure_resets_at"), (int, float)) and not isinstance(resets_at, bool):
  177. surface["resets_at"] = float(resets_at)
  178. return surface
  179. except Exception: # pragma: no cover — never break the error path
  180. logger.debug("error_surface: result classification failed", exc_info=True)
  181. return None
  182. def build_error_surface_from_exception(
  183. exc: BaseException, provider: str = "", model: str = "", api_key: Any = None,
  184. ) -> Optional[dict]:
  185. """Descriptor for an exception that escaped the turn dispatcher.
  186. API/transport exceptions go through ``classify_api_error`` (same taxonomy
  187. as the retry loop); anything else is a gateway-layer failure.
  188. """
  189. try:
  190. message = str(exc) or type(exc).__name__
  191. if _disk_full(exc):
  192. return _surface(LAYER_DISK, "disk_full", False, provider, model)
  193. api_like = (type(exc).__module__ or "").split(".")[0] in _API_EXC_MODULE_PREFIXES or hasattr(exc, "status_code")
  194. if not api_like or not isinstance(exc, Exception):
  195. return _surface(LAYER_GATEWAY, type(exc).__name__, True, provider, model)
  196. from agent.error_classifier import classify_api_error
  197. classified = classify_api_error(exc, provider=provider, model=model, api_key=api_key)
  198. synthetic = {"error": classified.message or message, "failure_reason": classified.reason.value}
  199. from agent.agent_runtime_helpers import extract_api_error_context
  200. from agent.credential_pool import _parse_absolute_timestamp
  201. if (resets_at := _parse_absolute_timestamp(extract_api_error_context(exc).get("reset_at"))) is not None:
  202. synthetic["failure_resets_at"] = resets_at
  203. surface = build_error_surface_from_result(synthetic, provider=provider, model=model)
  204. if surface is not None:
  205. surface["retryable"] = bool(classified.retryable)
  206. return surface
  207. except Exception: # pragma: no cover — never break the error path
  208. logger.debug("error_surface: exception classification failed", exc_info=True)
  209. return None