api_error_summary.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. """Provider API error summarising for ``AIAgent``.
  2. Entitlement-failure detection, xAI subscription decoration, structured-detail coercion, and log-safe
  3. redaction of provider error payloads.
  4. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
  5. """
  6. import functools
  7. import json
  8. import re
  9. from typing import Any, Dict, Optional
  10. from agent.redact import redact_sensitive_text
  11. # Substrings of the plain ``ValueError`` jiter (the openai/anthropic SDKs' SSE JSON parser)
  12. # raises for a truncated/corrupted event-stream frame — wire trouble, not local validation
  13. # (#65147). Serde-style vocabulary anchored on the "at line" suffix; classify through
  14. # ``is_provider_stream_parse_error`` rather than scanning this tuple directly.
  15. PROVIDER_STREAM_PARSE_MARKERS = (
  16. "expected ident at line",
  17. "expected value at line",
  18. "eof while parsing a value at line",
  19. "eof while parsing a string at line",
  20. "eof while parsing a list at line",
  21. "eof while parsing an object at line",
  22. "key must be a string at line",
  23. "trailing characters at line",
  24. "trailing comma at line",
  25. "expected `,` or `}` at line",
  26. "expected `,` or `]` at line",
  27. "expected `:` at line",
  28. "invalid escape at line",
  29. "invalid number at line",
  30. "found while parsing a string at line", # "control character (\u0000-\u001F) found while ..."
  31. )
  32. def is_provider_stream_parse_error(error: BaseException) -> bool:
  33. """True for a provider stream-parse ``ValueError`` (see ``PROVIDER_STREAM_PARSE_MARKERS``)."""
  34. return (isinstance(error, ValueError) and not isinstance(error, (UnicodeEncodeError, json.JSONDecodeError))
  35. and any(marker in str(error).lower() for marker in PROVIDER_STREAM_PARSE_MARKERS))
  36. # Offline DNS failures are wrapped in a generic "Connection error" by SDKs — inspect the chain.
  37. _NETWORK_RESOLUTION_MARKERS = (
  38. "temporary failure in name resolution",
  39. "name or service not known",
  40. "nodename nor servname provided, or not known",
  41. "getaddrinfo failed",
  42. "no address associated with hostname",
  43. "network is unreachable",
  44. )
  45. _XAI_ENTITLEMENT_HINT = (
  46. " — xAI rejected this OAuth account. NOTE: X Premium+ does NOT "
  47. "include xAI API access — only standalone SuperGrok subscribers "
  48. "can use this provider. Other possible causes: no Grok "
  49. "subscription, your tier doesn't include this model, or your "
  50. "quota is exhausted. Check https://grok.com/?_s=usage to see "
  51. "which, or run `/model` to switch providers."
  52. )
  53. _ERROR_DETAIL_KEYS = ("message", "detail", "error", "code", "type")
  54. def _is_xai_entitlement_text(lower: str) -> bool:
  55. """xAI's permission-denied body text for an unsubscribed / under-tiered / exhausted account."""
  56. return (
  57. "do not have an active grok subscription" in lower
  58. or ("out of available resources" in lower and "grok" in lower)
  59. or ("does not have permission" in lower and "grok" in lower)
  60. )
  61. def _http_prefix(error: Exception) -> str:
  62. status_code = getattr(error, "status_code", None)
  63. return f"HTTP {status_code}: " if status_code else ""
  64. class _BindsAgentProvider:
  65. """Static-style method that also receives ``provider=<agent.provider>`` when called on an agent.
  66. Class access (``ApiErrorSummaryMixin._summarize_api_error(err)``) returns the plain function, and
  67. callers keep the one-argument call shape, so one-argument stubs set on an instance still work.
  68. """
  69. def __init__(self, func):
  70. self._func = func
  71. def __get__(self, obj, objtype=None):
  72. if obj is None:
  73. return self._func
  74. return functools.partial(self._func, provider=getattr(obj, "provider", None))
  75. class ApiErrorSummaryMixin:
  76. """Provider error -> user/log-safe summary (see module docstring)."""
  77. @staticmethod
  78. def _is_entitlement_failure(
  79. error_context: Optional[dict[str, Any]], status_code: Optional[int]
  80. ) -> bool:
  81. """Detect subscription/entitlement 401/403s that masquerade as auth failures.
  82. Refreshing a token cannot fix an unsubscribed account, so callers surface the error instead of looping
  83. the pool. xAI returns the same permission-denied text for BOTH cases; a ``[WKE=unauthenticated:...]``
  84. suffix (or "access token could not be validated") means stale token → return False so the refresh path
  85. runs.
  86. Disambiguator for xAI (#29344): the same ``code`` text ("The caller does not have permission to
  87. execute the specified operation") is returned for BOTH an unsubscribed account AND a stale OAuth
  88. access token. xAI ships an explicit signal in the ``error`` field that tells the two apart: a
  89. ``[WKE=unauthenticated:...]`` suffix (and/or the ``OAuth2 access token could not be validated``
  90. phrasing) means the credentials failed validation — that's recoverable by refreshing the token, NOT
  91. by surfacing an entitlement message. When either signal is present we return False eagerly so the
  92. credential-pool refresh path runs, letting long-running TUI sessions recover from stale tokens
  93. without an exit/reopen cycle.
  94. """
  95. if status_code not in {401, 403, None}:
  96. return False
  97. if not isinstance(error_context, dict):
  98. return False
  99. # Single lowercase haystack over every field shape (message/reason and raw code/error).
  100. haystack = " ".join(
  101. str(error_context.get(k) or "").lower() for k in ("message", "reason", "code", "error")
  102. )
  103. if not haystack.strip():
  104. return False
  105. if "[wke=unauthenticated:" in haystack or "oauth2 access token could not be validated" in haystack:
  106. return False
  107. return _is_xai_entitlement_text(haystack)
  108. @staticmethod
  109. def _decorate_xai_entitlement_error(detail: str) -> str:
  110. """Append a neutral hint when xAI's OAuth surface returns the permission-denied 403.
  111. xAI's ``/v1/responses`` uses one body for several causes (no subscription, tier lacks the model, quota
  112. exhausted). The least obvious: X Premium+ does NOT include API access — only SuperGrok does. Lead with
  113. that, keep the raw text, point at https://grok.com/?_s=usage. Idempotent: a substring unique to the
  114. hint marks prior decoration.
  115. """
  116. if not detail or not _is_xai_entitlement_text(detail.lower()):
  117. return detail
  118. if "X Premium+ does NOT include" in detail:
  119. return detail
  120. return f"{detail}{_XAI_ENTITLEMENT_HINT}"
  121. @staticmethod
  122. def _coerce_api_error_detail(value: Any) -> str:
  123. """Return a display-safe string for structured provider error fields."""
  124. if isinstance(value, str):
  125. return value
  126. if isinstance(value, dict):
  127. for key in _ERROR_DETAIL_KEYS:
  128. nested = value.get(key)
  129. if isinstance(nested, str) and nested.strip():
  130. return nested
  131. for key in _ERROR_DETAIL_KEYS:
  132. if key in value:
  133. nested_detail = ApiErrorSummaryMixin._coerce_api_error_detail(value[key])
  134. if nested_detail:
  135. return nested_detail
  136. try:
  137. return json.dumps(value, ensure_ascii=False, sort_keys=True)
  138. except TypeError:
  139. return str(value)
  140. if isinstance(value, (list, tuple)):
  141. parts = [ApiErrorSummaryMixin._coerce_api_error_detail(item) for item in value]
  142. return "; ".join(part for part in parts if part)
  143. if value is None:
  144. return ""
  145. return str(value)
  146. @_BindsAgentProvider
  147. def _summarize_api_error(error: Exception, provider: Optional[str] = None) -> str:
  148. """Extract a human-readable one-liner from an API error.
  149. Cloudflare HTML pages → ``<title>``; network/DNS failures (even SDK-wrapped) → offline hint;
  150. transport timeouts → connect/read phase plus the setting to raise; else truncated str(error).
  151. """
  152. raw = str(error)
  153. current: Optional[BaseException] = error
  154. seen: set[int] = set()
  155. while current is not None and id(current) not in seen:
  156. seen.add(id(current))
  157. if any(marker in str(current).lower() for marker in _NETWORK_RESOLUTION_MARKERS):
  158. return (
  159. "Hermes can't reach the model provider. You may be offline. "
  160. "Check your internet connection and try again."
  161. )
  162. current = current.__cause__ or current.__context__
  163. from agent.timeout_error_summary import summarize_timeout_error
  164. timeout_summary = summarize_timeout_error(error, provider)
  165. if timeout_summary is not None:
  166. return timeout_summary
  167. if is_provider_stream_parse_error(error):
  168. return f"Malformed provider streaming response: {raw[:300]}"
  169. prefix = _http_prefix(error)
  170. # Cloudflare / proxy HTML pages: grab the <title> (and Ray ID) for a clean summary
  171. if "<!DOCTYPE" in raw or "<html" in raw:
  172. m = re.search(r"<title[^>]*>([^<]+)</title>", raw, re.IGNORECASE)
  173. title = m.group(1).strip() if m else "HTML error page (title not found)"
  174. ray = re.search(r"Cloudflare Ray ID:\s*<strong[^>]*>([^<]+)</strong>", raw)
  175. parts = [prefix[:-2]] if prefix else []
  176. parts.append(title)
  177. if ray:
  178. parts.append(f"Ray {ray.group(1).strip()}")
  179. return " — ".join(parts)
  180. # GeminiAPIError already composes a clean one-liner with guidance; don't re-extract the raw body.
  181. if type(error).__name__ == "GeminiAPIError":
  182. return redact_sensitive_text(raw[:1000])
  183. # JSON body errors from OpenAI/Anthropic SDKs
  184. body = getattr(error, "body", None)
  185. if isinstance(body, dict):
  186. msg = body.get("error", {}).get("message") if isinstance(body.get("error"), dict) else body.get("message")
  187. if msg:
  188. msg = ApiErrorSummaryMixin._coerce_api_error_detail(msg)
  189. return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}")
  190. # SDK may leave body empty while httpx has the payload. Redact: the body is attacker-influenced
  191. # and may echo Authorization / x-api-key / request JSON.
  192. # Redact before returning: the raw provider/proxy error body is attacker-influenced and may echo
  193. # Authorization / x-api-key / request JSON, which would otherwise leak into final_response + logs
  194. # (this path widens exposure vs the old empty-body "HTTP 400" string). See #36109.
  195. response = getattr(error, "response", None)
  196. if response is not None:
  197. try:
  198. snippet = (getattr(response, "text", None) or "").strip()
  199. except Exception:
  200. snippet = ""
  201. if snippet:
  202. try:
  203. payload = json.loads(snippet)
  204. except (json.JSONDecodeError, TypeError):
  205. payload = None
  206. if isinstance(payload, dict):
  207. err = payload.get("error")
  208. if isinstance(err, dict) and err.get("message"):
  209. return redact_sensitive_text(f"{prefix}{str(err['message'])[:300]}")
  210. if payload.get("message"):
  211. return redact_sensitive_text(f"{prefix}{str(payload['message'])[:300]}")
  212. return redact_sensitive_text(f"{prefix}{snippet[:300]}")
  213. # Fallback: truncate the raw string but give more room than 200 chars
  214. return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}")
  215. def _mask_api_key_for_logs(self, key: Any) -> Optional[str]:
  216. # Azure Foundry Entra ID bearer providers are callables — never invoke them in log
  217. # paths; identify the auth surface instead.
  218. if callable(key) and not isinstance(key, str):
  219. return "<entra-id-bearer>"
  220. if not key:
  221. return None
  222. if len(key) <= 12:
  223. return "***"
  224. return f"{key[:8]}...{key[-4:]}"
  225. def _clean_error_message(self, error_msg: str) -> str:
  226. """Clean up error messages for user display, removing HTML content and truncating."""
  227. if not error_msg:
  228. return "Unknown error"
  229. # HTML content is common with CloudFlare and gateway error pages
  230. if error_msg.strip().startswith('<!DOCTYPE html') or '<html' in error_msg:
  231. return "Service temporarily unavailable (HTML error page returned)"
  232. cleaned = ' '.join(error_msg.split())
  233. if len(cleaned) > 150:
  234. cleaned = cleaned[:150] + "..."
  235. return cleaned