error_classifier.py 92 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637
  1. """API error classification for smart failover and recovery.
  2. A priority-ordered pipeline maps an API exception to a ``ClassifiedError``
  3. whose recovery hints (retry, rotate credential, fallback, compress, abort) the
  4. retry loop in run_agent.py consults instead of re-matching strings itself.
  5. """
  6. from __future__ import annotations
  7. import enum
  8. import json
  9. import logging
  10. import re
  11. import time
  12. from dataclasses import dataclass, field
  13. from typing import Any, Callable, Dict, Iterator, Optional, Sequence
  14. logger = logging.getLogger(__name__)
  15. # Synthetic code for the OpenAI SDK rejecting a provider's SSE ``data:`` field
  16. # before any completion chunk arrives; distinct from generic JSON parse errors.
  17. PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data"
  18. # Same rejection with an EMPTY payload: the frame carried no ``data`` at all (``data:`` /
  19. # ``event: ping`` / ``id:`` with no content). Per the SSE spec those are legal keepalives /
  20. # no-ops, not malformed payloads — a degraded gateway answers every streaming request with
  21. # them, so the session switches to non-streaming instead of re-streaming into the same
  22. # window. See ``chat_completion_helpers._maybe_disable_streaming``.
  23. PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE = "provider_stream_empty_frame"
  24. # ── Error taxonomy ──────────────────────────────────────────────────────
  25. class FailoverReason(enum.Enum):
  26. """Why an API call failed — determines recovery strategy."""
  27. auth = "auth" # Transient auth (401/403) — refresh/rotate
  28. auth_permanent = "auth_permanent" # Auth failed after refresh — abort
  29. billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately
  30. rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate
  31. upstream_rate_limit = "upstream_rate_limit" # Aggregator's upstream model 429 — fallback model, key is healthy
  32. upstream_blocked = "upstream_blocked" # 403 from a WAF/CDN/proxy in front of the provider — key is healthy, fallback
  33. overloaded = "overloaded" # 503/529 — provider overloaded, backoff
  34. server_error = "server_error" # 500/502 — internal server error, retry
  35. timeout = "timeout" # Connection/read timeout — rebuild client + retry
  36. ssl_cert_verification = "ssl_cert_verification" # Deterministic TLS chain failure — fail fast with guidance
  37. context_overflow = "context_overflow" # Context too large — compress, not failover
  38. payload_too_large = "payload_too_large" # 413 — compress payload
  39. image_too_large = "image_too_large" # Native image part exceeds provider's per-image limit — shrink and retry
  40. image_corrupt = "image_corrupt" # Provider can't decode image bytes — strip and retry (shrinking won't help)
  41. model_not_found = "model_not_found" # 404 or invalid model — fallback to different model
  42. provider_policy_blocked = "provider_policy_blocked" # Aggregator account data/privacy policy excluded the only endpoint
  43. data_policy_denied = "data_policy_denied" # Context holds a connector result needing a no-training model; none serves it
  44. content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — deterministic per-request, don't retry unchanged
  45. model_entitlement = "model_entitlement" # This account cannot use the requested model — rotate credential (model-scoped), else fall back
  46. incomplete_response = "incomplete_response" # Codex/Responses turn stuck emitting reasoning only (no answer, no tool call) after replay + nudge — hand to a different provider
  47. format_error = "format_error" # 400 bad request — abort or strip + retry
  48. role_alternation = "role_alternation" # Strict chat template rejected adjacent same-role messages — merge them for this destination and retry
  49. invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry
  50. multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry
  51. reasoning_mandatory = "reasoning_mandatory" # Route rejects reasoning: {enabled: false} — send the disable no more this session and retry
  52. # Provider-specific
  53. thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid
  54. long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate
  55. oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth rejects 1M beta — disable beta and retry
  56. llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar rejects regex `pattern`/`format` — strip from tools and retry
  57. unknown = "unknown" # Unclassifiable — retry with backoff
  58. @dataclass
  59. class ClassifiedError:
  60. """Structured classification of an API error with recovery hints."""
  61. reason: FailoverReason
  62. status_code: Optional[int] = None
  63. provider: Optional[str] = None
  64. model: Optional[str] = None
  65. message: str = ""
  66. error_context: dict[str, Any] = field(default_factory=dict)
  67. # Recovery hints — the retry loop checks these instead of re-classifying.
  68. retryable: bool = True
  69. should_compress: bool = False
  70. should_rotate_credential: bool = False
  71. should_fallback: bool = False
  72. @property
  73. def is_auth(self) -> bool:
  74. return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
  75. @property
  76. def billing_unverified(self) -> bool:
  77. """True when a ``billing`` verdict rests on an ambiguous body (#82154)."""
  78. return bool(self.error_context.get("billing_unverified"))
  79. # ── Pattern tables (lowercased substrings) ──────────────────────────────
  80. # Billing exhaustion (not transient rate limit). "out of extra usage" is the
  81. # Anthropic OAuth Pro/Max overage bucket depleted (HTTP 400).
  82. # The Nous gateway's own words for "the free tier will not serve this" — a billing wall for a
  83. # named account, the tier refusing for an anonymous one (see ``_WELCOME_403_NAMED_PATTERNS``).
  84. _FREE_TIER_REFUSAL_PATTERNS = ("model_not_supported_on_free_tier", "not available on the free tier")
  85. _BILLING_PATTERNS = (
  86. "insufficient credits", "insufficient_quota", "insufficient balance", "credit balance",
  87. "credits exhausted", "credits have been exhausted", "requires available credits",
  88. "account balance is too low", "no usable credits", "top up your credits", "payment required",
  89. "billing hard limit", "exceeded your current quota", "account is deactivated", "plan does not include",
  90. "out of extra usage", "out of funds", "run out of funds", "balance_depleted",
  91. # OpenRouter org-level monthly cap arrives as 403 "Budget limit exceeded (monthly limit)" (#107166):
  92. # account exhaustion, not a credential problem.
  93. "budget limit exceeded",
  94. *_FREE_TIER_REFUSAL_PATTERNS,
  95. # LiteLLM proxies word a hard cap as "hard billing limit" (structured twin:
  96. # ``terminal_quota_exhausted`` in _BILLING_ERROR_CODES). "terminal billing
  97. # limit" free text is NOT matched: substring rules can't negate the
  98. # "non-terminal billing limit" wording, and the structured code covers it.
  99. "hard billing limit",
  100. )
  101. # Not proof of exhaustion: Anthropic returns the same "out of extra usage" body
  102. # for a content-filter rejection (#82154). Verdict stays ``billing`` but is
  103. # marked unverified so surfaces hedge and the pool uses a short cooldown.
  104. _UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",)
  105. # xAI's Grok credit-exhaustion code arrives as HTTP 403, not 402. Provider-
  106. # scoped on purpose: other providers' billing codes on a 403 stay auth failures.
  107. _XAI_SPENDING_LIMIT_ERROR_CODE = "personal-team-blocked:spending-limit"
  108. # Nous usage-window plan: included usage (or the large-model allowance) is spent AND
  109. # there are no top-up credits. Billing so the pool may rotate to a key the user
  110. # owns; the message names the reset, so it must never read as transient. Honoured
  111. # only on a Nous 402 (``_classify_402`` behind ``_is_nous_route``), never via
  112. # _BILLING_ERROR_CODES, so any other status or upstream keeps main's rules.
  113. _NOUS_USAGE_WINDOW_402_CODE = "usage_window_and_credits_exhausted"
  114. # Structured codes meaning the account cannot serve paid traffic.
  115. _BILLING_ERROR_CODES = frozenset({
  116. "insufficient_quota", "billing_not_active", "payment_required", "insufficient_credits",
  117. "no_usable_credits", "balance_depleted", "model_not_supported_on_free_tier",
  118. "member_spend_cap_exceeded", "terminal_quota_exhausted", _XAI_SPENDING_LIMIT_ERROR_CODE,
  119. # OpenAI (and OpenAI-compatible aggregators) spend/usage-limit family:
  120. # a credit balance or an org/project spend or usage cap is exhausted —
  121. # terminal for this credential until limits are raised.
  122. "credit_balance_exhausted", "organization_spend_limit_exceeded",
  123. "organization_usage_limit_exceeded", "project_spend_limit_exceeded",
  124. # Nous paid model behind an empty credit balance arrives as a 404 (#115702).
  125. "insufficient_credits_for_paid_model",
  126. })
  127. # Transient rate limiting. Bedrock "Throttling error: Too many tokens" also
  128. # contains an overflow phrase; rate limit is matched first so throttle wins.
  129. _RATE_LIMIT_PATTERNS = (
  130. "rate limit", "rate_limit", "too many requests", "throttled", "requests per minute",
  131. "tokens per minute", "requests per day", "try again in", "please retry after",
  132. "resource exhausted", "resource_exhausted", "resource-exhausted", "resourceexhausted",
  133. "rate increased too quickly", "throttlingexception", "too many concurrent requests",
  134. "servicequotaexceededexception", "throttling",
  135. )
  136. # Server busy, credential fine: back off on the same key, never rotate. Z.AI/
  137. # Zhipu reuse HTTP 429 for this, so the 429 path checks these first. Kept narrow
  138. # so a plain "you have been rate-limited" doesn't land here. (#14038, #15297)
  139. _OVERLOADED_PATTERNS = (
  140. "overloaded", "temporarily overloaded", "service is temporarily overloaded",
  141. "service may be temporarily overloaded", "server is overloaded", "server overloaded",
  142. "server overload", "server_overload",
  143. "service overloaded", "service is overloaded", "upstream overloaded", "currently overloaded",
  144. # CommandCode's 429 body for an unavailable upstream model — the key is healthy (#117111).
  145. "upstream model provider is temporarily unavailable. please try again in a moment.",
  146. "at capacity", "over capacity",
  147. )
  148. # Usage-limit patterns that need disambiguation (billing OR rate_limit), and
  149. # the signals that mark such a limit as transient (periodic quota, not billing).
  150. _USAGE_LIMIT_PATTERNS = ("usage limit", "quota", "limit exceeded", "key limit exceeded")
  151. _USAGE_LIMIT_TRANSIENT_SIGNALS = (
  152. "try again", "retry", "resets at", "reset in", "resets in", "reset after", "available in",
  153. "wait", "requests remaining", "periodic", "window", "per minute", "per second",
  154. )
  155. # 413 detected from message text (proxies embed the status or re-wrap
  156. # Anthropic's "request_too_large" type without one).
  157. _PAYLOAD_TOO_LARGE_PATTERNS = (
  158. "request entity too large", "payload too large", "error code: 413", "request_too_large",
  159. # Normally arrives with an HTTP 413 status (handled by the status path), but aggregators/proxies can
  160. # re-wrap it into a plain message with no status attribute — route it to the same compression recovery.
  161. # (port of anomalyco/opencode#37848)
  162. "request exceeds the maximum size",
  163. )
  164. # Request-BODY byte caps that byte-limited gateways report as a 400 (not 413) in wordings the exact list
  165. # above misses: "RequestSize(bytes): 34021227, Limit(bytes): 33554432", "request body too large, must be
  166. # less than 20MB" (Novita), "Request size exceeds the limit of {0}" (Z.AI). Without a match they fell to
  167. # format_error or token-overflow, and an image-heavy session (large in bytes, small in tokens) re-sent
  168. # the same oversized history. A subject AND an overflow signal are both required so field-level length
  169. # complaints ("tools[0].description content length exceeds maximum allowed") stay out; a
  170. # response-scoped subject never matches, because shrinking the request cannot fix a large response.
  171. _BYTE_SIZE_SUBJECT = re.compile(
  172. # The overflow verb must follow "... size" directly, so a parameter named there ("request size
  173. # parameter page_size exceeds the maximum of 100") is not the body.
  174. r"(?<!response )\b(?:(?:request ?size|(?:request )?(?:body|payload) size)(?: is)? (?:exceed|too large|too big)"
  175. r"|request body(?: is)? too large|(?:request )?payload(?: is)? too large)"
  176. )
  177. _BYTE_SIZE_OVERFLOW = re.compile(r"exceed|too large|too big")
  178. # A size subject that counts tokens is a token cap ("Request body too large for gpt-4.1 model. Max size:
  179. # 8000 tokens." from GitHub Models / Azure AI Inference), so it keeps the token rules; only the
  180. # context-overflow path adopts a quoted limit, and the byte path would compress against the old window.
  181. _TOKEN_COUNT = re.compile(r"\btokens?\b")
  182. _REQUEST_WORDS = {"request", "requestsize", "byte", "bytes"}
  183. def _is_byte_size_limit_message(error_msg: str) -> bool:
  184. """True when ``error_msg`` reports the request body over a byte-size cap (see above)."""
  185. msg = (error_msg or "").lower()
  186. if not _BYTE_SIZE_OVERFLOW.search(msg) or _TOKEN_COUNT.search(msg):
  187. return False
  188. if _BYTE_SIZE_SUBJECT.search(msg):
  189. return True
  190. # "content length" alone is a field-length complaint unless the request or bytes are named. "_" stays
  191. # inside a word: the SDK's str() carries the body, whose "invalid_request_error" type is not "request".
  192. content_length = "content length" in msg or "content-length" in msg
  193. return content_length and bool(_REQUEST_WORDS & set(re.split(r"[^a-z0-9_]+", msg)))
  194. # Per-image size/dimension 400s (Anthropic 5 MB / 8000 px; MiniMax "media
  195. # exceeds size limit" #76039) — a specific 400 before the request hits 413. A
  196. # non-image media hit is harmless: the shrink pass finds no image parts.
  197. # "patches after processing": OpenAI Codex Responses rejects an image whose
  198. # tile-patch budget (ceil(w/32)×ceil(h/32)) exceeds its 30000-patch ceiling
  199. # with wording that names no image-size vocabulary — without this pattern it
  200. # fell to format_error (non-retryable), bypassing the shrink recovery (#106337).
  201. # Byte caps enforced with a 400 instead of a 413 (#112473): NVIDIA NIM caps the whole
  202. # payload ("Please make sure your payload is below 26214400 bytes in size"); Alibaba
  203. # DashScope caps the base64 image string via Jackson ("String value length (N) exceeds the
  204. # maximum allowed (M, from `StreamReadConstraints.getMaxStringLength()`)"). Only an inline
  205. # image reaches those sizes, so shrinking is the recovery; the method-scoped Jackson token
  206. # is used because the bare class name also appears when Jackson caps a *token* length.
  207. _IMAGE_TOO_LARGE_PATTERNS = (
  208. "image exceeds", "image too large", "image_too_large", "image size exceeds", "image dimensions exceed",
  209. "dimensions exceed max allowed size", "max allowed size: 8000", "media exceeds", "media too large",
  210. "patches after processing", "make sure your payload is below", "streamreadconstraints.getmaxstringlength",
  211. )
  212. # Undecodable image bytes → strip-and-retry, never shrink. xAI wordings
  213. # (#69078); the last is the full sentence because shorter fragments also match
  214. # non-image download failures.
  215. _IMAGE_CORRUPT_PATTERNS = (
  216. "invalid png image", "invalid jpeg image", "base64 string of provided image cannot be decoded",
  217. "downloaded response does not contain a valid jpg, png, webp, or ico image",
  218. )
  219. # 400s rejecting list-type ``content`` in tool messages (Xiaomi MiMo "text is
  220. # not set", Alibaba, OpenAI-compat long tail). Recovery: strip image parts from
  221. # tool messages, remember (provider, model), retry. (#27344)
  222. # NVIDIA NIM's Rust gateway never names the field: its serde rejection says the
  223. # body "did not match any variant of untagged enum
  224. # ChatCompletionRequestToolMessageContent", which is the same list-type tool
  225. # content that every other wording here describes (#111231).
  226. _MULTIMODAL_TOOL_CONTENT_PATTERNS = (
  227. "text is not set", "tool message content must be a string", "tool content must be a string",
  228. "tool message must be a string", "expected string, got list", "expected string, got array",
  229. # Console Go / pydantic-v2 relays behind opencode-go (422, param ``messages.N.tool.content.str``, #104731).
  230. "tool_call.content must be string", "tool.content.str", "input should be a valid string",
  231. "chatcompletionrequesttoolmessagecontent",
  232. )
  233. # Local-inference memory/resource-ceiling rejections (oMLX/MLX memory guard,
  234. # llama.cpp/vLLM OOM, Metal/CUDA allocation ceilings). The server aborts on a
  235. # prefill memory PEAK, not a window limit, yet its remediation tail says
  236. # "reduce context length" — so without this list the request routes into
  237. # compression, which cannot lower a prefill peak: it burns the compression
  238. # budget, re-hits the wedged server each attempt and ends in a session reset.
  239. # Every token names memory/allocation in BYTES, never a token count, so the
  240. # list is disjoint from _CONTEXT_OVERFLOW_PATTERNS. Must be checked BEFORE
  241. # both overflow AND the usage-limit disambiguation ("memory limit exceeded"
  242. # contains "limit exceeded", which would otherwise read as billing). oMLX
  243. # reworded the accounting sentence in 0.5.7 ("predicted peak would require /
  244. # exceed"); the 0.5.6 wording is still in the field, so both stay. (#52261)
  245. _MEMORY_CEILING_PATTERNS = (
  246. "memory guard", "memory limit exceeded", "memory_guard_tier", "dynamic ceiling",
  247. "memory ceiling", "available memory", "out of memory", "insufficient memory",
  248. "prefill would require", "predicted peak would", "prefill safety cap", "metal_cap",
  249. )
  250. # Structured codes identifying the same rejection at the source, before an
  251. # OpenAI-compatible proxy flattens the body and drops the wording.
  252. _MEMORY_CEILING_ERROR_CODES = frozenset({
  253. "prefill_memory_exceeded", "prefill_memory_aborted", "omlx_prefill_memory_exceeded",
  254. })
  255. # Bare "max_tokens" is load-bearing: the output-cap-retry path keys off it;
  256. # empty-response advisories mentioning it are intercepted earlier. Groups:
  257. # generic; vLLM; Ollama; llama.cpp; Chinese; Z.AI (1210); Bedrock; Together.
  258. _CONTEXT_OVERFLOW_PATTERNS = (
  259. "context length", "context size", "maximum context", "token limit", "too many tokens",
  260. "reduce the length", "exceeds the limit", "context window", "prompt is too long",
  261. "prompt exceeds max length", "max_tokens", "maximum number of tokens",
  262. "exceeds the max_model_len", "max_model_len", "prompt length", "input is too long", "maximum model length",
  263. "context length exceeded", "truncating input",
  264. "slot context", "n_ctx_slot",
  265. "超过最大长度", "上下文长度",
  266. "tokens in request more than max tokens allowed",
  267. "input is too long", "max input token", "input token", "exceeds the maximum number of input tokens",
  268. # Together/Fireworks-style: "Input length 131393 exceeds the maximum allowed input length of 131040
  269. # tokens." No other pattern in this list matches that wording. (port of anomalyco/opencode#37848)
  270. "maximum allowed input length",
  271. )
  272. # Last entry: OpenRouter 404 when no endpoint supports tool calling —
  273. # model_not_found triggers fallback instead of burning retries (#58446).
  274. # Codex ChatGPT-account entitlement 400 — the account can never use the named slug (#71970, #106475).
  275. CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER = "model is not supported when using codex with a chatgpt account"
  276. _MODEL_NOT_FOUND_PATTERNS = (
  277. "is not a valid model", "invalid model", "model not found", "model_not_found", "does not exist",
  278. "no such model", "unknown model", "unsupported model", "no endpoints found that support tool use",
  279. # Nous 404 when the provider retires a :free route — the slug is dead for every
  280. # credential, so fall back instead of burning retries (#123180). Unlike the free-tier
  281. # billing wording, the account's balance/tier is not what rejected the call.
  282. "is no longer free",
  283. )
  284. # Qwen/vLLM chat-template "No user query found". Shared by the invalid-body
  285. # table (→ format_error) and the llama.cpp grammar guard so they cannot drift.
  286. _NO_USER_QUERY_SIGNAL = "no user query found"
  287. # Deterministic rejections of the *transcript* (e.g. a content-less assistant
  288. # stub after a dead stream). NOT overflow — input may be tiny and compression
  289. # cannot invent a missing turn — so fail fast as format_error.
  290. _INVALID_MESSAGE_BODY_PATTERNS = (
  291. "must have non-empty content", "messages must have non-empty", "invalid_request_body",
  292. "text content blocks must be non-empty", "content field is required",
  293. "messages: at least one message is required", _NO_USER_QUERY_SIGNAL,
  294. )
  295. # Strict-alternation chat templates (llama.cpp / vLLM Jinja templates, Mistral, some
  296. # OpenRouter routes) 400 when two adjacent messages share a role. Deterministic for the
  297. # request shape, and the only bad thing is the adjacency, so the caller that produced it
  298. # (the MoA aggregator appends ``user(guidance)`` after ``user(task)`` on iteration 1 —
  299. # #112358) merges the pair for THAT destination and retries once. Checked before the
  300. # request-validation table: the body usually also carries ``invalid_request_error``.
  301. _ROLE_ALTERNATION_PATTERNS = (
  302. "roles must alternate", "role must alternate", "must alternate between",
  303. "consecutive user messages", "consecutive messages with the same role",
  304. "consecutive messages of the same role", "same role in a row", "multiple user messages in a row",
  305. "adjacent messages with the same role",
  306. )
  307. # Proxy-side rejection of the model's own tool-call JSON (Ollama "invalid tool call arguments",
  308. # OpenRouter-wrapped "function_call arguments"). Checked before the generic 400 validation and
  309. # overflow heuristics: on a large session the bare message would otherwise read as overflow.
  310. _MALFORMED_TOOL_ARGS_PATTERNS = (
  311. "invalid tool call arguments", "invalid tool_call arguments", "invalid tool_calls arguments",
  312. "invalid function call arguments", "invalid function_call arguments",
  313. "tool call arguments are invalid", "tool_call arguments are invalid",
  314. "function call arguments are invalid", "function_call arguments are invalid",
  315. )
  316. # Malformed request, identical on every retry. Some gateways (codex.nekos.me)
  317. # return these as 5xx, so the 5xx path also checks them.
  318. _REQUEST_VALIDATION_PATTERNS = (
  319. "unknown parameter", "unsupported parameter", "unrecognized request argument",
  320. "invalid_request_error", "unknown_parameter", "unsupported_parameter",
  321. )
  322. # Parameters Hermes sends on SOME routes only → hosts where that is deliberate.
  323. # A rejection from any other host means the provider's gateway injected the
  324. # field itself: a server-side flake, not our request shape. prompt_cache_retention
  325. # is only sent for api.meta.ai / bedrock-mantle (agent/transports/codex.py).
  326. _SERVER_INJECTED_PARAM_SENDERS: dict[str, tuple] = {
  327. "prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
  328. }
  329. _PARAM_REJECTION_WORDS = ("not supported", "unsupported", "unknown", "unrecognized")
  330. # Anthropic thinking-block 400 wordings (see _provider_special_cases).
  331. _THINKING_MUTATION_WORDS = ("signature", "cannot be modified", "must remain as they were")
  332. def _is_anthropic_thinking_rejection(msg: str) -> bool:
  333. """A thinking block the upstream will not verify: a signature broken by transcript mutation, or a
  334. ``redacted_thinking`` blob it cannot open (another model's sealed reasoning after a /model switch or
  335. fallback: ``Invalid `data` in `redacted_thinking` block``). Both heal by stripping the replayed blocks."""
  336. return "thinking" in msg and (
  337. any(p in msg for p in _THINKING_MUTATION_WORDS) or ("redacted_thinking" in msg and "invalid" in msg))
  338. # Local MoA streaming adapter-shape bugs (see _moa_special_cases).
  339. _MOA_ADAPTER_SHAPE_BUGS = (
  340. "'types.SimpleNamespace' object is not iterable", "'types.SimpleNamespace' object has no attribute 'index'",
  341. )
  342. # OpenRouter 404 when the account data policy excludes the only endpoint. Not
  343. # model_not_found: the model exists, fallback can't help, body has the fix URL.
  344. _PROVIDER_POLICY_BLOCKED_PATTERNS = (
  345. "no endpoints available matching your guardrail", "no endpoints available matching your data policy",
  346. "no endpoints found matching your data policy",
  347. )
  348. # Upstream account ban relayed by an aggregator, often as HTTP 200 + an SSE error
  349. # event (no status): permanent for this account, so never the transient retry ladder.
  350. _ACCOUNT_POLICY_BLOCK_PATTERNS = ("blocked for a previous policy violation",)
  351. # Per-prompt safety-filter blocks: deterministic for the unchanged request, so
  352. # fallback immediately. Each phrase is verbatim from one provider (Codex cyber
  353. # flags #18028, OpenAI moderation, Anthropic safety, Azure token, MiniMax
  354. # #32421, CommandCode gateway moderation #115218, Alibaba Model Studio
  355. # guardrail `data_inspection_failed`, input and output side) — never a generic
  356. # word like "policy" that collides with billing/auth.
  357. # "content_filter" deliberately excludes the space variant seen in echoed config.
  358. _CONTENT_POLICY_BLOCKED_PATTERNS = (
  359. "flagged for possible cybersecurity risk", "trusted access for cyber",
  360. "violates our usage policies", "violates openai's usage policies", "your request was flagged by",
  361. "prompt was flagged by our safety", "responses cannot be generated due to safety",
  362. "content_filter", "responsibleaipolicyviolation", "new_sensitive",
  363. "content exists risk", "data may contain inappropriate content", "data_inspection_failed",
  364. )
  365. # Auth patterns (non-status-code signals).
  366. _AUTH_PATTERNS = (
  367. "invalid api key", "invalid_api_key", "gateway_auth_failed", "authentication", "unauthorized",
  368. "forbidden", "invalid token", "token expired", "token revoked", "access denied",
  369. # Codex backend rejecting an OAuth access token without a usable
  370. # ``chatgpt_account_id`` claim; arrives as a bare ``detail`` string.
  371. "failed to extract accountid from token",
  372. )
  373. # Empty-response advisories (OpenRouter / nano-gpt). Checked before overflow
  374. # because the text often mentions "max_tokens" (caused compression spirals).
  375. _EMPTY_PROVIDER_RESPONSE_PATTERNS = (
  376. "returned an empty response", "empty response despite retries", "provider returned an empty response",
  377. "model returning empty responses", "empty response stream",
  378. )
  379. # Timeout wording from generic exception types the type heuristics would miss.
  380. _TIMEOUT_MESSAGE_PATTERNS = (
  381. "timed out", "turn timed out", "request timed out", "deadline exceeded", "operation timed out",
  382. "upstream timed out",
  383. )
  384. # Connect/DNS failures from generic exception types with no status. EXCLUDES
  385. # mid-stream disconnects (_SERVER_DISCONNECT_PATTERNS may route large sessions
  386. # to compression; a never-established connection cannot be an overflow).
  387. # Groups: TCP connect; DNS (Python/glibc/macOS/Node); undici bridge; Envoy.
  388. _CONNECTION_MESSAGE_PATTERNS = (
  389. "connection refused", "econnrefused", "no route to host", "network is unreachable", "network unreachable",
  390. "name or service not known", "temporary failure in name resolution", "nodename nor servname provided",
  391. "getaddrinfo failed", "getaddrinfo enotfound", "eai_again",
  392. "fetch failed", "failed to fetch",
  393. "upstream connect error",
  394. )
  395. # SSL names keep provider-wrapped SSL errors (chain lost) as transport, not
  396. # unknown; OpenAI SDK errors are not subclasses of Python builtins.
  397. _TRANSPORT_ERROR_TYPES = frozenset({
  398. "ReadTimeout", "ConnectTimeout", "PoolTimeout", "ConnectError", "RemoteProtocolError",
  399. "ConnectionError", "ConnectionResetError", "ConnectionAbortedError", "BrokenPipeError",
  400. "TimeoutError", "ReadError", "ServerDisconnectedError",
  401. "SSLError", "SSLZeroReturnError", "SSLWantReadError", "SSLWantWriteError", "SSLEOFError", "SSLSyscallError",
  402. "APIConnectionError", "APITimeoutError",
  403. })
  404. # Ambiguous disconnects (no status): transient hiccup OR a gateway dropping an
  405. # oversized request. A large session + one of these → context-overflow path.
  406. _SERVER_DISCONNECT_PATTERNS = (
  407. "server disconnected", "peer closed connection", "connection reset by peer", "connection was closed",
  408. "network connection lost", "unexpected eof", "incomplete chunked read",
  409. )
  410. # Deterministic cert failures (proxy, missing CA, expired/self-signed) — fail
  411. # fast. Checked BEFORE _SSL_TRANSIENT_PATTERNS: these also contain "[SSL:".
  412. _SSL_CERT_VERIFY_PATTERNS = (
  413. "certificate verify failed", "certificate_verify_failed", "unable to get local issuer certificate",
  414. "self-signed certificate", "self signed certificate", "certificate has expired",
  415. "hostname mismatch, certificate is not valid", "unable to verify the first certificate",
  416. )
  417. # Transient SSL alerts: retry but NOT compression (kept apart from disconnects).
  418. # Both space and underscore forms because OpenSSL 3 changed token separators
  419. # (SSLV3_ALERT_... → SSL/TLS_ALERT_...); "[ssl:" is the Python ssl prefix.
  420. _SSL_TRANSIENT_PATTERNS = (
  421. "bad record mac", "ssl alert", "tls alert", "ssl handshake failure", "tlsv1 alert", "sslv3 alert",
  422. "bad_record_mac", "ssl_alert", "tls_alert", "tls_alert_internal_error", "[ssl:",
  423. )
  424. # A 403 body written by a WAF/CDN/proxy rather than the provider's API: Cloudflare's browser
  425. # challenge and block pages, plus the plain-text block relays return when they reject the SDK
  426. # User-Agent (#53099). Matched only on 403 (see ``_status_403``); a bare "access denied" or
  427. # "forbidden" stays auth because providers word real permission errors that way too.
  428. _UPSTREAM_BLOCKED_PATTERNS = (
  429. "your request was blocked", "request blocked", "sorry, you have been blocked",
  430. "enable javascript and cookies to continue", "cdn-cgi/challenge-platform", "cf-browser-verification",
  431. "challenge-error-text", "__cf_chl", "cf-error-details", "attention required! | cloudflare",
  432. )
  433. # ── Verdicts and rule tables ────────────────────────────────────────────
  434. # A verdict is the ClassifiedError kwargs a stage decided on: ``reason`` plus
  435. # hint overrides (unlisted hints keep dataclass defaults). Rule tables are
  436. # ordered ``(patterns, verdict)`` pairs matched first-hit; ``verdict`` may be
  437. # a callable of the error message.
  438. Verdict = dict[str, Any]
  439. def _v(reason: FailoverReason, **hints: Any) -> Verdict:
  440. return {"reason": reason, **hints}
  441. _ROTATE_FALLBACK = {"should_rotate_credential": True, "should_fallback": True}
  442. _ABORT_FALLBACK = {"retryable": False, "should_fallback": True}
  443. _R = FailoverReason
  444. _V_BILLING = _v(_R.billing, retryable=False, **_ROTATE_FALLBACK)
  445. _V_RATE_LIMIT = _v(_R.rate_limit, **_ROTATE_FALLBACK)
  446. _V_AUTH_ROTATE = _v(_R.auth, retryable=False, **_ROTATE_FALLBACK)
  447. _V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK)
  448. _V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK)
  449. _V_UPSTREAM_BLOCKED = _v(_R.upstream_blocked, **_ABORT_FALLBACK)
  450. _V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK)
  451. # Another account in the same pool may hold the entitlement; the credential itself is healthy.
  452. _V_MODEL_ENTITLEMENT = _v(_R.model_entitlement, retryable=False, **_ROTATE_FALLBACK)
  453. _V_FORMAT_ERROR = _v(_R.format_error, **_ABORT_FALLBACK)
  454. # A different provider (direct instead of the aggregator; another host's TLS chain) can fix these.
  455. _V_POLICY_BLOCKED = _v(_R.provider_policy_blocked, **_ABORT_FALLBACK)
  456. _V_SSL_CERT = _v(_R.ssl_cert_verification, **_ABORT_FALLBACK)
  457. _V_CONTEXT_OVERFLOW = _v(_R.context_overflow, should_compress=True)
  458. _V_PAYLOAD_TOO_LARGE = _v(_R.payload_too_large, should_compress=True)
  459. _V_OVERLOADED, _V_SERVER_ERROR, _V_TIMEOUT, _V_UNKNOWN = map(_v, (_R.overloaded, _R.server_error, _R.timeout, _R.unknown))
  460. _V_IMAGE_TOO_LARGE, _V_IMAGE_CORRUPT = _v(_R.image_too_large), _v(_R.image_corrupt)
  461. _V_MULTIMODAL, _V_INVALID_ENCRYPTED = _v(_R.multimodal_tool_content_unsupported), _v(_R.invalid_encrypted_content)
  462. _V_REASONING_MANDATORY = _v(_R.reasoning_mandatory, should_compress=False, should_fallback=False)
  463. # Same recovery hints as format_error: consumers without a merge-and-retry step (the main loop
  464. # already merges adjacent users before the call) keep aborting to the fallback chain.
  465. _V_ROLE_ALTERNATION = _v(_R.role_alternation, **_ABORT_FALLBACK)
  466. # The MODEL emitted unparseable tool-call JSON and the proxy (Ollama, OpenRouter) rejected it: no
  467. # other provider can fix that output, so falling back only replays the same broken turn 4-5 times
  468. # (20-60s per occurrence, #12770). Abort this call; the loop's argument repair handles the retry.
  469. _V_MALFORMED_TOOL_ARGS = _v(_R.format_error, retryable=False, should_fallback=False)
  470. # A reasoning-mandatory route answering ``reasoning: {enabled: false}`` (Nous Portal + OpenRouter wording).
  471. _REASONING_MANDATORY_PATTERN = "reasoning is mandatory"
  472. # Generic markers a provider 400 puts next to the offending parameter name. Bedrock Converse
  473. # rejects sampling params for reasoning-first models with the contraction ("This model doesn't
  474. # support the temperature field", xAI Grok) and inference-profile Claude with "`temperature` is
  475. # deprecated for this model" (#111043); strict pydantic gateways (Fireworks) name the unknown
  476. # field as "extra inputs are not permitted" (#109774). Enum-rejecting aggregators (commandcode.ai)
  477. # say "Invalid option: expected one of ..." with no "unsupported" anywhere, naming the field only
  478. # in the structured 'param' tail (#115277). Shared with the auxiliary retry ladder
  479. # (``agent.auxiliary_client._is_unsupported_parameter_error``).
  480. UNSUPPORTED_PARAM_MARKERS = (
  481. "unsupported parameter", "unsupported_parameter", "not supported", "does not support",
  482. "doesn't support", "is deprecated for this model",
  483. "unknown parameter", "unrecognized request argument", "unrecognized parameter",
  484. "invalid parameter", "extra inputs are not permitted",
  485. "invalid option: expected one of",
  486. )
  487. # Reasoning wire-field names (the profile reasoning controls minus ``verbosity``), longest first.
  488. # Standalone only: never a model-id segment ("The model kimi-k2-thinking is not supported when
  489. # using this account" is route gating for the provider-fallback rung) nor the adjective in
  490. # "... not supported with reasoning models".
  491. _REASONING_FIELD_TOKEN = re.compile(
  492. r"(?<![\w\-/])(?:reasoning_effort|thinking_config|thinking_budget|enable_thinking|thinkingconfig"
  493. r"|thinkingbudget|reasoning|thinking|think)(?![\w\-/])(?!\s+models?\b)"
  494. )
  495. # Structured rejection of a reasoning field, read from the stringified body: OpenAI-style
  496. # ``param`` naming a reasoning field (``reasoning_effort`` on chat, ``reasoning.effort`` on
  497. # Responses) or an ``invalid_reasoning_effort`` code. Custom Responses relays send this with NO
  498. # message at all (#100536), so no wording rule can match it — and without a match the message-less
  499. # 400 fell through to the generic large-session overflow heuristic and started compression.
  500. _REASONING_PARAM_REJECTION = re.compile(
  501. r"""['"]param['"]\s*:\s*['"](?:reasoning(?:[._]effort)?|thinking(?:_config|_budget)?|enable_thinking)['"]"""
  502. r"""|invalid_reasoning_effort"""
  503. )
  504. _REASONING_REQUIRED_MARKERS = (
  505. "mandatory", "cannot be disabled", "can't be disabled", "must be enabled", "is required",
  506. "always enabled", "cannot be turned off",
  507. )
  508. # Same rejection, wording by *value*: an endpoint that publishes a closed level set refuses the
  509. # thinking-OFF encoding because ``none`` is not in it — "'reasoning_effort' must be one of:
  510. # 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra'" (Alibaba token-plan / DashScope
  511. # routes, code ``invalid_parameter_error``). The field is understood and has to stay on the wire;
  512. # only the disabled level is invalid, so the floor step-up (the weakest accepted level) is the
  513. # right recovery. Neither the strip rung nor its "unsupported" wording matches this message, so
  514. # without these markers the title lane fails outright instead of retrying at the floor.
  515. _REASONING_LEVEL_REJECTION_MARKERS = ("must be one of", "invalid value for")
  516. def is_reasoning_level_rejection(error_msg: str) -> bool:
  517. """Provider 400 rejecting the *value* of a reasoning wire control against the endpoint's closed
  518. level set ("'reasoning_effort' must be one of: 'minimal', 'low', …", Alibaba token-plan /
  519. DashScope routes, code ``invalid_parameter_error``). The reasoning-field token must be present
  520. next to the wording — a 400 enum complaint about some other field is not this class. The field is
  521. understood and has to stay on the wire; only the disabled level ("none") is invalid, so the
  522. recovery is the floor step-up in the auxiliary ladder and the disable-drop in the main loop, not
  523. the strip rung and not an abort to the fallback chain."""
  524. msg = (error_msg or "").lower()
  525. token = _REASONING_FIELD_TOKEN.search(msg)
  526. if token is None:
  527. return False
  528. near = msg[max(0, token.start() - 48):token.end() + 96]
  529. return any(m in near for m in _REASONING_LEVEL_REJECTION_MARKERS)
  530. def is_reasoning_required_rejection(error_msg: str) -> bool:
  531. """Provider 400 saying the model's reasoning cannot be switched OFF ("Reasoning is mandatory for
  532. this endpoint and cannot be disabled", the Nous Portal on gpt-6-astra) — or refusing the OFF
  533. *level* because the endpoint's closed set starts at ``minimal``
  534. (``is_reasoning_level_rejection``). The opposite of ``is_reasoning_field_rejection``: the field is
  535. understood, the *disable* is refused, so the right reaction is to step the effort up to the
  536. lowest level rather than drop the field (a dropped field also works, but tells the caller nothing
  537. about the next call)."""
  538. msg = (error_msg or "").lower()
  539. token = _REASONING_FIELD_TOKEN.search(msg)
  540. if token is None:
  541. return False
  542. near = msg[max(0, token.start() - 48):token.end() + 96]
  543. return any(m in near for m in _REASONING_REQUIRED_MARKERS) or is_reasoning_level_rejection(msg)
  544. def is_reasoning_field_rejection(error_msg: str) -> bool:
  545. """Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
  546. ``thinking``/``think``): the field token plus either a generic unsupported marker ("Unrecognized
  547. request argument supplied: reasoning_effort", #112781) or a standalone "unsupported" next to the
  548. field in either word order ("unsupported reasoning_effort"; "reasoning_effort 'none' unsupported;
  549. use minimal|low|medium|high|xhigh", #114460). The route default is the right answer for such a
  550. model, so both the main loop and the auxiliary ladder retry once without the disable. A body
  551. whose structured ``param``/code names the reasoning field (``'param': 'reasoning.effort'``,
  552. ``invalid_reasoning_effort``, #100536) is a rejection whatever the message says — even none.
  553. Known trade-off: a 400 about a thinking *state* ("Function calling is not supported when
  554. thinking is enabled") also matches — the marker sits right next to the token, so no proximity
  555. rule separates it from the forward wordings. Cost is one dropped-disable retry before the
  556. spent path takes the fallback chain; the auxiliary ladder already treated it this way."""
  557. msg = (error_msg or "").lower()
  558. if _REASONING_PARAM_REJECTION.search(msg):
  559. return True
  560. token = _REASONING_FIELD_TOKEN.search(msg)
  561. if token is None:
  562. return False
  563. near = msg[max(0, token.start() - 32):token.end() + 32]
  564. return "unsupported" in near or any(m in msg for m in UNSUPPORTED_PARAM_MARKERS)
  565. def _billing_hints(error_msg: str) -> Verdict:
  566. """Billing verdict carrying the #82154 ambiguity marker when applicable."""
  567. ctx: dict[str, Any] = {}
  568. if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS):
  569. ctx = {"billing_unverified": True, "possible_content_filter": True}
  570. return {**_V_BILLING, "error_context": ctx}
  571. def _first_match(error_msg: str, rules: Sequence[tuple[Sequence[str], Any]]) -> Optional[Verdict]:
  572. """Verdict of the first rule whose pattern list hits ``error_msg``."""
  573. for patterns, verdict in rules:
  574. if any(p in error_msg for p in patterns):
  575. return verdict(error_msg) if callable(verdict) else verdict
  576. return None
  577. # Image/tool-content 400s, ordered: multimodal recovery ≠ image shrink; corrupt
  578. # bytes need strip not shrink; image-shrink is cheaper than context compression.
  579. _IMAGE_TOOL_RULES = (
  580. (_MULTIMODAL_TOOL_CONTENT_PATTERNS, _V_MULTIMODAL), (_IMAGE_CORRUPT_PATTERNS, _V_IMAGE_CORRUPT),
  581. (_IMAGE_TOO_LARGE_PATTERNS, _V_IMAGE_TOO_LARGE),
  582. )
  583. # Overflow signals arriving as 5xx (llama.cpp reports overflow as 500; busy /
  584. # model-load OOM as 503). Empty-response advisories must not enter compression.
  585. _OVERFLOW_AS_5XX_RULES = (
  586. (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR), (_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
  587. (_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW),
  588. )
  589. # 404: Nous API surfaces credit depletion as a paid model vanishing from the
  590. # Free Tier (billing, not missing model); policy block before model_not_found.
  591. _404_RULES = (
  592. (_BILLING_PATTERNS, _V_BILLING), (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED),
  593. (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
  594. )
  595. # 400 tail after the deterministic request-shape checks. Some providers return
  596. # model-not-found / rate-limit / billing as 400 instead of 404/429/402.
  597. _400_TAIL_RULES = _OVERFLOW_AS_5XX_RULES + (
  598. (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
  599. (_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_BILLING_PATTERNS, _billing_hints),
  600. )
  601. # LM Studio / llama.cpp raise a bare status-less ``APIError`` when chat-template Jinja rendering
  602. # fails mid-stream (#62662). Deterministic for the request, so fall back instead of retrying.
  603. _STREAM_RENDER_ERROR_PATTERNS = ("error rendering", "rendering prompt", "jinja template", "jinja render")
  604. # Status-less message path, head (before usage-limit disambiguation).
  605. _MESSAGE_HEAD_RULES = ((_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
  606. (_PAYLOAD_TOO_LARGE_PATTERNS, _V_PAYLOAD_TOO_LARGE),
  607. (_ROLE_ALTERNATION_PATTERNS, _V_ROLE_ALTERNATION),
  608. (_STREAM_RENDER_ERROR_PATTERNS, _V_FORMAT_ERROR)) + _IMAGE_TOOL_RULES
  609. # Status-less tail. Overload before rate_limit/billing so "overloaded" backs off
  610. # instead of rotating; policy block before model_not_found; timeout/connection
  611. # wording last, classified as transport (never compression).
  612. _MESSAGE_TAIL_RULES = (
  613. (_OVERLOADED_PATTERNS, _V_OVERLOADED), (_BILLING_PATTERNS, _billing_hints),
  614. (_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR),
  615. (_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW), (_AUTH_PATTERNS, _V_AUTH_ROTATE),
  616. (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
  617. (_TIMEOUT_MESSAGE_PATTERNS, _V_TIMEOUT), (_CONNECTION_MESSAGE_PATTERNS, _V_TIMEOUT),
  618. )
  619. # Structured error code → verdict. The error-code rate_limit verdict rotates
  620. # but does not set should_fallback (unlike the message/status paths).
  621. _ERROR_CODE_VERDICTS: dict[str, Verdict] = {
  622. **dict.fromkeys(("resource_exhausted", "throttled", "rate_limit_exceeded"),
  623. _v(_R.rate_limit, should_rotate_credential=True)),
  624. **dict.fromkeys(_BILLING_ERROR_CODES, _V_BILLING),
  625. **dict.fromkeys(("model_not_found", "model_not_available", "invalid_model"), _V_MODEL_NOT_FOUND),
  626. **dict.fromkeys(("context_length_exceeded", "max_tokens_exceeded"), _V_CONTEXT_OVERFLOW),
  627. **dict.fromkeys(_MEMORY_CEILING_ERROR_CODES, _V_OVERLOADED),
  628. "invalid_encrypted_content": _V_INVALID_ENCRYPTED,
  629. }
  630. # Provider-native status codes that arrive as a bare ``{"error": {"code": …}}`` body
  631. # (no HTTP status, no prose): gRPC canonical names from Gemini, Anthropic error
  632. # types, OpenAI's ``server_error``. Scoped per provider so a coincidentally named
  633. # code from another backend stays ``unknown`` (#70414). Provider aliases collapse
  634. # to the family key before lookup.
  635. _PROVIDER_CODE_FAMILIES = {"openai-codex": "openai", "google": "gemini", "google-gemini": "gemini",
  636. "google-ai-studio": "gemini", "vertex": "gemini", "google-vertex": "gemini"}
  637. _PROVIDER_CODE_VERDICTS: dict[str, dict[str, Verdict]] = {
  638. "openai": {"server_error": _V_SERVER_ERROR},
  639. "gemini": {"unavailable": _V_OVERLOADED, "deadline_exceeded": _V_TIMEOUT, "internal": _V_SERVER_ERROR},
  640. "anthropic": {"api_error": _V_SERVER_ERROR, "rate_limit_error": _V_RATE_LIMIT},
  641. }
  642. # Generic ``invalid_request_error`` is deliberately NOT a 400 validation
  643. # signal — OpenAI stamps it on genuine overflow 400s too.
  644. _400_VALIDATION_CODES = {"unknown_parameter", "unsupported_parameter"}
  645. _5XX_VALIDATION_CODES = _400_VALIDATION_CODES | {"invalid_request_error"}
  646. _400_VALIDATION_PATTERNS = tuple(p for p in _REQUEST_VALIDATION_PATTERNS if p != "invalid_request_error")
  647. # ── Classification pipeline ─────────────────────────────────────────────
  648. @dataclass
  649. class _Ctx:
  650. """Everything the classifier stages need about one failed call."""
  651. error: Exception
  652. status_code: Optional[int]
  653. body: dict
  654. msg: str # lowercased str(error) + body message(s)
  655. provider: str # as passed by the caller
  656. model: str
  657. approx_tokens: int
  658. context_length: int
  659. num_messages: int
  660. base_url: str = "" # the route the call went to; "" when the caller did not say
  661. anonymous: bool = False
  662. def __post_init__(self) -> None:
  663. self.error_type = type(self.error).__name__
  664. self.error_code = _extract_error_code(self.body)
  665. self.code = self.error_code.lower()
  666. self.headers = _from_cause_chain(self.error, _headers_of, {})
  667. self.provider_slug = (self.provider or "").strip().lower()
  668. self.model_slug = (self.model or "").strip().lower()
  669. def large_session(self, frac: float, tokens: int, messages: int) -> bool:
  670. """Absolute thresholds only proxy for smaller context windows."""
  671. return self.approx_tokens > self.context_length * frac or (
  672. self.context_length <= 256000 and (self.approx_tokens > tokens or self.num_messages > messages)
  673. )
  674. def _plugin_verdict(c: _Ctx) -> Optional[Verdict]:
  675. """First valid plugin classification (runs before the built-in pipeline so a
  676. provider plugin can add or correct verdicts). invoke_hook isolates callback
  677. failures; this guard only covers import/dispatch failure."""
  678. try:
  679. from hermes_cli.plugins import get_plugin_error_classification
  680. verdict = get_plugin_error_classification(
  681. provider=c.provider, model=c.model, status_code=c.status_code, error_type=c.error_type,
  682. error_code=c.error_code, error_message=c.msg, error_body=c.body, error=c.error,
  683. approx_tokens=c.approx_tokens, context_length=c.context_length, num_messages=c.num_messages,
  684. )
  685. except Exception as exc:
  686. logger.debug("Plugin error classification unavailable: %s", exc)
  687. return None
  688. if verdict is not None:
  689. logger.info("API error classified by plugin hook: %s (provider=%s, status=%s)",
  690. verdict["reason"].value, c.provider, c.status_code)
  691. return verdict
  692. def _profile_verdict(c: _Ctx) -> Optional[Verdict]:
  693. """The current provider's own ``ProviderProfile.classify_api_error`` verdict, or None.
  694. A ``kind: model-provider`` plugin never enters the PluginManager hook lifecycle, so without this a
  695. vendor-specific body (a 403 ``quota_exhausted`` that is billing, not auth) could only be corrected by
  696. shipping a second plugin component. Scoped to the provider that produced the error; no name table."""
  697. if not c.provider_slug:
  698. return None
  699. try:
  700. from providers import get_provider_profile
  701. hook = getattr(get_provider_profile(c.provider_slug), "classify_api_error", None)
  702. if not callable(hook):
  703. return None
  704. result = hook(c.error, status_code=c.status_code, error_code=c.error_code, message=c.msg,
  705. body=c.body, model=c.model)
  706. except Exception as exc:
  707. logger.debug("Provider profile error classification failed for %s: %s", c.provider_slug, exc)
  708. return None
  709. if not isinstance(result, dict):
  710. return None
  711. reason = result.get("reason")
  712. if isinstance(reason, str):
  713. try:
  714. reason = FailoverReason(reason.strip().lower())
  715. except ValueError:
  716. return None
  717. if not isinstance(reason, FailoverReason):
  718. return None
  719. hints = {k: bool(result[k]) for k in _HINT_FLAGS if k in result}
  720. # turn_api_error walks the fallback chain only for non-retryable verdicts outside
  721. # RETRYABLE_CLIENT_REASONS; the built-in terminal verdicts (billing, auth, model_not_found …) pin
  722. # retryable=False, the rate-limit family stays retryable and reaches fallback after backoff. Give
  723. # a hook that asks for fallback the built-in default for its reason, so it cascades like one.
  724. if hints.get("should_fallback") and "retryable" not in hints and reason not in RETRYABLE_CLIENT_REASONS:
  725. hints["retryable"] = False
  726. verdict = _v(reason, **hints)
  727. if isinstance(result.get("error_context"), dict):
  728. verdict["error_context"] = result["error_context"]
  729. logger.info("API error classified by provider profile: %s (provider=%s, status=%s)",
  730. reason.value, c.provider, c.status_code)
  731. return verdict
  732. _HINT_FLAGS = ("retryable", "should_compress", "should_rotate_credential", "should_fallback")
  733. # Reasons the retry loop keeps retrying (with backoff) even though the verdict may also carry
  734. # ``should_fallback``: the cascade for these runs after the backoff budget, not immediately.
  735. RETRYABLE_CLIENT_REASONS = frozenset({
  736. FailoverReason.rate_limit, FailoverReason.upstream_rate_limit, FailoverReason.overloaded,
  737. FailoverReason.context_overflow, FailoverReason.payload_too_large, FailoverReason.long_context_tier,
  738. FailoverReason.thinking_signature,
  739. })
  740. # A welcome-host 403 that spells one of these out is a safety block or a billing wall, not the
  741. # tier refusing. The free-tier refusal phrases are left OUT: on the free route they mean exactly
  742. # "the tier refused", and an anonymous session has no credits to check.
  743. _WELCOME_403_NAMED_PATTERNS = _CONTENT_POLICY_BLOCKED_PATTERNS + tuple(
  744. p for p in _BILLING_PATTERNS if p not in _FREE_TIER_REFUSAL_PATTERNS)
  745. def _nous_welcome_tier(c: _Ctx) -> Optional[Verdict]:
  746. """The Nous inference gateway's welcome-tier (free tier) refusals, read from the structured body.
  747. A 429 carrying a fairshare ``reason`` is either a tier gate (``model_not_free`` /
  748. ``feature_not_free``: the model or feature is never served on the free tier, so retrying is
  749. pointless — abort this route and fall back) or capacity (``at_capacity`` / ``admission_closed``
  750. / ``rate_limited``: honour ``retry_after``, never rotate the free tier's only credential). A
  751. 400/403 whose message names the wrong host or a dark tier is deterministic for the request.
  752. The parsed refusal rides ``error_context`` so the terminal copy can say what happened.
  753. """
  754. from hermes_cli.anon_auth import (
  755. WELCOME_TIER_GATE_REASONS, parse_welcome_refusal, route_is_welcome_host, welcome_route_refusal)
  756. status = c.status_code
  757. if not c.anonymous:
  758. # A named credential's fairshare 429 is an ordinary rate limit, whatever its body says. The
  759. # one welcome refusal it does receive is the gateway's mirror 400 on the welcome host; its
  760. # reconnect copy stands, only the sign-in card is withheld (``_welcome_surface_kind``).
  761. if c.provider == "nous" and status == 400 and welcome_route_refusal(status, c.msg) == "named_on_welcome_host":
  762. return _v(_R.format_error, retryable=False, should_fallback=True,
  763. error_context={"welcome_route": "named_on_welcome_host"})
  764. return None
  765. if status == 402 and route_is_welcome_host(c.base_url):
  766. # The free tier has no credits to top up: a payment wall on it is the tier refusing. Off the welcome
  767. # host a free-tier JWT keeps the ordinary 402 handling (the route heal, billing copy).
  768. refusal = {"reason": "refused", "retry_after": 0, "alternates": [], "upgrade_url": ""}
  769. return _v(_R.auth_permanent, retryable=False, should_fallback=True,
  770. error_context={"welcome_refusal": refusal})
  771. if status == 429:
  772. # Every free-tier 429 is the allowance talking, so even one without a ``reason`` (header-only,
  773. # quota words) is a refusal; otherwise it would fall to billing or the generic rate-limit copy.
  774. from agent.nous_rate_guard import welcome_refusal_from_headers
  775. refusal = parse_welcome_refusal(c.body) or welcome_refusal_from_headers(
  776. c.headers, body_wait=_rate_limit_reset_seconds(c.msg, c.body, c.headers))
  777. ctx = {"welcome_refusal": refusal}
  778. if refusal["reason"] in WELCOME_TIER_GATE_REASONS:
  779. return _v(_R.model_not_found, retryable=False, should_fallback=True, error_context=ctx)
  780. if refusal["retry_after"] > 0:
  781. ctx["reset_at"] = time.time() + refusal["retry_after"]
  782. return _v(_R.rate_limit, should_fallback=True, error_context=ctx)
  783. # The route-keyed dark-tier 403 applies only to a 403 that says nothing else: a safety refusal
  784. # or a billing wall on the welcome host keeps its own classification (and its own recovery).
  785. plain_403 = not any(p in c.msg for p in _WELCOME_403_NAMED_PATTERNS)
  786. kind = welcome_route_refusal(status, c.msg, c.base_url if plain_403 else None)
  787. if kind is None:
  788. return None
  789. ctx = {"welcome_route": kind}
  790. if status == 403:
  791. return _v(_R.auth_permanent, retryable=False, should_fallback=True, error_context=ctx)
  792. return _v(_R.format_error, retryable=False, should_fallback=True, error_context=ctx)
  793. def _is_gemini_prompt_block(error: Exception) -> bool:
  794. """The native Gemini adapter's ``promptFeedback.blockReason`` error, matched on its structured code
  795. (a 200 with no candidates, so no status or body to classify; the message text is not trusted)."""
  796. return _from_cause_chain(error, _gemini_prompt_block_marker, False)
  797. def _gemini_prompt_block_marker(exc: Any) -> Optional[bool]:
  798. is_block = type(exc).__name__ == "GeminiAPIError" and getattr(exc, "code", None) == "gemini_prompt_blocked"
  799. return True if is_block else None
  800. def _provider_special_cases(c: _Ctx) -> Optional[Verdict]:
  801. """Highest-priority provider-specific shapes that a status code would misroute."""
  802. msg, status = c.msg, c.status_code
  803. # Deterministic for this context: another provider would receive the same connector result.
  804. if c.code == "data_policy_denied":
  805. return _v(_R.data_policy_denied, retryable=False, should_fallback=False)
  806. welcome = _nous_welcome_tier(c)
  807. if welcome is not None:
  808. return welcome
  809. # Safety refusal before status classification so a 400 block isn't downgraded
  810. # to format_error and a status-less block isn't left retryable (#18028).
  811. if any(p in msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS) or _is_gemini_prompt_block(c.error):
  812. return _V_CONTENT_BLOCKED
  813. # Status-agnostic: the stream-relayed ban has no status, and a 403 variant is not a bad key.
  814. if any(p in msg for p in _ACCOUNT_POLICY_BLOCK_PATTERNS):
  815. return _V_POLICY_BLOCKED
  816. # ChatGPT Codex masks a rejected encrypted-reasoning replay behind the same bare
  817. # ``invalid_prompt: Request blocked.`` it uses for real blocks (#92353). Exact envelope
  818. # + provider only. The verdict keeps format_error's abort-and-fallback hints; the one
  819. # extra thing it buys is turn_recovery's replay strip, which still requires cached
  820. # ``codex_reasoning_items`` — a genuine block with nothing to strip behaves as before.
  821. if _is_codex_masked_replay_rejection(c):
  822. return _v(_R.invalid_encrypted_content, **_ABORT_FALLBACK)
  823. # OpenAI Responses rejects a stale encrypted-reasoning replay with this code (#70595). It contains
  824. # both "thinking" and "signature", so it must beat the Anthropic heuristic below: that recovery
  825. # strips Anthropic thinking blocks and resends the same encrypted item forever.
  826. if status == 400 and (c.code == "thinking_signature_invalid" or "thinking_signature_invalid" in msg):
  827. return _V_INVALID_ENCRYPTED
  828. # Anthropic thinking-block 400s (signature mismatch after transcript mutation, or a sealed blob
  829. # the upstream cannot open). Not gated on provider — OpenRouter proxies Anthropic errors.
  830. if status == 400 and _is_anthropic_thinking_rejection(msg):
  831. return _v(_R.thinking_signature)
  832. # Anthropic long-context tier gate (429 "extra usage" + "long context").
  833. if status == 429 and "extra usage" in msg and "long context" in msg:
  834. return _v(_R.long_context_tier, should_compress=True)
  835. # Anthropic OAuth rejects the 1M beta header; run_agent retries without it.
  836. if status == 400 and "long context beta" in msg and "not yet available" in msg:
  837. return _v(_R.oauth_long_context_beta_forbidden)
  838. # llama.cpp grammar rejects regex ``pattern``/``format`` in tool schemas; the
  839. # retry loop strips them. Exclude the Qwen/vLLM "No user query found" error
  840. # local engines wrap as "Unable to generate parser for this template" —
  841. # that is a poisoned transcript (→ format_error), not a grammar problem.
  842. # Strict OpenAI-compatible schema validators reject regex lookaround in ``pattern``
  843. # with a different sentence ("Invalid JSON schema: regex lookaround is not supported",
  844. # #42631); same recovery — strip ``pattern``/``format`` and retry once.
  845. grammar_hit = "error parsing grammar" in msg or "json-schema-to-grammar" in msg or (
  846. "unable to generate parser" in msg and "template" in msg
  847. ) or ("invalid json schema" in msg and "regex lookaround" in msg and "not supported" in msg)
  848. if status == 400 and grammar_hit and _NO_USER_QUERY_SIGNAL not in msg:
  849. return _v(_R.llama_cpp_grammar_pattern)
  850. # xAI Grok entitlement as an SSE ``type=error`` frame: no status, matches no
  851. # pattern list, would otherwise burn max_retries as ``unknown``.
  852. if "do not have an active grok subscription" in msg or ("out of available resources" in msg and "grok" in msg):
  853. return _V_AUTH_FALLBACK
  854. return None
  855. def _moa_special_cases(c: _Ctx) -> Optional[Verdict]:
  856. # Local MoA streaming adapter-shape bugs are not a provider outage; falling
  857. # back would silently replace the MoA route with a single model (#55933).
  858. if c.provider_slug == "moa" and any(s in str(c.error) for s in _MOA_ADAPTER_SHAPE_BUGS):
  859. return _v(_R.format_error, retryable=False)
  860. # Persisted MoA preset name that was renamed/deleted — deterministic config error.
  861. from agent.errors import MoAPresetNotFoundError
  862. return _v(_R.model_not_found, retryable=False) if isinstance(c.error, MoAPresetNotFoundError) else None
  863. def _by_error_code(c: _Ctx) -> Optional[Verdict]:
  864. """Structured error codes from the response body."""
  865. # Request-validation failure as plain-text ``event: error`` SSE data behind
  866. # HTTP 200: retrying cannot succeed, a configured fallback still may.
  867. if c.code == PROVIDER_STREAM_NON_JSON_ERROR_CODE and "request validation failed:" in c.msg:
  868. return _V_FORMAT_ERROR
  869. verdict = _ERROR_CODE_VERDICTS.get(c.code)
  870. if verdict is None:
  871. family = _PROVIDER_CODE_FAMILIES.get(c.provider_slug, c.provider_slug)
  872. verdict = _PROVIDER_CODE_VERDICTS.get(family, {}).get(c.code)
  873. return verdict
  874. def _by_message(c: _Ctx) -> Optional[Verdict]:
  875. """Message patterns when no status code settled it; status-less usage
  876. limits get the same disambiguation as 402."""
  877. head = _first_match(c.msg, _MESSAGE_HEAD_RULES)
  878. if head is not None:
  879. return head
  880. if _is_byte_size_limit_message(c.msg):
  881. return _V_PAYLOAD_TOO_LARGE
  882. usage_limit = any(p in c.msg for p in _USAGE_LIMIT_PATTERNS)
  883. return _classify_402(c.msg, dict) if usage_limit else _first_match(c.msg, _MESSAGE_TAIL_RULES)
  884. def _by_transport(c: _Ctx) -> Optional[Verdict]:
  885. """SSL, disconnect, circuit-breaker and transport-type heuristics, in that order."""
  886. msg = c.msg
  887. # Cert failure → fail fast (checked first: also contains "[ssl:"); transient
  888. # alert → retry, before disconnects so a flaky handshake never compresses.
  889. ssl = _first_match(msg, ((_SSL_CERT_VERIFY_PATTERNS, _V_SSL_CERT), (_SSL_TRANSIENT_PATTERNS, _V_TIMEOUT)))
  890. if ssl is not None:
  891. return ssl
  892. # Disconnect + large session → probable overflow rejection, not a hiccup.
  893. if any(p in msg for p in _SERVER_DISCONNECT_PATTERNS) and not c.status_code:
  894. # Reasoning models: far more likely the gateway idle-killed a long
  895. # thinking stream — never compress on a phantom overflow (#52310).
  896. # Reasoning-model override: a transport disconnect on a reasoning model is much more likely the
  897. # upstream proxy idle-killing a long thinking stream than a true context overflow — even on large
  898. # sessions. The default disconnect+large-session routing below would otherwise send the user into
  899. # the compression branch (should_compress=True) and silently delete conversation history on a
  900. # phantom context-length error. Reasoning models have multi-minute thinking phases that routinely
  901. # exceed the cloud gateway's idle window (NVIDIA NIM ~120s — first-party repro at
  902. # NVIDIA/NemoClaw#4846; OpenAI worker / Anthropic stream-idle similar). The per-reasoning-model
  903. # stale-timeout floor in agent/reasoning_timeouts.py raises the stale-detector threshold to tolerate
  904. # long thinking, so a true transport-layer failure here is recoverable via the retry path — not via
  905. # context compression. Reclassify as timeout. (Part 1 of Fixes #52310.)
  906. from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
  907. if get_reasoning_stale_timeout_floor(c.model) is not None:
  908. return _V_TIMEOUT
  909. return _V_CONTEXT_OVERFLOW if c.large_session(0.6, 120000, 200) else _V_TIMEOUT
  910. # Stale-call circuit breaker (_check_stale_giveup RuntimeError before any
  911. # network call): as ``unknown`` it would burn every retry instantly.
  912. if c.error_type == "RuntimeError" and "consecutive stale attempts" in msg and "aborting this call" in msg:
  913. return _v(_R.timeout, **_ABORT_FALLBACK)
  914. transport = c.error_type in _TRANSPORT_ERROR_TYPES or isinstance(c.error, (TimeoutError, ConnectionError, OSError))
  915. return _V_TIMEOUT if transport else None
  916. def _by_status(c: _Ctx) -> Optional[Verdict]:
  917. """HTTP status code with message-aware refinement (unlisted 4xx/5xx → generic)."""
  918. status = c.status_code
  919. if status is None:
  920. return None
  921. default = _V_FORMAT_ERROR if 400 <= status < 500 else _V_SERVER_ERROR if 500 <= status < 600 else None
  922. return _STATUS_HANDLERS[status](c) if status in _STATUS_HANDLERS else default
  923. # Stage order: plugin hooks → the provider's own profile hook → provider-specific special cases →
  924. # HTTP status → MoA shapes → structured error code → message patterns → SSL → disconnect +
  925. # large session → transport types → unknown (retryable with backoff).
  926. _STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = (
  927. _plugin_verdict, _profile_verdict, _provider_special_cases, _by_status, _moa_special_cases,
  928. _by_error_code, _by_message, _by_transport,
  929. )
  930. def classify_api_error(
  931. error: Exception, *, provider: str = "", model: str = "",
  932. approx_tokens: int = 0, context_length: int = 200000, num_messages: int = 0,
  933. base_url: str = "",
  934. api_key: Any = None,
  935. ) -> ClassifiedError:
  936. """Classify an API error into a structured recovery recommendation (see ``_STAGES``).
  937. ``base_url`` (optional) is the route the call went to; the Nous welcome tier keys its
  938. dark-tier 403 on it because that refusal carries no distinguishing message.
  939. ``api_key`` identifies an anonymous request; a host or fairshare reason alone does not.
  940. The credential is never included in the returned context."""
  941. from hermes_cli.anon_auth import is_anonymous_request
  942. status_code = _extract_status_code(error)
  943. # Some SDK RateLimitErrors (OpenAI-compatible relays) may not set .status_code; force 429.
  944. if status_code is None and type(error).__name__ == "RateLimitError":
  945. status_code = 429
  946. body = _extract_error_body(error)
  947. c = _Ctx(
  948. error, status_code, body, _build_error_msg(error, body), provider, model,
  949. approx_tokens, context_length, num_messages, str(base_url or ""),
  950. anonymous=is_anonymous_request(provider, api_key),
  951. )
  952. verdict = next((v for v in (stage(c) for stage in _STAGES) if v is not None), _V_UNKNOWN)
  953. message = _extract_message(error, body)
  954. if verdict["reason"] in (_R.auth, _R.auth_permanent):
  955. # An auth refusal from a non-stock route names the host, so a credential posted to the
  956. # wrong endpoint (a stale ``model.base_url`` after a provider switch, #113719) reads as
  957. # such — not as a bad key.
  958. host = _off_route_host(c)
  959. if host:
  960. message = f"{message} (endpoint: {host})"
  961. base = {"status_code": status_code, "provider": provider, "model": model, "message": message}
  962. return ClassifiedError(**{**base, **verdict})
  963. def _off_route_host(c: _Ctx) -> str:
  964. """The contacted host when ``base_url`` is set and is not the provider's own endpoint; ``""`` otherwise."""
  965. from hermes_cli.route_identity import provider_owns_route
  966. from utils import base_url_hostname
  967. host = base_url_hostname(c.base_url)
  968. if not host or provider_owns_route(c.provider_slug, c.base_url) is True:
  969. return ""
  970. return host
  971. # ── Status code handlers ────────────────────────────────────────────────
  972. # Structured codes some gateways put on a 403 that mean "the upstream is down,
  973. # retry later" — not a credential refusal (#75388). Checked before the auth
  974. # default so the configured retry budget applies and no credential is benched.
  975. _403_TRANSIENT_CODES = frozenset({"upstream_unavailable"})
  976. def _status_403(c: _Ctx) -> Verdict:
  977. if c.code in _403_TRANSIENT_CODES:
  978. return _V_OVERLOADED
  979. # OpenRouter 403 "key limit exceeded" and similar plan/credit exhaustion are billing.
  980. xai_spend = c.provider_slug == "xai-oauth" and c.code == _XAI_SPENDING_LIMIT_ERROR_CODE
  981. billing = xai_spend or any(p in c.msg for p in ("key limit exceeded", "spending limit") + _BILLING_PATTERNS)
  982. if billing:
  983. return _V_BILLING
  984. # A WAF/CDN in front of the provider answered, not the provider: the credential never
  985. # reached it, so key guidance and credential rotation are wrong (#53099, #70566). Gated on
  986. # 403 and on established block/challenge markers; any other 403 stays auth.
  987. if any(p in c.msg for p in _UPSTREAM_BLOCKED_PATTERNS):
  988. return _V_UPSTREAM_BLOCKED
  989. return _V_AUTH_FALLBACK
  990. def _status_404(c: _Ctx) -> Verdict:
  991. # Structured billing code first, as in _status_429: this handler always returns,
  992. # so _by_error_code never sees it; a bare "Not Found" message has nothing to match.
  993. if c.code in _BILLING_ERROR_CODES:
  994. return _V_BILLING
  995. verdict = _first_match(c.msg, _404_RULES)
  996. if verdict is not None:
  997. return verdict
  998. # Bare id the catalogue only knows prefixed → malformed id (NVIDIA NIM "404
  999. # page not found", #78796). A generic 404 (wrong path, proxy glitch) stays
  1000. # unknown so the real error surfaces instead of a silent misreported fallback.
  1001. return _V_MODEL_NOT_FOUND if _model_id_missing_known_prefix(c.model_slug, c.provider_slug) else _V_UNKNOWN
  1002. def _status_429(c: _Ctx) -> Verdict:
  1003. # A structured billing code is decisive: LiteLLM stamps
  1004. # ``terminal_quota_exhausted`` (a hard cap, not throttling) on 429s, and
  1005. # this handler always returns, so _by_error_code never sees the code.
  1006. if c.code in _BILLING_ERROR_CODES:
  1007. return _V_BILLING
  1008. # Z.AI/Zhipu reuse 429 for server-wide overload: back off on the same
  1009. # key instead of burning the pool (#14038).
  1010. if any(p in c.msg for p in _OVERLOADED_PATTERNS):
  1011. return _V_OVERLOADED
  1012. # OpenRouter-wrapped upstream 429: the key is healthy — fall back, don't bench.
  1013. if _is_openrouter_upstream_error(c.body, c.provider_slug):
  1014. upstream = _extract_upstream_provider_name(c.body)
  1015. ctx = {"upstream_provider": upstream} if upstream else {}
  1016. return _v(_R.upstream_rate_limit, should_fallback=True, error_context=ctx)
  1017. # Quota walls as 429 (Anthropic ``usage_limit_reached``, "quota", billing
  1018. # phrases) are billing ONLY when the body is not itself a rate-limit phrase
  1019. # ("Rate limit exceeded" contains "limit exceeded") and carries no reset/
  1020. # retry signal (#93419, #39441).
  1021. quota_wall = c.code == "usage_limit_reached" or any(
  1022. p in c.msg for p in ("usage_limit_reached",) + _USAGE_LIMIT_PATTERNS + _BILLING_PATTERNS
  1023. )
  1024. explicit_rate_limit = any(p in c.msg for p in _RATE_LIMIT_PATTERNS)
  1025. if quota_wall and not explicit_rate_limit and not _has_usage_limit_transient_signal(c.msg, c.body, c.headers):
  1026. return _V_BILLING
  1027. # Carry the reset window so the terminal copy can name it instead of "wait a minute" (#89401).
  1028. reset = _rate_limit_reset_seconds(c.msg, c.body, c.headers)
  1029. if reset:
  1030. return _v(_R.rate_limit, **_ROTATE_FALLBACK, error_context={"reset_at": time.time() + reset})
  1031. return _V_RATE_LIMIT
  1032. def _status_5xx(c: _Ctx) -> Verdict:
  1033. # Request-validation errors as 5xx (codex.nekos.me) fail fast instead of
  1034. # retry-flooding — unless the parameter was injected server-side.
  1035. validation = any(p in c.msg for p in _REQUEST_VALIDATION_PATTERNS) or c.code in _5XX_VALIDATION_CODES
  1036. if validation and not _is_server_injected_param_rejection(c.msg, c.provider_slug):
  1037. return _V_FORMAT_ERROR
  1038. return _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_SERVER_ERROR
  1039. def _is_nous_route(c: _Ctx) -> bool:
  1040. """The usage-window 402 code is Nous's: any other upstream sending it keeps main's text rules."""
  1041. if (c.provider or "").strip().lower() == "nous":
  1042. return True
  1043. from utils import base_url_host_matches
  1044. return bool(c.base_url) and base_url_host_matches(c.base_url, "nousresearch.com")
  1045. def _classify_402(error_msg: str, result_fn: Callable[..., Any], code: str = "", body: Any = None) -> Any:
  1046. """Disambiguate 402: "usage limit, try again in 5 minutes" is a periodic quota, not billing.
  1047. The Nous usage-window code is decisive (its prose names the reset time, which must not
  1048. demote it to a transient rate limit); every other 402 keeps the text rules. A usage-window 402's
  1049. optional ``display_message`` (server-set wording) rides in ``error_context`` for the refusal line."""
  1050. if code == _NOUS_USAGE_WINDOW_402_CODE:
  1051. # Tagged so recovery skips the paid-entitlement refresh (none restores included usage before the reset).
  1052. context: dict[str, Any] = {"usage_window": True}
  1053. if display := _usage_window_display_message(body):
  1054. context["display_message"] = display
  1055. # No fallback: another model is the user's call (the limit card offers it), never automatic.
  1056. return result_fn(**{**_V_BILLING, "should_fallback": False}, error_context=context)
  1057. transient = any(p in error_msg for p in _USAGE_LIMIT_PATTERNS) and any(
  1058. p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS
  1059. )
  1060. return result_fn(**(_V_RATE_LIMIT if transient else _V_BILLING))
  1061. def _usage_window_display_message(body: Any) -> str:
  1062. """The server's optional wording for a usage-window 402 (top level or under ``error``), trimmed and
  1063. capped; "" when absent."""
  1064. if not isinstance(body, dict):
  1065. return ""
  1066. for holder in (body, body.get("error")):
  1067. value = holder.get("display_message") if isinstance(holder, dict) else None
  1068. if isinstance(value, str) and value.strip():
  1069. return value.strip()[:500]
  1070. return ""
  1071. def _has_large_inline_image(content: Any) -> bool:
  1072. """True when a rejected ``content`` list carries a ``data:image/`` part the shrink pass would rewrite
  1073. (over ``conversation_compression._IMAGE_SHRINK_TARGET_BYTES``; below it a shrink retry is a no-op)."""
  1074. from agent.conversation_compression import _IMAGE_SHRINK_TARGET_BYTES
  1075. for part in content if isinstance(content, list) else ():
  1076. image = part.get("image_url") if isinstance(part, dict) else None
  1077. url = image.get("url") if isinstance(image, dict) else image
  1078. if isinstance(url, str) and url.startswith("data:image/") and len(url) > _IMAGE_SHRINK_TARGET_BYTES:
  1079. return True
  1080. return False
  1081. def _oversized_message_content_rejection(body: Any) -> bool:
  1082. """400 rejecting a *message* ``content`` field whose rejected value carries a large inline image.
  1083. Nebius Token Factory caps a single image at 10 MiB and reports the violation through the field that
  1084. failed to coerce — pydantic ``{"type": "string_type", "loc": ["body","messages",N,"content","str"],
  1085. "msg": "Input should be a valid string", "input": [...]}`` — naming no size vocabulary, so the
  1086. keyword multimodal *tool*-content rule (#104731) claimed it and spent its retry stripping tool images
  1087. that were never there (#112473). The same list-shaped content with a small image succeeds, so the
  1088. image bytes are the trigger. Tool-scoped locs (``messages.N.tool.content.str``) stay with #104731.
  1089. """
  1090. details = body.get("detail") if isinstance(body, dict) else None
  1091. for detail in details if isinstance(details, list) else ():
  1092. loc = detail.get("loc") if isinstance(detail, dict) else None
  1093. if detail.get("type") != "string_type" or not isinstance(loc, list) or len(loc) < 2:
  1094. continue
  1095. parts = [str(x).lower() for x in loc]
  1096. if parts[:2] == ["body", "messages"] and parts[-2:] == ["content", "str"] and not any(
  1097. x.startswith("tool") for x in parts
  1098. ) and _has_large_inline_image(detail.get("input")):
  1099. return True
  1100. return False
  1101. def _classify_400(c: _Ctx) -> Verdict:
  1102. """400 Bad Request — image/tool shapes, request-shape rejections, overflow, or generic."""
  1103. msg, code = c.msg, c.code
  1104. # A size cap reported *through* a message content field must beat the keyword
  1105. # multimodal rule, which would otherwise claim "input should be a valid string".
  1106. if _oversized_message_content_rejection(c.body):
  1107. return _V_IMAGE_TOO_LARGE
  1108. verdict = _first_match(msg, _IMAGE_TOOL_RULES)
  1109. if verdict is not None:
  1110. return verdict
  1111. # Codex ChatGPT-account model rejection: exact normalized text only, so arbitrary 400s never
  1112. # rotate. Before request-validation, whose "not supported" wording would abort as format_error (#71970).
  1113. if CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER in msg:
  1114. return _V_MODEL_ENTITLEMENT
  1115. # Invalid encrypted reasoning replay blob (OpenAI Responses); before
  1116. # overflow because "encrypted content … could not be verified" trips it.
  1117. if code == "invalid_encrypted_content" or "invalid_encrypted_content" in msg or (
  1118. "encrypted content for item" in msg and "could not be verified" in msg
  1119. ) or "could not decrypt the provided encrypted_content" in msg or (
  1120. # Custom Responses endpoints wrap a replay rejection in a generic bad_request (#95834).
  1121. "encrypted content could not be decrypted or parsed" in msg
  1122. ) or (
  1123. # OpenCode Zen wraps this OpenAI replay rejection in ``invalid_request_error`` (#111309).
  1124. "encrypted_content" in msg and "was not issued to this caller" in msg
  1125. ) or (
  1126. # Azure Foundry (gpt-6-astra) rejects replayed reasoning from several prior responses this way (#105369).
  1127. "conflicting authenticated continuation identities" in msg
  1128. ):
  1129. return _V_INVALID_ENCRYPTED
  1130. # Route rejecting a reasoning disable: a reasoning-mandatory route (GLM-5.3 on Nous Portal /
  1131. # OpenRouter), a chat-only relay that does not accept ``reasoning_effort: none`` at all
  1132. # (#114460), or an endpoint whose closed level set starts above the disable ("must be one of:
  1133. # 'minimal', …" — Alibaba token-plan). Deterministic for the request shape, but the only bad
  1134. # value is the disable — the loop drops it (or, when the disable is refused rather than the
  1135. # field, steps the effort up to the floor) and retries once. Must precede request-validation,
  1136. # which would abort as format_error.
  1137. if (_REASONING_MANDATORY_PATTERN in msg or is_reasoning_field_rejection(msg)
  1138. or is_reasoning_level_rejection(msg)):
  1139. return _V_REASONING_MANDATORY
  1140. # 400 blaming a field this route never sent (Codex OAuth injects then rejects
  1141. # prompt_cache_retention ~20% of the time): transient, retry identical request.
  1142. if _is_server_injected_param_rejection(msg, c.provider_slug):
  1143. return _V_SERVER_ERROR
  1144. if any(p in msg for p in _MALFORMED_TOOL_ARGS_PATTERNS):
  1145. return _V_MALFORMED_TOOL_ARGS
  1146. if any(p in msg for p in _ROLE_ALTERNATION_PATTERNS):
  1147. return _V_ROLE_ALTERNATION
  1148. # Body byte cap before the invalid-body and token-overflow rules: Novita tags it
  1149. # INVALID_REQUEST_BODY and Z.AI's "request size exceeds the limit" trips "exceeds the limit".
  1150. if _is_byte_size_limit_message(msg):
  1151. return _V_PAYLOAD_TOO_LARGE
  1152. # Before overflow: GPT-5's "Unsupported parameter: 'max_tokens'" contains it.
  1153. if any(p in msg for p in _400_VALIDATION_PATTERNS) or code in _400_VALIDATION_CODES:
  1154. return _V_FORMAT_ERROR
  1155. # Malformed message array before overflow: input can be tiny and compression
  1156. # cannot fix it. litellm/Bedrock proxies use errorCode=INVALID_REQUEST_BODY.
  1157. if any(p in msg for p in _INVALID_MESSAGE_BODY_PATTERNS) or code == "invalid_request_body":
  1158. logger.warning(
  1159. "Malformed message array 400 (invalid request body) classified as format_error, NOT context "
  1160. "overflow — failing fast + falling back instead of entering the compression loop. This usually "
  1161. "means an empty-content assistant stub is in the transcript; num_messages=%s approx_tokens=%s. "
  1162. "error=%.200s", c.num_messages, c.approx_tokens, msg,
  1163. )
  1164. return _V_FORMAT_ERROR
  1165. # Memory ceiling by code: _by_status runs before _by_error_code, so a
  1166. # 400 whose wording a proxy stripped would fall through to format_error.
  1167. if code in _MEMORY_CEILING_ERROR_CODES:
  1168. return _V_OVERLOADED
  1169. verdict = _first_match(msg, _400_TAIL_RULES)
  1170. if verdict is not None:
  1171. return verdict
  1172. # Generic 400 + large session → probable overflow (Anthropic can return a
  1173. # bare "Error"); proxy shapes are read so a descriptive rejection isn't "bare".
  1174. body_msg = next((m for m in (str(x or "").strip().lower() for x in _body_message_candidates(c.body)) if m), "")
  1175. is_generic = len(body_msg) < 30 or body_msg in {"error", ""}
  1176. if is_generic and c.large_session(0.4, 80000, 80):
  1177. return _V_CONTEXT_OVERFLOW
  1178. return _V_FORMAT_ERROR
  1179. def _classify_image_tool_422(c: _Ctx) -> Verdict:
  1180. """422: pydantic relays report the same content-field shapes as 400 (#104731, #112473)."""
  1181. if _oversized_message_content_rejection(c.body):
  1182. return _V_IMAGE_TOO_LARGE
  1183. return _first_match(c.msg, _IMAGE_TOOL_RULES) or _V_FORMAT_ERROR
  1184. # 401 not retryable on its own: rotation/refresh run before the retryability
  1185. # check, then the client-error abort path (fallback first) is correct. 408 is
  1186. # retry-safe (RFC 9110 §15.5.9; proxies emit it when generation outruns the
  1187. # read window). Unlisted 4xx → format_error, 5xx → server_error.
  1188. _STATUS_HANDLERS: dict[int, Callable[[_Ctx], Verdict]] = {
  1189. 400: _classify_400, 401: lambda c: _V_AUTH_ROTATE,
  1190. 402: lambda c: _classify_402(c.msg, dict, c.code if _is_nous_route(c) else "", c.body),
  1191. 403: _status_403, 404: _status_404, 408: lambda c: _V_TIMEOUT, 413: lambda c: _V_PAYLOAD_TOO_LARGE,
  1192. 422: lambda c: _classify_image_tool_422(c),
  1193. 429: _status_429, 500: _status_5xx, 502: _status_5xx,
  1194. 503: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
  1195. 529: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
  1196. }
  1197. # ── Helpers ─────────────────────────────────────────────────────────────
  1198. _RESET_FIELDS = ("resets_in_seconds", "resets_at", "reset_at", "retry_after")
  1199. _RESET_HEADERS = ("retry-after", "Retry-After", "x-ratelimit-reset", "X-RateLimit-Reset")
  1200. def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_headers) -> bool:
  1201. """Whether a usage-limit response identifies a reset window (message, body fields, or headers)."""
  1202. if any(pattern in error_msg for pattern in _USAGE_LIMIT_TRANSIENT_SIGNALS):
  1203. return True
  1204. payloads = [p for p in (body, _error_obj(body)) if isinstance(p, dict)]
  1205. if any(payload.get(f) not in (None, "") for payload in payloads for f in _RESET_FIELDS):
  1206. return True
  1207. if response_headers and hasattr(response_headers, "get"):
  1208. return any(response_headers.get(h) not in (None, "") for h in _RESET_HEADERS)
  1209. return False
  1210. def _rate_limit_reset_seconds(error_msg: str, body: dict, response_headers) -> Optional[float]:
  1211. """Seconds until a 429's window reopens, from the body's reset fields, ``Retry-After`` or the
  1212. message grammar (``retry after Ns`` / ``resets in 4hr``); None when the response names none."""
  1213. from agent.retry_utils import parse_retry_after_seconds, reset_delay_from_message
  1214. for payload in (p for p in (body, _error_obj(body)) if isinstance(p, dict)):
  1215. for name in _RESET_FIELDS:
  1216. value = payload.get(name)
  1217. if value in (None, ""):
  1218. continue
  1219. if name.endswith("_at") and isinstance(value, (int, float)):
  1220. return max(0.0, float(value) - time.time())
  1221. if (seconds := parse_retry_after_seconds(value)) is not None:
  1222. return seconds
  1223. if (seconds := parse_retry_after_seconds(response_headers)) is not None:
  1224. return seconds
  1225. return reset_delay_from_message(error_msg)
  1226. def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
  1227. """True when a bare model id is only known to the provider as ``vendor/id``.
  1228. Never guesses: an id absent from the curated catalogue returns False so real
  1229. endpoint problems keep their retryable ``unknown`` classification.
  1230. """
  1231. name = (model or "").strip()
  1232. if not name or "/" in name:
  1233. return False
  1234. try:
  1235. from hermes_cli.model_normalize import suggest_prefixed_model_id
  1236. return bool(suggest_prefixed_model_id((provider or "").strip(), name))
  1237. except Exception:
  1238. return False
  1239. def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
  1240. """True when a 400 blames a one-route-only parameter this route never sends.
  1241. Conservative: known parameters only, and only when ``provider`` is not a
  1242. sender, so a genuine bad parameter (``max_tokens`` on GPT-5) stays format_error.
  1243. """
  1244. provider_slug = (provider or "").strip().lower()
  1245. for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
  1246. if error_msg and param in error_msg and any(w in error_msg for w in _PARAM_REJECTION_WORDS):
  1247. return not any(sender in provider_slug for sender in senders)
  1248. return False
  1249. _CODEX_MASKED_REPLAY_MESSAGE = "request blocked."
  1250. _CODEX_UNSUPPORTED_CONTENT_DETAIL = "unsupported content type"
  1251. def _is_codex_masked_replay_rejection(c: _Ctx) -> bool:
  1252. """HTTP 400 / status-less ``{code: invalid_prompt, message: "Request blocked."}`` from
  1253. ``openai-codex`` — as an SDK error body, a Responses ``error`` SSE frame, or the
  1254. ``response.failed`` text ``"invalid_prompt: Request blocked."`` — or the bare
  1255. ``{"detail": "Unsupported content type"}`` envelope the same backend returns for a rejected
  1256. encrypted-reasoning replay (#51512). Both are exact envelopes, provider-gated."""
  1257. if c.provider_slug != "openai-codex" or c.status_code not in (None, 400):
  1258. return False
  1259. body = c.body if isinstance(c.body, dict) else {}
  1260. if str(body.get("detail") or "").strip().lower() == _CODEX_UNSUPPORTED_CONTENT_DETAIL or (
  1261. not body and _CODEX_UNSUPPORTED_CONTENT_DETAIL in c.msg and "detail" in c.msg
  1262. ):
  1263. return True
  1264. # The OpenAI SDK unwraps ``body["error"]`` on status errors; stream frames keep the envelope.
  1265. body_msg = next((str(m).strip().lower() for m in _body_message_candidates(body) if m), "")
  1266. return (c.code == "invalid_prompt" and body_msg == _CODEX_MASKED_REPLAY_MESSAGE) or (
  1267. c.msg.strip() == f"invalid_prompt: {_CODEX_MASKED_REPLAY_MESSAGE}"
  1268. )
  1269. def _error_obj(body: Any) -> dict:
  1270. """``body["error"]`` when it is a dict, else ``{}``."""
  1271. err = body.get("error") if isinstance(body, dict) else None
  1272. return err if isinstance(err, dict) else {}
  1273. def _json_dict(text: Any) -> Optional[dict]:
  1274. """Parse a JSON object string; None for non-strings, blanks, invalid JSON or non-objects."""
  1275. if not (isinstance(text, str) and text.strip()):
  1276. return None
  1277. try:
  1278. inner = json.loads(text)
  1279. except (json.JSONDecodeError, TypeError):
  1280. return None
  1281. return inner if isinstance(inner, dict) else None
  1282. def _openrouter_wrapped_message(err_obj: dict) -> str:
  1283. """Lowercased inner message from OpenRouter's ``error.metadata.raw`` JSON wrapper."""
  1284. metadata = err_obj.get("metadata", {})
  1285. inner = _json_dict(metadata.get("raw")) if isinstance(metadata, dict) else None
  1286. return str(_error_obj(inner).get("message") or "").lower() if inner else ""
  1287. def _build_error_msg(error: Exception, body: Any) -> str:
  1288. """Lowercased str(error) + body message + OpenRouter-wrapped upstream message
  1289. (OpenAI SDK's APIStatusError.__str__ omits the body, so it is appended)."""
  1290. raw_msg = str(error).lower()
  1291. body_msg = metadata_msg = ""
  1292. if isinstance(body, dict):
  1293. err_obj = _error_obj(body)
  1294. body_msg = str(err_obj.get("message") or "").lower() or str(body.get("message") or "").lower()
  1295. metadata_msg = _openrouter_wrapped_message(err_obj) if err_obj else ""
  1296. parts = [raw_msg]
  1297. if body_msg and body_msg not in raw_msg:
  1298. parts.append(body_msg)
  1299. if metadata_msg and metadata_msg not in raw_msg and metadata_msg not in body_msg:
  1300. parts.append(metadata_msg)
  1301. return " ".join(parts)
  1302. def _body_message_candidates(body: dict) -> Iterator[Any]:
  1303. """Body message fields in priority order (OpenAI, flat, litellm/Bedrock proxy, FastAPI shapes)."""
  1304. yield _error_obj(body).get("message")
  1305. yield body.get("message")
  1306. yield body.get("errorMessage")
  1307. args = body.get("errorArgs")
  1308. yield args.get("reason") if isinstance(args, dict) else None
  1309. # FastAPI/Starlette relays and the Codex gateway answer {"detail": "..."} (or a nested
  1310. # OpenAI-ish object); without it a descriptive rejection reads as a bare 400 and the
  1311. # large-session heuristic sends it into compression (#81558). A list here is pydantic's
  1312. # validation shape, read by _oversized_message_content_rejection.
  1313. detail = body.get("detail")
  1314. yield detail.get("message") if isinstance(detail, dict) else detail if isinstance(detail, str) else None
  1315. def _from_cause_chain(error: Exception, pick: Callable[[Any], Any], default: Any) -> Any:
  1316. """First non-None ``pick(exc)`` over the error and its __cause__/__context__ chain (max 5 deep)."""
  1317. current = error
  1318. for _ in range(5):
  1319. found = pick(current)
  1320. if found is not None:
  1321. return found
  1322. cause = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
  1323. if cause is None or cause is current:
  1324. break
  1325. current = cause
  1326. return default
  1327. def _status_of(exc: Any) -> Optional[int]:
  1328. code = getattr(exc, "status_code", None)
  1329. if isinstance(code, int):
  1330. return code
  1331. code = getattr(exc, "status", None) # some SDKs use .status
  1332. return code if isinstance(code, int) and 100 <= code < 600 else None
  1333. def _body_of(exc: Any) -> Optional[dict]:
  1334. body = getattr(exc, "body", None)
  1335. if isinstance(body, dict):
  1336. return body
  1337. response = getattr(exc, "response", None)
  1338. try:
  1339. json_body = response.json() if response is not None else None
  1340. except Exception:
  1341. return None
  1342. return json_body if isinstance(json_body, dict) else None
  1343. def _headers_of(exc: Any) -> Any:
  1344. headers = getattr(getattr(exc, "response", None), "headers", None)
  1345. return headers if headers and hasattr(headers, "get") else None
  1346. def _status_code_from_body(body: Any) -> Optional[int]:
  1347. """Numeric HTTP error status (400-599) from ``error.code``/``code`` in a structured body.
  1348. An aggregator/relay can deliver the upstream failure only this way — as an
  1349. error object inside an HTTP-200 SSE stream — leaving the SDK to raise a
  1350. status-less ``APIError`` whose ``body`` carries the status (#121270). String
  1351. codes stay symbolic (``_code_from_payload``'s ``"400" is not a code``), unlike the
  1352. string-parsing text-SSE sibling ``chat_completion_helpers._status_code_from_payload``."""
  1353. if not isinstance(body, dict):
  1354. return None
  1355. error_obj = _error_obj(body)
  1356. candidates = [error_obj.get(k) for k in ("status_code", "status", "http_status", "code")] + [body.get("code")]
  1357. return next(
  1358. (c for c in candidates if isinstance(c, int) and not isinstance(c, bool) and 400 <= c < 600),
  1359. None,
  1360. )
  1361. def _extract_status_code(error: Exception) -> Optional[int]:
  1362. """HTTP status code from the error or its cause chain; a body-carried numeric
  1363. ``code`` counts when the exception itself carries no status (#121270)."""
  1364. status = _from_cause_chain(error, _status_of, None)
  1365. if status is None:
  1366. status = _status_code_from_body(_from_cause_chain(error, _body_of, {}))
  1367. return status
  1368. def _extract_error_body(error: Exception) -> dict:
  1369. """Structured error body from an SDK exception or its cause chain."""
  1370. return _from_cause_chain(error, _body_of, {})
  1371. def _code_from_payload(payload: Any, top_keys: Sequence[str], peek_message: bool) -> str:
  1372. """Code/type from ``payload.error`` or a top-level key; ``"400"`` is not a code.
  1373. ``peek_message`` also parses a JSON ``error.message`` for a nested code
  1374. (Responses API surfaces ``invalid_encrypted_content`` this way). Gemini
  1375. puts the HTTP status in ``error.code`` and the symbolic code
  1376. (``UNAVAILABLE``) in ``error.status``, so a numeric code defers to it."""
  1377. if not isinstance(payload, dict):
  1378. return ""
  1379. error_obj = payload.get("error", {})
  1380. if isinstance(error_obj, dict):
  1381. code = error_obj.get("code") or error_obj.get("type") or ""
  1382. if not isinstance(code, str):
  1383. code = error_obj.get("status") or code
  1384. if isinstance(code, str) and code.strip() and code.strip() != "400":
  1385. return code.strip()
  1386. message = error_obj.get("message")
  1387. if peek_message and isinstance(message, str) and message.strip().startswith("{"):
  1388. nested_code = _code_from_payload(_json_dict(message), ("code", "error_code"), False)
  1389. if nested_code:
  1390. return nested_code
  1391. code = next((payload.get(k) for k in top_keys if payload.get(k)), "")
  1392. text = str(code).strip() if isinstance(code, (str, int)) else ""
  1393. return text if text and text != "400" else ""
  1394. def _extract_error_code(body: dict) -> str:
  1395. """Extract an error code string from the response body."""
  1396. return _code_from_payload(body, ("code", "error_code", "errorCode"), True) if body else ""
  1397. def _extract_message(error: Exception, body: dict) -> str:
  1398. """Extract the most informative error message (structured body first)."""
  1399. msg = next((m for m in _body_message_candidates(body or {}) if isinstance(m, str) and m.strip()), None)
  1400. return (msg.strip() if msg else str(error))[:500]
  1401. def _is_openrouter_upstream_error(body: Any, provider: str) -> bool:
  1402. """OpenRouter's "Provider returned error" wrapper: the key is healthy, the
  1403. upstream failed, so credential rotation is the wrong recovery."""
  1404. err = _error_obj(body)
  1405. if str(err.get("message") or "").strip().lower() != "provider returned error":
  1406. return False
  1407. if (provider or "").strip().lower() == "openrouter":
  1408. return True
  1409. # Otherwise require the metadata shape only OpenRouter produces.
  1410. metadata = err.get("metadata")
  1411. return isinstance(metadata, dict) and ("raw" in metadata or "provider_name" in metadata)
  1412. def _extract_upstream_provider_name(body: Any) -> Optional[str]:
  1413. """Pull the upstream provider name out of OpenRouter's error metadata."""
  1414. metadata = _error_obj(body).get("metadata")
  1415. name = metadata.get("provider_name") if isinstance(metadata, dict) else None
  1416. return name.strip() if isinstance(name, str) and name.strip() else None