| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637 |
- """API error classification for smart failover and recovery.
- A priority-ordered pipeline maps an API exception to a ``ClassifiedError``
- whose recovery hints (retry, rotate credential, fallback, compress, abort) the
- retry loop in run_agent.py consults instead of re-matching strings itself.
- """
- from __future__ import annotations
- import enum
- import json
- import logging
- import re
- import time
- from dataclasses import dataclass, field
- from typing import Any, Callable, Dict, Iterator, Optional, Sequence
- logger = logging.getLogger(__name__)
- # Synthetic code for the OpenAI SDK rejecting a provider's SSE ``data:`` field
- # before any completion chunk arrives; distinct from generic JSON parse errors.
- PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data"
- # Same rejection with an EMPTY payload: the frame carried no ``data`` at all (``data:`` /
- # ``event: ping`` / ``id:`` with no content). Per the SSE spec those are legal keepalives /
- # no-ops, not malformed payloads — a degraded gateway answers every streaming request with
- # them, so the session switches to non-streaming instead of re-streaming into the same
- # window. See ``chat_completion_helpers._maybe_disable_streaming``.
- PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE = "provider_stream_empty_frame"
- # ── Error taxonomy ──────────────────────────────────────────────────────
- class FailoverReason(enum.Enum):
- """Why an API call failed — determines recovery strategy."""
- auth = "auth" # Transient auth (401/403) — refresh/rotate
- auth_permanent = "auth_permanent" # Auth failed after refresh — abort
- billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately
- rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate
- upstream_rate_limit = "upstream_rate_limit" # Aggregator's upstream model 429 — fallback model, key is healthy
- upstream_blocked = "upstream_blocked" # 403 from a WAF/CDN/proxy in front of the provider — key is healthy, fallback
- overloaded = "overloaded" # 503/529 — provider overloaded, backoff
- server_error = "server_error" # 500/502 — internal server error, retry
- timeout = "timeout" # Connection/read timeout — rebuild client + retry
- ssl_cert_verification = "ssl_cert_verification" # Deterministic TLS chain failure — fail fast with guidance
- context_overflow = "context_overflow" # Context too large — compress, not failover
- payload_too_large = "payload_too_large" # 413 — compress payload
- image_too_large = "image_too_large" # Native image part exceeds provider's per-image limit — shrink and retry
- image_corrupt = "image_corrupt" # Provider can't decode image bytes — strip and retry (shrinking won't help)
- model_not_found = "model_not_found" # 404 or invalid model — fallback to different model
- provider_policy_blocked = "provider_policy_blocked" # Aggregator account data/privacy policy excluded the only endpoint
- data_policy_denied = "data_policy_denied" # Context holds a connector result needing a no-training model; none serves it
- content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — deterministic per-request, don't retry unchanged
- model_entitlement = "model_entitlement" # This account cannot use the requested model — rotate credential (model-scoped), else fall back
- incomplete_response = "incomplete_response" # Codex/Responses turn stuck emitting reasoning only (no answer, no tool call) after replay + nudge — hand to a different provider
- format_error = "format_error" # 400 bad request — abort or strip + retry
- role_alternation = "role_alternation" # Strict chat template rejected adjacent same-role messages — merge them for this destination and retry
- invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry
- multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry
- reasoning_mandatory = "reasoning_mandatory" # Route rejects reasoning: {enabled: false} — send the disable no more this session and retry
- # Provider-specific
- thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid
- long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate
- oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth rejects 1M beta — disable beta and retry
- llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar rejects regex `pattern`/`format` — strip from tools and retry
- unknown = "unknown" # Unclassifiable — retry with backoff
- @dataclass
- class ClassifiedError:
- """Structured classification of an API error with recovery hints."""
- reason: FailoverReason
- status_code: Optional[int] = None
- provider: Optional[str] = None
- model: Optional[str] = None
- message: str = ""
- error_context: dict[str, Any] = field(default_factory=dict)
- # Recovery hints — the retry loop checks these instead of re-classifying.
- retryable: bool = True
- should_compress: bool = False
- should_rotate_credential: bool = False
- should_fallback: bool = False
- @property
- def is_auth(self) -> bool:
- return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
- @property
- def billing_unverified(self) -> bool:
- """True when a ``billing`` verdict rests on an ambiguous body (#82154)."""
- return bool(self.error_context.get("billing_unverified"))
- # ── Pattern tables (lowercased substrings) ──────────────────────────────
- # Billing exhaustion (not transient rate limit). "out of extra usage" is the
- # Anthropic OAuth Pro/Max overage bucket depleted (HTTP 400).
- # The Nous gateway's own words for "the free tier will not serve this" — a billing wall for a
- # named account, the tier refusing for an anonymous one (see ``_WELCOME_403_NAMED_PATTERNS``).
- _FREE_TIER_REFUSAL_PATTERNS = ("model_not_supported_on_free_tier", "not available on the free tier")
- _BILLING_PATTERNS = (
- "insufficient credits", "insufficient_quota", "insufficient balance", "credit balance",
- "credits exhausted", "credits have been exhausted", "requires available credits",
- "account balance is too low", "no usable credits", "top up your credits", "payment required",
- "billing hard limit", "exceeded your current quota", "account is deactivated", "plan does not include",
- "out of extra usage", "out of funds", "run out of funds", "balance_depleted",
- # OpenRouter org-level monthly cap arrives as 403 "Budget limit exceeded (monthly limit)" (#107166):
- # account exhaustion, not a credential problem.
- "budget limit exceeded",
- *_FREE_TIER_REFUSAL_PATTERNS,
- # LiteLLM proxies word a hard cap as "hard billing limit" (structured twin:
- # ``terminal_quota_exhausted`` in _BILLING_ERROR_CODES). "terminal billing
- # limit" free text is NOT matched: substring rules can't negate the
- # "non-terminal billing limit" wording, and the structured code covers it.
- "hard billing limit",
- )
- # Not proof of exhaustion: Anthropic returns the same "out of extra usage" body
- # for a content-filter rejection (#82154). Verdict stays ``billing`` but is
- # marked unverified so surfaces hedge and the pool uses a short cooldown.
- _UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",)
- # xAI's Grok credit-exhaustion code arrives as HTTP 403, not 402. Provider-
- # scoped on purpose: other providers' billing codes on a 403 stay auth failures.
- _XAI_SPENDING_LIMIT_ERROR_CODE = "personal-team-blocked:spending-limit"
- # Nous usage-window plan: included usage (or the large-model allowance) is spent AND
- # there are no top-up credits. Billing so the pool may rotate to a key the user
- # owns; the message names the reset, so it must never read as transient. Honoured
- # only on a Nous 402 (``_classify_402`` behind ``_is_nous_route``), never via
- # _BILLING_ERROR_CODES, so any other status or upstream keeps main's rules.
- _NOUS_USAGE_WINDOW_402_CODE = "usage_window_and_credits_exhausted"
- # Structured codes meaning the account cannot serve paid traffic.
- _BILLING_ERROR_CODES = frozenset({
- "insufficient_quota", "billing_not_active", "payment_required", "insufficient_credits",
- "no_usable_credits", "balance_depleted", "model_not_supported_on_free_tier",
- "member_spend_cap_exceeded", "terminal_quota_exhausted", _XAI_SPENDING_LIMIT_ERROR_CODE,
- # OpenAI (and OpenAI-compatible aggregators) spend/usage-limit family:
- # a credit balance or an org/project spend or usage cap is exhausted —
- # terminal for this credential until limits are raised.
- "credit_balance_exhausted", "organization_spend_limit_exceeded",
- "organization_usage_limit_exceeded", "project_spend_limit_exceeded",
- # Nous paid model behind an empty credit balance arrives as a 404 (#115702).
- "insufficient_credits_for_paid_model",
- })
- # Transient rate limiting. Bedrock "Throttling error: Too many tokens" also
- # contains an overflow phrase; rate limit is matched first so throttle wins.
- _RATE_LIMIT_PATTERNS = (
- "rate limit", "rate_limit", "too many requests", "throttled", "requests per minute",
- "tokens per minute", "requests per day", "try again in", "please retry after",
- "resource exhausted", "resource_exhausted", "resource-exhausted", "resourceexhausted",
- "rate increased too quickly", "throttlingexception", "too many concurrent requests",
- "servicequotaexceededexception", "throttling",
- )
- # Server busy, credential fine: back off on the same key, never rotate. Z.AI/
- # Zhipu reuse HTTP 429 for this, so the 429 path checks these first. Kept narrow
- # so a plain "you have been rate-limited" doesn't land here. (#14038, #15297)
- _OVERLOADED_PATTERNS = (
- "overloaded", "temporarily overloaded", "service is temporarily overloaded",
- "service may be temporarily overloaded", "server is overloaded", "server overloaded",
- "server overload", "server_overload",
- "service overloaded", "service is overloaded", "upstream overloaded", "currently overloaded",
- # CommandCode's 429 body for an unavailable upstream model — the key is healthy (#117111).
- "upstream model provider is temporarily unavailable. please try again in a moment.",
- "at capacity", "over capacity",
- )
- # Usage-limit patterns that need disambiguation (billing OR rate_limit), and
- # the signals that mark such a limit as transient (periodic quota, not billing).
- _USAGE_LIMIT_PATTERNS = ("usage limit", "quota", "limit exceeded", "key limit exceeded")
- _USAGE_LIMIT_TRANSIENT_SIGNALS = (
- "try again", "retry", "resets at", "reset in", "resets in", "reset after", "available in",
- "wait", "requests remaining", "periodic", "window", "per minute", "per second",
- )
- # 413 detected from message text (proxies embed the status or re-wrap
- # Anthropic's "request_too_large" type without one).
- _PAYLOAD_TOO_LARGE_PATTERNS = (
- "request entity too large", "payload too large", "error code: 413", "request_too_large",
- # Normally arrives with an HTTP 413 status (handled by the status path), but aggregators/proxies can
- # re-wrap it into a plain message with no status attribute — route it to the same compression recovery.
- # (port of anomalyco/opencode#37848)
- "request exceeds the maximum size",
- )
- # Request-BODY byte caps that byte-limited gateways report as a 400 (not 413) in wordings the exact list
- # above misses: "RequestSize(bytes): 34021227, Limit(bytes): 33554432", "request body too large, must be
- # less than 20MB" (Novita), "Request size exceeds the limit of {0}" (Z.AI). Without a match they fell to
- # format_error or token-overflow, and an image-heavy session (large in bytes, small in tokens) re-sent
- # the same oversized history. A subject AND an overflow signal are both required so field-level length
- # complaints ("tools[0].description content length exceeds maximum allowed") stay out; a
- # response-scoped subject never matches, because shrinking the request cannot fix a large response.
- _BYTE_SIZE_SUBJECT = re.compile(
- # The overflow verb must follow "... size" directly, so a parameter named there ("request size
- # parameter page_size exceeds the maximum of 100") is not the body.
- r"(?<!response )\b(?:(?:request ?size|(?:request )?(?:body|payload) size)(?: is)? (?:exceed|too large|too big)"
- r"|request body(?: is)? too large|(?:request )?payload(?: is)? too large)"
- )
- _BYTE_SIZE_OVERFLOW = re.compile(r"exceed|too large|too big")
- # A size subject that counts tokens is a token cap ("Request body too large for gpt-4.1 model. Max size:
- # 8000 tokens." from GitHub Models / Azure AI Inference), so it keeps the token rules; only the
- # context-overflow path adopts a quoted limit, and the byte path would compress against the old window.
- _TOKEN_COUNT = re.compile(r"\btokens?\b")
- _REQUEST_WORDS = {"request", "requestsize", "byte", "bytes"}
- def _is_byte_size_limit_message(error_msg: str) -> bool:
- """True when ``error_msg`` reports the request body over a byte-size cap (see above)."""
- msg = (error_msg or "").lower()
- if not _BYTE_SIZE_OVERFLOW.search(msg) or _TOKEN_COUNT.search(msg):
- return False
- if _BYTE_SIZE_SUBJECT.search(msg):
- return True
- # "content length" alone is a field-length complaint unless the request or bytes are named. "_" stays
- # inside a word: the SDK's str() carries the body, whose "invalid_request_error" type is not "request".
- content_length = "content length" in msg or "content-length" in msg
- return content_length and bool(_REQUEST_WORDS & set(re.split(r"[^a-z0-9_]+", msg)))
- # Per-image size/dimension 400s (Anthropic 5 MB / 8000 px; MiniMax "media
- # exceeds size limit" #76039) — a specific 400 before the request hits 413. A
- # non-image media hit is harmless: the shrink pass finds no image parts.
- # "patches after processing": OpenAI Codex Responses rejects an image whose
- # tile-patch budget (ceil(w/32)×ceil(h/32)) exceeds its 30000-patch ceiling
- # with wording that names no image-size vocabulary — without this pattern it
- # fell to format_error (non-retryable), bypassing the shrink recovery (#106337).
- # Byte caps enforced with a 400 instead of a 413 (#112473): NVIDIA NIM caps the whole
- # payload ("Please make sure your payload is below 26214400 bytes in size"); Alibaba
- # DashScope caps the base64 image string via Jackson ("String value length (N) exceeds the
- # maximum allowed (M, from `StreamReadConstraints.getMaxStringLength()`)"). Only an inline
- # image reaches those sizes, so shrinking is the recovery; the method-scoped Jackson token
- # is used because the bare class name also appears when Jackson caps a *token* length.
- _IMAGE_TOO_LARGE_PATTERNS = (
- "image exceeds", "image too large", "image_too_large", "image size exceeds", "image dimensions exceed",
- "dimensions exceed max allowed size", "max allowed size: 8000", "media exceeds", "media too large",
- "patches after processing", "make sure your payload is below", "streamreadconstraints.getmaxstringlength",
- )
- # Undecodable image bytes → strip-and-retry, never shrink. xAI wordings
- # (#69078); the last is the full sentence because shorter fragments also match
- # non-image download failures.
- _IMAGE_CORRUPT_PATTERNS = (
- "invalid png image", "invalid jpeg image", "base64 string of provided image cannot be decoded",
- "downloaded response does not contain a valid jpg, png, webp, or ico image",
- )
- # 400s rejecting list-type ``content`` in tool messages (Xiaomi MiMo "text is
- # not set", Alibaba, OpenAI-compat long tail). Recovery: strip image parts from
- # tool messages, remember (provider, model), retry. (#27344)
- # NVIDIA NIM's Rust gateway never names the field: its serde rejection says the
- # body "did not match any variant of untagged enum
- # ChatCompletionRequestToolMessageContent", which is the same list-type tool
- # content that every other wording here describes (#111231).
- _MULTIMODAL_TOOL_CONTENT_PATTERNS = (
- "text is not set", "tool message content must be a string", "tool content must be a string",
- "tool message must be a string", "expected string, got list", "expected string, got array",
- # Console Go / pydantic-v2 relays behind opencode-go (422, param ``messages.N.tool.content.str``, #104731).
- "tool_call.content must be string", "tool.content.str", "input should be a valid string",
- "chatcompletionrequesttoolmessagecontent",
- )
- # Local-inference memory/resource-ceiling rejections (oMLX/MLX memory guard,
- # llama.cpp/vLLM OOM, Metal/CUDA allocation ceilings). The server aborts on a
- # prefill memory PEAK, not a window limit, yet its remediation tail says
- # "reduce context length" — so without this list the request routes into
- # compression, which cannot lower a prefill peak: it burns the compression
- # budget, re-hits the wedged server each attempt and ends in a session reset.
- # Every token names memory/allocation in BYTES, never a token count, so the
- # list is disjoint from _CONTEXT_OVERFLOW_PATTERNS. Must be checked BEFORE
- # both overflow AND the usage-limit disambiguation ("memory limit exceeded"
- # contains "limit exceeded", which would otherwise read as billing). oMLX
- # reworded the accounting sentence in 0.5.7 ("predicted peak would require /
- # exceed"); the 0.5.6 wording is still in the field, so both stay. (#52261)
- _MEMORY_CEILING_PATTERNS = (
- "memory guard", "memory limit exceeded", "memory_guard_tier", "dynamic ceiling",
- "memory ceiling", "available memory", "out of memory", "insufficient memory",
- "prefill would require", "predicted peak would", "prefill safety cap", "metal_cap",
- )
- # Structured codes identifying the same rejection at the source, before an
- # OpenAI-compatible proxy flattens the body and drops the wording.
- _MEMORY_CEILING_ERROR_CODES = frozenset({
- "prefill_memory_exceeded", "prefill_memory_aborted", "omlx_prefill_memory_exceeded",
- })
- # Bare "max_tokens" is load-bearing: the output-cap-retry path keys off it;
- # empty-response advisories mentioning it are intercepted earlier. Groups:
- # generic; vLLM; Ollama; llama.cpp; Chinese; Z.AI (1210); Bedrock; Together.
- _CONTEXT_OVERFLOW_PATTERNS = (
- "context length", "context size", "maximum context", "token limit", "too many tokens",
- "reduce the length", "exceeds the limit", "context window", "prompt is too long",
- "prompt exceeds max length", "max_tokens", "maximum number of tokens",
- "exceeds the max_model_len", "max_model_len", "prompt length", "input is too long", "maximum model length",
- "context length exceeded", "truncating input",
- "slot context", "n_ctx_slot",
- "超过最大长度", "上下文长度",
- "tokens in request more than max tokens allowed",
- "input is too long", "max input token", "input token", "exceeds the maximum number of input tokens",
- # Together/Fireworks-style: "Input length 131393 exceeds the maximum allowed input length of 131040
- # tokens." No other pattern in this list matches that wording. (port of anomalyco/opencode#37848)
- "maximum allowed input length",
- )
- # Last entry: OpenRouter 404 when no endpoint supports tool calling —
- # model_not_found triggers fallback instead of burning retries (#58446).
- # Codex ChatGPT-account entitlement 400 — the account can never use the named slug (#71970, #106475).
- CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER = "model is not supported when using codex with a chatgpt account"
- _MODEL_NOT_FOUND_PATTERNS = (
- "is not a valid model", "invalid model", "model not found", "model_not_found", "does not exist",
- "no such model", "unknown model", "unsupported model", "no endpoints found that support tool use",
- # Nous 404 when the provider retires a :free route — the slug is dead for every
- # credential, so fall back instead of burning retries (#123180). Unlike the free-tier
- # billing wording, the account's balance/tier is not what rejected the call.
- "is no longer free",
- )
- # Qwen/vLLM chat-template "No user query found". Shared by the invalid-body
- # table (→ format_error) and the llama.cpp grammar guard so they cannot drift.
- _NO_USER_QUERY_SIGNAL = "no user query found"
- # Deterministic rejections of the *transcript* (e.g. a content-less assistant
- # stub after a dead stream). NOT overflow — input may be tiny and compression
- # cannot invent a missing turn — so fail fast as format_error.
- _INVALID_MESSAGE_BODY_PATTERNS = (
- "must have non-empty content", "messages must have non-empty", "invalid_request_body",
- "text content blocks must be non-empty", "content field is required",
- "messages: at least one message is required", _NO_USER_QUERY_SIGNAL,
- )
- # Strict-alternation chat templates (llama.cpp / vLLM Jinja templates, Mistral, some
- # OpenRouter routes) 400 when two adjacent messages share a role. Deterministic for the
- # request shape, and the only bad thing is the adjacency, so the caller that produced it
- # (the MoA aggregator appends ``user(guidance)`` after ``user(task)`` on iteration 1 —
- # #112358) merges the pair for THAT destination and retries once. Checked before the
- # request-validation table: the body usually also carries ``invalid_request_error``.
- _ROLE_ALTERNATION_PATTERNS = (
- "roles must alternate", "role must alternate", "must alternate between",
- "consecutive user messages", "consecutive messages with the same role",
- "consecutive messages of the same role", "same role in a row", "multiple user messages in a row",
- "adjacent messages with the same role",
- )
- # Proxy-side rejection of the model's own tool-call JSON (Ollama "invalid tool call arguments",
- # OpenRouter-wrapped "function_call arguments"). Checked before the generic 400 validation and
- # overflow heuristics: on a large session the bare message would otherwise read as overflow.
- _MALFORMED_TOOL_ARGS_PATTERNS = (
- "invalid tool call arguments", "invalid tool_call arguments", "invalid tool_calls arguments",
- "invalid function call arguments", "invalid function_call arguments",
- "tool call arguments are invalid", "tool_call arguments are invalid",
- "function call arguments are invalid", "function_call arguments are invalid",
- )
- # Malformed request, identical on every retry. Some gateways (codex.nekos.me)
- # return these as 5xx, so the 5xx path also checks them.
- _REQUEST_VALIDATION_PATTERNS = (
- "unknown parameter", "unsupported parameter", "unrecognized request argument",
- "invalid_request_error", "unknown_parameter", "unsupported_parameter",
- )
- # Parameters Hermes sends on SOME routes only → hosts where that is deliberate.
- # A rejection from any other host means the provider's gateway injected the
- # field itself: a server-side flake, not our request shape. prompt_cache_retention
- # is only sent for api.meta.ai / bedrock-mantle (agent/transports/codex.py).
- _SERVER_INJECTED_PARAM_SENDERS: dict[str, tuple] = {
- "prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
- }
- _PARAM_REJECTION_WORDS = ("not supported", "unsupported", "unknown", "unrecognized")
- # Anthropic thinking-block 400 wordings (see _provider_special_cases).
- _THINKING_MUTATION_WORDS = ("signature", "cannot be modified", "must remain as they were")
- def _is_anthropic_thinking_rejection(msg: str) -> bool:
- """A thinking block the upstream will not verify: a signature broken by transcript mutation, or a
- ``redacted_thinking`` blob it cannot open (another model's sealed reasoning after a /model switch or
- fallback: ``Invalid `data` in `redacted_thinking` block``). Both heal by stripping the replayed blocks."""
- return "thinking" in msg and (
- any(p in msg for p in _THINKING_MUTATION_WORDS) or ("redacted_thinking" in msg and "invalid" in msg))
- # Local MoA streaming adapter-shape bugs (see _moa_special_cases).
- _MOA_ADAPTER_SHAPE_BUGS = (
- "'types.SimpleNamespace' object is not iterable", "'types.SimpleNamespace' object has no attribute 'index'",
- )
- # OpenRouter 404 when the account data policy excludes the only endpoint. Not
- # model_not_found: the model exists, fallback can't help, body has the fix URL.
- _PROVIDER_POLICY_BLOCKED_PATTERNS = (
- "no endpoints available matching your guardrail", "no endpoints available matching your data policy",
- "no endpoints found matching your data policy",
- )
- # Upstream account ban relayed by an aggregator, often as HTTP 200 + an SSE error
- # event (no status): permanent for this account, so never the transient retry ladder.
- _ACCOUNT_POLICY_BLOCK_PATTERNS = ("blocked for a previous policy violation",)
- # Per-prompt safety-filter blocks: deterministic for the unchanged request, so
- # fallback immediately. Each phrase is verbatim from one provider (Codex cyber
- # flags #18028, OpenAI moderation, Anthropic safety, Azure token, MiniMax
- # #32421, CommandCode gateway moderation #115218, Alibaba Model Studio
- # guardrail `data_inspection_failed`, input and output side) — never a generic
- # word like "policy" that collides with billing/auth.
- # "content_filter" deliberately excludes the space variant seen in echoed config.
- _CONTENT_POLICY_BLOCKED_PATTERNS = (
- "flagged for possible cybersecurity risk", "trusted access for cyber",
- "violates our usage policies", "violates openai's usage policies", "your request was flagged by",
- "prompt was flagged by our safety", "responses cannot be generated due to safety",
- "content_filter", "responsibleaipolicyviolation", "new_sensitive",
- "content exists risk", "data may contain inappropriate content", "data_inspection_failed",
- )
- # Auth patterns (non-status-code signals).
- _AUTH_PATTERNS = (
- "invalid api key", "invalid_api_key", "gateway_auth_failed", "authentication", "unauthorized",
- "forbidden", "invalid token", "token expired", "token revoked", "access denied",
- # Codex backend rejecting an OAuth access token without a usable
- # ``chatgpt_account_id`` claim; arrives as a bare ``detail`` string.
- "failed to extract accountid from token",
- )
- # Empty-response advisories (OpenRouter / nano-gpt). Checked before overflow
- # because the text often mentions "max_tokens" (caused compression spirals).
- _EMPTY_PROVIDER_RESPONSE_PATTERNS = (
- "returned an empty response", "empty response despite retries", "provider returned an empty response",
- "model returning empty responses", "empty response stream",
- )
- # Timeout wording from generic exception types the type heuristics would miss.
- _TIMEOUT_MESSAGE_PATTERNS = (
- "timed out", "turn timed out", "request timed out", "deadline exceeded", "operation timed out",
- "upstream timed out",
- )
- # Connect/DNS failures from generic exception types with no status. EXCLUDES
- # mid-stream disconnects (_SERVER_DISCONNECT_PATTERNS may route large sessions
- # to compression; a never-established connection cannot be an overflow).
- # Groups: TCP connect; DNS (Python/glibc/macOS/Node); undici bridge; Envoy.
- _CONNECTION_MESSAGE_PATTERNS = (
- "connection refused", "econnrefused", "no route to host", "network is unreachable", "network unreachable",
- "name or service not known", "temporary failure in name resolution", "nodename nor servname provided",
- "getaddrinfo failed", "getaddrinfo enotfound", "eai_again",
- "fetch failed", "failed to fetch",
- "upstream connect error",
- )
- # SSL names keep provider-wrapped SSL errors (chain lost) as transport, not
- # unknown; OpenAI SDK errors are not subclasses of Python builtins.
- _TRANSPORT_ERROR_TYPES = frozenset({
- "ReadTimeout", "ConnectTimeout", "PoolTimeout", "ConnectError", "RemoteProtocolError",
- "ConnectionError", "ConnectionResetError", "ConnectionAbortedError", "BrokenPipeError",
- "TimeoutError", "ReadError", "ServerDisconnectedError",
- "SSLError", "SSLZeroReturnError", "SSLWantReadError", "SSLWantWriteError", "SSLEOFError", "SSLSyscallError",
- "APIConnectionError", "APITimeoutError",
- })
- # Ambiguous disconnects (no status): transient hiccup OR a gateway dropping an
- # oversized request. A large session + one of these → context-overflow path.
- _SERVER_DISCONNECT_PATTERNS = (
- "server disconnected", "peer closed connection", "connection reset by peer", "connection was closed",
- "network connection lost", "unexpected eof", "incomplete chunked read",
- )
- # Deterministic cert failures (proxy, missing CA, expired/self-signed) — fail
- # fast. Checked BEFORE _SSL_TRANSIENT_PATTERNS: these also contain "[SSL:".
- _SSL_CERT_VERIFY_PATTERNS = (
- "certificate verify failed", "certificate_verify_failed", "unable to get local issuer certificate",
- "self-signed certificate", "self signed certificate", "certificate has expired",
- "hostname mismatch, certificate is not valid", "unable to verify the first certificate",
- )
- # Transient SSL alerts: retry but NOT compression (kept apart from disconnects).
- # Both space and underscore forms because OpenSSL 3 changed token separators
- # (SSLV3_ALERT_... → SSL/TLS_ALERT_...); "[ssl:" is the Python ssl prefix.
- _SSL_TRANSIENT_PATTERNS = (
- "bad record mac", "ssl alert", "tls alert", "ssl handshake failure", "tlsv1 alert", "sslv3 alert",
- "bad_record_mac", "ssl_alert", "tls_alert", "tls_alert_internal_error", "[ssl:",
- )
- # A 403 body written by a WAF/CDN/proxy rather than the provider's API: Cloudflare's browser
- # challenge and block pages, plus the plain-text block relays return when they reject the SDK
- # User-Agent (#53099). Matched only on 403 (see ``_status_403``); a bare "access denied" or
- # "forbidden" stays auth because providers word real permission errors that way too.
- _UPSTREAM_BLOCKED_PATTERNS = (
- "your request was blocked", "request blocked", "sorry, you have been blocked",
- "enable javascript and cookies to continue", "cdn-cgi/challenge-platform", "cf-browser-verification",
- "challenge-error-text", "__cf_chl", "cf-error-details", "attention required! | cloudflare",
- )
- # ── Verdicts and rule tables ────────────────────────────────────────────
- # A verdict is the ClassifiedError kwargs a stage decided on: ``reason`` plus
- # hint overrides (unlisted hints keep dataclass defaults). Rule tables are
- # ordered ``(patterns, verdict)`` pairs matched first-hit; ``verdict`` may be
- # a callable of the error message.
- Verdict = dict[str, Any]
- def _v(reason: FailoverReason, **hints: Any) -> Verdict:
- return {"reason": reason, **hints}
- _ROTATE_FALLBACK = {"should_rotate_credential": True, "should_fallback": True}
- _ABORT_FALLBACK = {"retryable": False, "should_fallback": True}
- _R = FailoverReason
- _V_BILLING = _v(_R.billing, retryable=False, **_ROTATE_FALLBACK)
- _V_RATE_LIMIT = _v(_R.rate_limit, **_ROTATE_FALLBACK)
- _V_AUTH_ROTATE = _v(_R.auth, retryable=False, **_ROTATE_FALLBACK)
- _V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK)
- _V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK)
- _V_UPSTREAM_BLOCKED = _v(_R.upstream_blocked, **_ABORT_FALLBACK)
- _V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK)
- # Another account in the same pool may hold the entitlement; the credential itself is healthy.
- _V_MODEL_ENTITLEMENT = _v(_R.model_entitlement, retryable=False, **_ROTATE_FALLBACK)
- _V_FORMAT_ERROR = _v(_R.format_error, **_ABORT_FALLBACK)
- # A different provider (direct instead of the aggregator; another host's TLS chain) can fix these.
- _V_POLICY_BLOCKED = _v(_R.provider_policy_blocked, **_ABORT_FALLBACK)
- _V_SSL_CERT = _v(_R.ssl_cert_verification, **_ABORT_FALLBACK)
- _V_CONTEXT_OVERFLOW = _v(_R.context_overflow, should_compress=True)
- _V_PAYLOAD_TOO_LARGE = _v(_R.payload_too_large, should_compress=True)
- _V_OVERLOADED, _V_SERVER_ERROR, _V_TIMEOUT, _V_UNKNOWN = map(_v, (_R.overloaded, _R.server_error, _R.timeout, _R.unknown))
- _V_IMAGE_TOO_LARGE, _V_IMAGE_CORRUPT = _v(_R.image_too_large), _v(_R.image_corrupt)
- _V_MULTIMODAL, _V_INVALID_ENCRYPTED = _v(_R.multimodal_tool_content_unsupported), _v(_R.invalid_encrypted_content)
- _V_REASONING_MANDATORY = _v(_R.reasoning_mandatory, should_compress=False, should_fallback=False)
- # Same recovery hints as format_error: consumers without a merge-and-retry step (the main loop
- # already merges adjacent users before the call) keep aborting to the fallback chain.
- _V_ROLE_ALTERNATION = _v(_R.role_alternation, **_ABORT_FALLBACK)
- # The MODEL emitted unparseable tool-call JSON and the proxy (Ollama, OpenRouter) rejected it: no
- # other provider can fix that output, so falling back only replays the same broken turn 4-5 times
- # (20-60s per occurrence, #12770). Abort this call; the loop's argument repair handles the retry.
- _V_MALFORMED_TOOL_ARGS = _v(_R.format_error, retryable=False, should_fallback=False)
- # A reasoning-mandatory route answering ``reasoning: {enabled: false}`` (Nous Portal + OpenRouter wording).
- _REASONING_MANDATORY_PATTERN = "reasoning is mandatory"
- # Generic markers a provider 400 puts next to the offending parameter name. Bedrock Converse
- # rejects sampling params for reasoning-first models with the contraction ("This model doesn't
- # support the temperature field", xAI Grok) and inference-profile Claude with "`temperature` is
- # deprecated for this model" (#111043); strict pydantic gateways (Fireworks) name the unknown
- # field as "extra inputs are not permitted" (#109774). Enum-rejecting aggregators (commandcode.ai)
- # say "Invalid option: expected one of ..." with no "unsupported" anywhere, naming the field only
- # in the structured 'param' tail (#115277). Shared with the auxiliary retry ladder
- # (``agent.auxiliary_client._is_unsupported_parameter_error``).
- UNSUPPORTED_PARAM_MARKERS = (
- "unsupported parameter", "unsupported_parameter", "not supported", "does not support",
- "doesn't support", "is deprecated for this model",
- "unknown parameter", "unrecognized request argument", "unrecognized parameter",
- "invalid parameter", "extra inputs are not permitted",
- "invalid option: expected one of",
- )
- # Reasoning wire-field names (the profile reasoning controls minus ``verbosity``), longest first.
- # Standalone only: never a model-id segment ("The model kimi-k2-thinking is not supported when
- # using this account" is route gating for the provider-fallback rung) nor the adjective in
- # "... not supported with reasoning models".
- _REASONING_FIELD_TOKEN = re.compile(
- r"(?<![\w\-/])(?:reasoning_effort|thinking_config|thinking_budget|enable_thinking|thinkingconfig"
- r"|thinkingbudget|reasoning|thinking|think)(?![\w\-/])(?!\s+models?\b)"
- )
- # Structured rejection of a reasoning field, read from the stringified body: OpenAI-style
- # ``param`` naming a reasoning field (``reasoning_effort`` on chat, ``reasoning.effort`` on
- # Responses) or an ``invalid_reasoning_effort`` code. Custom Responses relays send this with NO
- # message at all (#100536), so no wording rule can match it — and without a match the message-less
- # 400 fell through to the generic large-session overflow heuristic and started compression.
- _REASONING_PARAM_REJECTION = re.compile(
- r"""['"]param['"]\s*:\s*['"](?:reasoning(?:[._]effort)?|thinking(?:_config|_budget)?|enable_thinking)['"]"""
- r"""|invalid_reasoning_effort"""
- )
- _REASONING_REQUIRED_MARKERS = (
- "mandatory", "cannot be disabled", "can't be disabled", "must be enabled", "is required",
- "always enabled", "cannot be turned off",
- )
- # Same rejection, wording by *value*: an endpoint that publishes a closed level set refuses the
- # thinking-OFF encoding because ``none`` is not in it — "'reasoning_effort' must be one of:
- # 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra'" (Alibaba token-plan / DashScope
- # routes, code ``invalid_parameter_error``). The field is understood and has to stay on the wire;
- # only the disabled level is invalid, so the floor step-up (the weakest accepted level) is the
- # right recovery. Neither the strip rung nor its "unsupported" wording matches this message, so
- # without these markers the title lane fails outright instead of retrying at the floor.
- _REASONING_LEVEL_REJECTION_MARKERS = ("must be one of", "invalid value for")
- def is_reasoning_level_rejection(error_msg: str) -> bool:
- """Provider 400 rejecting the *value* of a reasoning wire control against the endpoint's closed
- level set ("'reasoning_effort' must be one of: 'minimal', 'low', …", Alibaba token-plan /
- DashScope routes, code ``invalid_parameter_error``). The reasoning-field token must be present
- next to the wording — a 400 enum complaint about some other field is not this class. The field is
- understood and has to stay on the wire; only the disabled level ("none") is invalid, so the
- recovery is the floor step-up in the auxiliary ladder and the disable-drop in the main loop, not
- the strip rung and not an abort to the fallback chain."""
- msg = (error_msg or "").lower()
- token = _REASONING_FIELD_TOKEN.search(msg)
- if token is None:
- return False
- near = msg[max(0, token.start() - 48):token.end() + 96]
- return any(m in near for m in _REASONING_LEVEL_REJECTION_MARKERS)
- def is_reasoning_required_rejection(error_msg: str) -> bool:
- """Provider 400 saying the model's reasoning cannot be switched OFF ("Reasoning is mandatory for
- this endpoint and cannot be disabled", the Nous Portal on gpt-6-astra) — or refusing the OFF
- *level* because the endpoint's closed set starts at ``minimal``
- (``is_reasoning_level_rejection``). The opposite of ``is_reasoning_field_rejection``: the field is
- understood, the *disable* is refused, so the right reaction is to step the effort up to the
- lowest level rather than drop the field (a dropped field also works, but tells the caller nothing
- about the next call)."""
- msg = (error_msg or "").lower()
- token = _REASONING_FIELD_TOKEN.search(msg)
- if token is None:
- return False
- near = msg[max(0, token.start() - 48):token.end() + 96]
- return any(m in near for m in _REASONING_REQUIRED_MARKERS) or is_reasoning_level_rejection(msg)
- def is_reasoning_field_rejection(error_msg: str) -> bool:
- """Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
- ``thinking``/``think``): the field token plus either a generic unsupported marker ("Unrecognized
- request argument supplied: reasoning_effort", #112781) or a standalone "unsupported" next to the
- field in either word order ("unsupported reasoning_effort"; "reasoning_effort 'none' unsupported;
- use minimal|low|medium|high|xhigh", #114460). The route default is the right answer for such a
- model, so both the main loop and the auxiliary ladder retry once without the disable. A body
- whose structured ``param``/code names the reasoning field (``'param': 'reasoning.effort'``,
- ``invalid_reasoning_effort``, #100536) is a rejection whatever the message says — even none.
- Known trade-off: a 400 about a thinking *state* ("Function calling is not supported when
- thinking is enabled") also matches — the marker sits right next to the token, so no proximity
- rule separates it from the forward wordings. Cost is one dropped-disable retry before the
- spent path takes the fallback chain; the auxiliary ladder already treated it this way."""
- msg = (error_msg or "").lower()
- if _REASONING_PARAM_REJECTION.search(msg):
- return True
- token = _REASONING_FIELD_TOKEN.search(msg)
- if token is None:
- return False
- near = msg[max(0, token.start() - 32):token.end() + 32]
- return "unsupported" in near or any(m in msg for m in UNSUPPORTED_PARAM_MARKERS)
- def _billing_hints(error_msg: str) -> Verdict:
- """Billing verdict carrying the #82154 ambiguity marker when applicable."""
- ctx: dict[str, Any] = {}
- if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS):
- ctx = {"billing_unverified": True, "possible_content_filter": True}
- return {**_V_BILLING, "error_context": ctx}
- def _first_match(error_msg: str, rules: Sequence[tuple[Sequence[str], Any]]) -> Optional[Verdict]:
- """Verdict of the first rule whose pattern list hits ``error_msg``."""
- for patterns, verdict in rules:
- if any(p in error_msg for p in patterns):
- return verdict(error_msg) if callable(verdict) else verdict
- return None
- # Image/tool-content 400s, ordered: multimodal recovery ≠ image shrink; corrupt
- # bytes need strip not shrink; image-shrink is cheaper than context compression.
- _IMAGE_TOOL_RULES = (
- (_MULTIMODAL_TOOL_CONTENT_PATTERNS, _V_MULTIMODAL), (_IMAGE_CORRUPT_PATTERNS, _V_IMAGE_CORRUPT),
- (_IMAGE_TOO_LARGE_PATTERNS, _V_IMAGE_TOO_LARGE),
- )
- # Overflow signals arriving as 5xx (llama.cpp reports overflow as 500; busy /
- # model-load OOM as 503). Empty-response advisories must not enter compression.
- _OVERFLOW_AS_5XX_RULES = (
- (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR), (_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
- (_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW),
- )
- # 404: Nous API surfaces credit depletion as a paid model vanishing from the
- # Free Tier (billing, not missing model); policy block before model_not_found.
- _404_RULES = (
- (_BILLING_PATTERNS, _V_BILLING), (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED),
- (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
- )
- # 400 tail after the deterministic request-shape checks. Some providers return
- # model-not-found / rate-limit / billing as 400 instead of 404/429/402.
- _400_TAIL_RULES = _OVERFLOW_AS_5XX_RULES + (
- (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
- (_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_BILLING_PATTERNS, _billing_hints),
- )
- # LM Studio / llama.cpp raise a bare status-less ``APIError`` when chat-template Jinja rendering
- # fails mid-stream (#62662). Deterministic for the request, so fall back instead of retrying.
- _STREAM_RENDER_ERROR_PATTERNS = ("error rendering", "rendering prompt", "jinja template", "jinja render")
- # Status-less message path, head (before usage-limit disambiguation).
- _MESSAGE_HEAD_RULES = ((_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
- (_PAYLOAD_TOO_LARGE_PATTERNS, _V_PAYLOAD_TOO_LARGE),
- (_ROLE_ALTERNATION_PATTERNS, _V_ROLE_ALTERNATION),
- (_STREAM_RENDER_ERROR_PATTERNS, _V_FORMAT_ERROR)) + _IMAGE_TOOL_RULES
- # Status-less tail. Overload before rate_limit/billing so "overloaded" backs off
- # instead of rotating; policy block before model_not_found; timeout/connection
- # wording last, classified as transport (never compression).
- _MESSAGE_TAIL_RULES = (
- (_OVERLOADED_PATTERNS, _V_OVERLOADED), (_BILLING_PATTERNS, _billing_hints),
- (_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR),
- (_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW), (_AUTH_PATTERNS, _V_AUTH_ROTATE),
- (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
- (_TIMEOUT_MESSAGE_PATTERNS, _V_TIMEOUT), (_CONNECTION_MESSAGE_PATTERNS, _V_TIMEOUT),
- )
- # Structured error code → verdict. The error-code rate_limit verdict rotates
- # but does not set should_fallback (unlike the message/status paths).
- _ERROR_CODE_VERDICTS: dict[str, Verdict] = {
- **dict.fromkeys(("resource_exhausted", "throttled", "rate_limit_exceeded"),
- _v(_R.rate_limit, should_rotate_credential=True)),
- **dict.fromkeys(_BILLING_ERROR_CODES, _V_BILLING),
- **dict.fromkeys(("model_not_found", "model_not_available", "invalid_model"), _V_MODEL_NOT_FOUND),
- **dict.fromkeys(("context_length_exceeded", "max_tokens_exceeded"), _V_CONTEXT_OVERFLOW),
- **dict.fromkeys(_MEMORY_CEILING_ERROR_CODES, _V_OVERLOADED),
- "invalid_encrypted_content": _V_INVALID_ENCRYPTED,
- }
- # Provider-native status codes that arrive as a bare ``{"error": {"code": …}}`` body
- # (no HTTP status, no prose): gRPC canonical names from Gemini, Anthropic error
- # types, OpenAI's ``server_error``. Scoped per provider so a coincidentally named
- # code from another backend stays ``unknown`` (#70414). Provider aliases collapse
- # to the family key before lookup.
- _PROVIDER_CODE_FAMILIES = {"openai-codex": "openai", "google": "gemini", "google-gemini": "gemini",
- "google-ai-studio": "gemini", "vertex": "gemini", "google-vertex": "gemini"}
- _PROVIDER_CODE_VERDICTS: dict[str, dict[str, Verdict]] = {
- "openai": {"server_error": _V_SERVER_ERROR},
- "gemini": {"unavailable": _V_OVERLOADED, "deadline_exceeded": _V_TIMEOUT, "internal": _V_SERVER_ERROR},
- "anthropic": {"api_error": _V_SERVER_ERROR, "rate_limit_error": _V_RATE_LIMIT},
- }
- # Generic ``invalid_request_error`` is deliberately NOT a 400 validation
- # signal — OpenAI stamps it on genuine overflow 400s too.
- _400_VALIDATION_CODES = {"unknown_parameter", "unsupported_parameter"}
- _5XX_VALIDATION_CODES = _400_VALIDATION_CODES | {"invalid_request_error"}
- _400_VALIDATION_PATTERNS = tuple(p for p in _REQUEST_VALIDATION_PATTERNS if p != "invalid_request_error")
- # ── Classification pipeline ─────────────────────────────────────────────
- @dataclass
- class _Ctx:
- """Everything the classifier stages need about one failed call."""
- error: Exception
- status_code: Optional[int]
- body: dict
- msg: str # lowercased str(error) + body message(s)
- provider: str # as passed by the caller
- model: str
- approx_tokens: int
- context_length: int
- num_messages: int
- base_url: str = "" # the route the call went to; "" when the caller did not say
- anonymous: bool = False
- def __post_init__(self) -> None:
- self.error_type = type(self.error).__name__
- self.error_code = _extract_error_code(self.body)
- self.code = self.error_code.lower()
- self.headers = _from_cause_chain(self.error, _headers_of, {})
- self.provider_slug = (self.provider or "").strip().lower()
- self.model_slug = (self.model or "").strip().lower()
- def large_session(self, frac: float, tokens: int, messages: int) -> bool:
- """Absolute thresholds only proxy for smaller context windows."""
- return self.approx_tokens > self.context_length * frac or (
- self.context_length <= 256000 and (self.approx_tokens > tokens or self.num_messages > messages)
- )
- def _plugin_verdict(c: _Ctx) -> Optional[Verdict]:
- """First valid plugin classification (runs before the built-in pipeline so a
- provider plugin can add or correct verdicts). invoke_hook isolates callback
- failures; this guard only covers import/dispatch failure."""
- try:
- from hermes_cli.plugins import get_plugin_error_classification
- verdict = get_plugin_error_classification(
- provider=c.provider, model=c.model, status_code=c.status_code, error_type=c.error_type,
- error_code=c.error_code, error_message=c.msg, error_body=c.body, error=c.error,
- approx_tokens=c.approx_tokens, context_length=c.context_length, num_messages=c.num_messages,
- )
- except Exception as exc:
- logger.debug("Plugin error classification unavailable: %s", exc)
- return None
- if verdict is not None:
- logger.info("API error classified by plugin hook: %s (provider=%s, status=%s)",
- verdict["reason"].value, c.provider, c.status_code)
- return verdict
- def _profile_verdict(c: _Ctx) -> Optional[Verdict]:
- """The current provider's own ``ProviderProfile.classify_api_error`` verdict, or None.
- A ``kind: model-provider`` plugin never enters the PluginManager hook lifecycle, so without this a
- vendor-specific body (a 403 ``quota_exhausted`` that is billing, not auth) could only be corrected by
- shipping a second plugin component. Scoped to the provider that produced the error; no name table."""
- if not c.provider_slug:
- return None
- try:
- from providers import get_provider_profile
- hook = getattr(get_provider_profile(c.provider_slug), "classify_api_error", None)
- if not callable(hook):
- return None
- result = hook(c.error, status_code=c.status_code, error_code=c.error_code, message=c.msg,
- body=c.body, model=c.model)
- except Exception as exc:
- logger.debug("Provider profile error classification failed for %s: %s", c.provider_slug, exc)
- return None
- if not isinstance(result, dict):
- return None
- reason = result.get("reason")
- if isinstance(reason, str):
- try:
- reason = FailoverReason(reason.strip().lower())
- except ValueError:
- return None
- if not isinstance(reason, FailoverReason):
- return None
- hints = {k: bool(result[k]) for k in _HINT_FLAGS if k in result}
- # turn_api_error walks the fallback chain only for non-retryable verdicts outside
- # RETRYABLE_CLIENT_REASONS; the built-in terminal verdicts (billing, auth, model_not_found …) pin
- # retryable=False, the rate-limit family stays retryable and reaches fallback after backoff. Give
- # a hook that asks for fallback the built-in default for its reason, so it cascades like one.
- if hints.get("should_fallback") and "retryable" not in hints and reason not in RETRYABLE_CLIENT_REASONS:
- hints["retryable"] = False
- verdict = _v(reason, **hints)
- if isinstance(result.get("error_context"), dict):
- verdict["error_context"] = result["error_context"]
- logger.info("API error classified by provider profile: %s (provider=%s, status=%s)",
- reason.value, c.provider, c.status_code)
- return verdict
- _HINT_FLAGS = ("retryable", "should_compress", "should_rotate_credential", "should_fallback")
- # Reasons the retry loop keeps retrying (with backoff) even though the verdict may also carry
- # ``should_fallback``: the cascade for these runs after the backoff budget, not immediately.
- RETRYABLE_CLIENT_REASONS = frozenset({
- FailoverReason.rate_limit, FailoverReason.upstream_rate_limit, FailoverReason.overloaded,
- FailoverReason.context_overflow, FailoverReason.payload_too_large, FailoverReason.long_context_tier,
- FailoverReason.thinking_signature,
- })
- # A welcome-host 403 that spells one of these out is a safety block or a billing wall, not the
- # tier refusing. The free-tier refusal phrases are left OUT: on the free route they mean exactly
- # "the tier refused", and an anonymous session has no credits to check.
- _WELCOME_403_NAMED_PATTERNS = _CONTENT_POLICY_BLOCKED_PATTERNS + tuple(
- p for p in _BILLING_PATTERNS if p not in _FREE_TIER_REFUSAL_PATTERNS)
- def _nous_welcome_tier(c: _Ctx) -> Optional[Verdict]:
- """The Nous inference gateway's welcome-tier (free tier) refusals, read from the structured body.
- A 429 carrying a fairshare ``reason`` is either a tier gate (``model_not_free`` /
- ``feature_not_free``: the model or feature is never served on the free tier, so retrying is
- pointless — abort this route and fall back) or capacity (``at_capacity`` / ``admission_closed``
- / ``rate_limited``: honour ``retry_after``, never rotate the free tier's only credential). A
- 400/403 whose message names the wrong host or a dark tier is deterministic for the request.
- The parsed refusal rides ``error_context`` so the terminal copy can say what happened.
- """
- from hermes_cli.anon_auth import (
- WELCOME_TIER_GATE_REASONS, parse_welcome_refusal, route_is_welcome_host, welcome_route_refusal)
- status = c.status_code
- if not c.anonymous:
- # A named credential's fairshare 429 is an ordinary rate limit, whatever its body says. The
- # one welcome refusal it does receive is the gateway's mirror 400 on the welcome host; its
- # reconnect copy stands, only the sign-in card is withheld (``_welcome_surface_kind``).
- if c.provider == "nous" and status == 400 and welcome_route_refusal(status, c.msg) == "named_on_welcome_host":
- return _v(_R.format_error, retryable=False, should_fallback=True,
- error_context={"welcome_route": "named_on_welcome_host"})
- return None
- if status == 402 and route_is_welcome_host(c.base_url):
- # The free tier has no credits to top up: a payment wall on it is the tier refusing. Off the welcome
- # host a free-tier JWT keeps the ordinary 402 handling (the route heal, billing copy).
- refusal = {"reason": "refused", "retry_after": 0, "alternates": [], "upgrade_url": ""}
- return _v(_R.auth_permanent, retryable=False, should_fallback=True,
- error_context={"welcome_refusal": refusal})
- if status == 429:
- # Every free-tier 429 is the allowance talking, so even one without a ``reason`` (header-only,
- # quota words) is a refusal; otherwise it would fall to billing or the generic rate-limit copy.
- from agent.nous_rate_guard import welcome_refusal_from_headers
- refusal = parse_welcome_refusal(c.body) or welcome_refusal_from_headers(
- c.headers, body_wait=_rate_limit_reset_seconds(c.msg, c.body, c.headers))
- ctx = {"welcome_refusal": refusal}
- if refusal["reason"] in WELCOME_TIER_GATE_REASONS:
- return _v(_R.model_not_found, retryable=False, should_fallback=True, error_context=ctx)
- if refusal["retry_after"] > 0:
- ctx["reset_at"] = time.time() + refusal["retry_after"]
- return _v(_R.rate_limit, should_fallback=True, error_context=ctx)
- # The route-keyed dark-tier 403 applies only to a 403 that says nothing else: a safety refusal
- # or a billing wall on the welcome host keeps its own classification (and its own recovery).
- plain_403 = not any(p in c.msg for p in _WELCOME_403_NAMED_PATTERNS)
- kind = welcome_route_refusal(status, c.msg, c.base_url if plain_403 else None)
- if kind is None:
- return None
- ctx = {"welcome_route": kind}
- if status == 403:
- return _v(_R.auth_permanent, retryable=False, should_fallback=True, error_context=ctx)
- return _v(_R.format_error, retryable=False, should_fallback=True, error_context=ctx)
- def _is_gemini_prompt_block(error: Exception) -> bool:
- """The native Gemini adapter's ``promptFeedback.blockReason`` error, matched on its structured code
- (a 200 with no candidates, so no status or body to classify; the message text is not trusted)."""
- return _from_cause_chain(error, _gemini_prompt_block_marker, False)
- def _gemini_prompt_block_marker(exc: Any) -> Optional[bool]:
- is_block = type(exc).__name__ == "GeminiAPIError" and getattr(exc, "code", None) == "gemini_prompt_blocked"
- return True if is_block else None
- def _provider_special_cases(c: _Ctx) -> Optional[Verdict]:
- """Highest-priority provider-specific shapes that a status code would misroute."""
- msg, status = c.msg, c.status_code
- # Deterministic for this context: another provider would receive the same connector result.
- if c.code == "data_policy_denied":
- return _v(_R.data_policy_denied, retryable=False, should_fallback=False)
- welcome = _nous_welcome_tier(c)
- if welcome is not None:
- return welcome
- # Safety refusal before status classification so a 400 block isn't downgraded
- # to format_error and a status-less block isn't left retryable (#18028).
- if any(p in msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS) or _is_gemini_prompt_block(c.error):
- return _V_CONTENT_BLOCKED
- # Status-agnostic: the stream-relayed ban has no status, and a 403 variant is not a bad key.
- if any(p in msg for p in _ACCOUNT_POLICY_BLOCK_PATTERNS):
- return _V_POLICY_BLOCKED
- # ChatGPT Codex masks a rejected encrypted-reasoning replay behind the same bare
- # ``invalid_prompt: Request blocked.`` it uses for real blocks (#92353). Exact envelope
- # + provider only. The verdict keeps format_error's abort-and-fallback hints; the one
- # extra thing it buys is turn_recovery's replay strip, which still requires cached
- # ``codex_reasoning_items`` — a genuine block with nothing to strip behaves as before.
- if _is_codex_masked_replay_rejection(c):
- return _v(_R.invalid_encrypted_content, **_ABORT_FALLBACK)
- # OpenAI Responses rejects a stale encrypted-reasoning replay with this code (#70595). It contains
- # both "thinking" and "signature", so it must beat the Anthropic heuristic below: that recovery
- # strips Anthropic thinking blocks and resends the same encrypted item forever.
- if status == 400 and (c.code == "thinking_signature_invalid" or "thinking_signature_invalid" in msg):
- return _V_INVALID_ENCRYPTED
- # Anthropic thinking-block 400s (signature mismatch after transcript mutation, or a sealed blob
- # the upstream cannot open). Not gated on provider — OpenRouter proxies Anthropic errors.
- if status == 400 and _is_anthropic_thinking_rejection(msg):
- return _v(_R.thinking_signature)
- # Anthropic long-context tier gate (429 "extra usage" + "long context").
- if status == 429 and "extra usage" in msg and "long context" in msg:
- return _v(_R.long_context_tier, should_compress=True)
- # Anthropic OAuth rejects the 1M beta header; run_agent retries without it.
- if status == 400 and "long context beta" in msg and "not yet available" in msg:
- return _v(_R.oauth_long_context_beta_forbidden)
- # llama.cpp grammar rejects regex ``pattern``/``format`` in tool schemas; the
- # retry loop strips them. Exclude the Qwen/vLLM "No user query found" error
- # local engines wrap as "Unable to generate parser for this template" —
- # that is a poisoned transcript (→ format_error), not a grammar problem.
- # Strict OpenAI-compatible schema validators reject regex lookaround in ``pattern``
- # with a different sentence ("Invalid JSON schema: regex lookaround is not supported",
- # #42631); same recovery — strip ``pattern``/``format`` and retry once.
- grammar_hit = "error parsing grammar" in msg or "json-schema-to-grammar" in msg or (
- "unable to generate parser" in msg and "template" in msg
- ) or ("invalid json schema" in msg and "regex lookaround" in msg and "not supported" in msg)
- if status == 400 and grammar_hit and _NO_USER_QUERY_SIGNAL not in msg:
- return _v(_R.llama_cpp_grammar_pattern)
- # xAI Grok entitlement as an SSE ``type=error`` frame: no status, matches no
- # pattern list, would otherwise burn max_retries as ``unknown``.
- if "do not have an active grok subscription" in msg or ("out of available resources" in msg and "grok" in msg):
- return _V_AUTH_FALLBACK
- return None
- def _moa_special_cases(c: _Ctx) -> Optional[Verdict]:
- # Local MoA streaming adapter-shape bugs are not a provider outage; falling
- # back would silently replace the MoA route with a single model (#55933).
- if c.provider_slug == "moa" and any(s in str(c.error) for s in _MOA_ADAPTER_SHAPE_BUGS):
- return _v(_R.format_error, retryable=False)
- # Persisted MoA preset name that was renamed/deleted — deterministic config error.
- from agent.errors import MoAPresetNotFoundError
- return _v(_R.model_not_found, retryable=False) if isinstance(c.error, MoAPresetNotFoundError) else None
- def _by_error_code(c: _Ctx) -> Optional[Verdict]:
- """Structured error codes from the response body."""
- # Request-validation failure as plain-text ``event: error`` SSE data behind
- # HTTP 200: retrying cannot succeed, a configured fallback still may.
- if c.code == PROVIDER_STREAM_NON_JSON_ERROR_CODE and "request validation failed:" in c.msg:
- return _V_FORMAT_ERROR
- verdict = _ERROR_CODE_VERDICTS.get(c.code)
- if verdict is None:
- family = _PROVIDER_CODE_FAMILIES.get(c.provider_slug, c.provider_slug)
- verdict = _PROVIDER_CODE_VERDICTS.get(family, {}).get(c.code)
- return verdict
- def _by_message(c: _Ctx) -> Optional[Verdict]:
- """Message patterns when no status code settled it; status-less usage
- limits get the same disambiguation as 402."""
- head = _first_match(c.msg, _MESSAGE_HEAD_RULES)
- if head is not None:
- return head
- if _is_byte_size_limit_message(c.msg):
- return _V_PAYLOAD_TOO_LARGE
- usage_limit = any(p in c.msg for p in _USAGE_LIMIT_PATTERNS)
- return _classify_402(c.msg, dict) if usage_limit else _first_match(c.msg, _MESSAGE_TAIL_RULES)
- def _by_transport(c: _Ctx) -> Optional[Verdict]:
- """SSL, disconnect, circuit-breaker and transport-type heuristics, in that order."""
- msg = c.msg
- # Cert failure → fail fast (checked first: also contains "[ssl:"); transient
- # alert → retry, before disconnects so a flaky handshake never compresses.
- ssl = _first_match(msg, ((_SSL_CERT_VERIFY_PATTERNS, _V_SSL_CERT), (_SSL_TRANSIENT_PATTERNS, _V_TIMEOUT)))
- if ssl is not None:
- return ssl
- # Disconnect + large session → probable overflow rejection, not a hiccup.
- if any(p in msg for p in _SERVER_DISCONNECT_PATTERNS) and not c.status_code:
- # Reasoning models: far more likely the gateway idle-killed a long
- # thinking stream — never compress on a phantom overflow (#52310).
- # Reasoning-model override: a transport disconnect on a reasoning model is much more likely the
- # upstream proxy idle-killing a long thinking stream than a true context overflow — even on large
- # sessions. The default disconnect+large-session routing below would otherwise send the user into
- # the compression branch (should_compress=True) and silently delete conversation history on a
- # phantom context-length error. Reasoning models have multi-minute thinking phases that routinely
- # exceed the cloud gateway's idle window (NVIDIA NIM ~120s — first-party repro at
- # NVIDIA/NemoClaw#4846; OpenAI worker / Anthropic stream-idle similar). The per-reasoning-model
- # stale-timeout floor in agent/reasoning_timeouts.py raises the stale-detector threshold to tolerate
- # long thinking, so a true transport-layer failure here is recoverable via the retry path — not via
- # context compression. Reclassify as timeout. (Part 1 of Fixes #52310.)
- from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
- if get_reasoning_stale_timeout_floor(c.model) is not None:
- return _V_TIMEOUT
- return _V_CONTEXT_OVERFLOW if c.large_session(0.6, 120000, 200) else _V_TIMEOUT
- # Stale-call circuit breaker (_check_stale_giveup RuntimeError before any
- # network call): as ``unknown`` it would burn every retry instantly.
- if c.error_type == "RuntimeError" and "consecutive stale attempts" in msg and "aborting this call" in msg:
- return _v(_R.timeout, **_ABORT_FALLBACK)
- transport = c.error_type in _TRANSPORT_ERROR_TYPES or isinstance(c.error, (TimeoutError, ConnectionError, OSError))
- return _V_TIMEOUT if transport else None
- def _by_status(c: _Ctx) -> Optional[Verdict]:
- """HTTP status code with message-aware refinement (unlisted 4xx/5xx → generic)."""
- status = c.status_code
- if status is None:
- return None
- default = _V_FORMAT_ERROR if 400 <= status < 500 else _V_SERVER_ERROR if 500 <= status < 600 else None
- return _STATUS_HANDLERS[status](c) if status in _STATUS_HANDLERS else default
- # Stage order: plugin hooks → the provider's own profile hook → provider-specific special cases →
- # HTTP status → MoA shapes → structured error code → message patterns → SSL → disconnect +
- # large session → transport types → unknown (retryable with backoff).
- _STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = (
- _plugin_verdict, _profile_verdict, _provider_special_cases, _by_status, _moa_special_cases,
- _by_error_code, _by_message, _by_transport,
- )
- def classify_api_error(
- error: Exception, *, provider: str = "", model: str = "",
- approx_tokens: int = 0, context_length: int = 200000, num_messages: int = 0,
- base_url: str = "",
- api_key: Any = None,
- ) -> ClassifiedError:
- """Classify an API error into a structured recovery recommendation (see ``_STAGES``).
- ``base_url`` (optional) is the route the call went to; the Nous welcome tier keys its
- dark-tier 403 on it because that refusal carries no distinguishing message.
- ``api_key`` identifies an anonymous request; a host or fairshare reason alone does not.
- The credential is never included in the returned context."""
- from hermes_cli.anon_auth import is_anonymous_request
- status_code = _extract_status_code(error)
- # Some SDK RateLimitErrors (OpenAI-compatible relays) may not set .status_code; force 429.
- if status_code is None and type(error).__name__ == "RateLimitError":
- status_code = 429
- body = _extract_error_body(error)
- c = _Ctx(
- error, status_code, body, _build_error_msg(error, body), provider, model,
- approx_tokens, context_length, num_messages, str(base_url or ""),
- anonymous=is_anonymous_request(provider, api_key),
- )
- verdict = next((v for v in (stage(c) for stage in _STAGES) if v is not None), _V_UNKNOWN)
- message = _extract_message(error, body)
- if verdict["reason"] in (_R.auth, _R.auth_permanent):
- # An auth refusal from a non-stock route names the host, so a credential posted to the
- # wrong endpoint (a stale ``model.base_url`` after a provider switch, #113719) reads as
- # such — not as a bad key.
- host = _off_route_host(c)
- if host:
- message = f"{message} (endpoint: {host})"
- base = {"status_code": status_code, "provider": provider, "model": model, "message": message}
- return ClassifiedError(**{**base, **verdict})
- def _off_route_host(c: _Ctx) -> str:
- """The contacted host when ``base_url`` is set and is not the provider's own endpoint; ``""`` otherwise."""
- from hermes_cli.route_identity import provider_owns_route
- from utils import base_url_hostname
- host = base_url_hostname(c.base_url)
- if not host or provider_owns_route(c.provider_slug, c.base_url) is True:
- return ""
- return host
- # ── Status code handlers ────────────────────────────────────────────────
- # Structured codes some gateways put on a 403 that mean "the upstream is down,
- # retry later" — not a credential refusal (#75388). Checked before the auth
- # default so the configured retry budget applies and no credential is benched.
- _403_TRANSIENT_CODES = frozenset({"upstream_unavailable"})
- def _status_403(c: _Ctx) -> Verdict:
- if c.code in _403_TRANSIENT_CODES:
- return _V_OVERLOADED
- # OpenRouter 403 "key limit exceeded" and similar plan/credit exhaustion are billing.
- xai_spend = c.provider_slug == "xai-oauth" and c.code == _XAI_SPENDING_LIMIT_ERROR_CODE
- billing = xai_spend or any(p in c.msg for p in ("key limit exceeded", "spending limit") + _BILLING_PATTERNS)
- if billing:
- return _V_BILLING
- # A WAF/CDN in front of the provider answered, not the provider: the credential never
- # reached it, so key guidance and credential rotation are wrong (#53099, #70566). Gated on
- # 403 and on established block/challenge markers; any other 403 stays auth.
- if any(p in c.msg for p in _UPSTREAM_BLOCKED_PATTERNS):
- return _V_UPSTREAM_BLOCKED
- return _V_AUTH_FALLBACK
- def _status_404(c: _Ctx) -> Verdict:
- # Structured billing code first, as in _status_429: this handler always returns,
- # so _by_error_code never sees it; a bare "Not Found" message has nothing to match.
- if c.code in _BILLING_ERROR_CODES:
- return _V_BILLING
- verdict = _first_match(c.msg, _404_RULES)
- if verdict is not None:
- return verdict
- # Bare id the catalogue only knows prefixed → malformed id (NVIDIA NIM "404
- # page not found", #78796). A generic 404 (wrong path, proxy glitch) stays
- # unknown so the real error surfaces instead of a silent misreported fallback.
- return _V_MODEL_NOT_FOUND if _model_id_missing_known_prefix(c.model_slug, c.provider_slug) else _V_UNKNOWN
- def _status_429(c: _Ctx) -> Verdict:
- # A structured billing code is decisive: LiteLLM stamps
- # ``terminal_quota_exhausted`` (a hard cap, not throttling) on 429s, and
- # this handler always returns, so _by_error_code never sees the code.
- if c.code in _BILLING_ERROR_CODES:
- return _V_BILLING
- # Z.AI/Zhipu reuse 429 for server-wide overload: back off on the same
- # key instead of burning the pool (#14038).
- if any(p in c.msg for p in _OVERLOADED_PATTERNS):
- return _V_OVERLOADED
- # OpenRouter-wrapped upstream 429: the key is healthy — fall back, don't bench.
- if _is_openrouter_upstream_error(c.body, c.provider_slug):
- upstream = _extract_upstream_provider_name(c.body)
- ctx = {"upstream_provider": upstream} if upstream else {}
- return _v(_R.upstream_rate_limit, should_fallback=True, error_context=ctx)
- # Quota walls as 429 (Anthropic ``usage_limit_reached``, "quota", billing
- # phrases) are billing ONLY when the body is not itself a rate-limit phrase
- # ("Rate limit exceeded" contains "limit exceeded") and carries no reset/
- # retry signal (#93419, #39441).
- quota_wall = c.code == "usage_limit_reached" or any(
- p in c.msg for p in ("usage_limit_reached",) + _USAGE_LIMIT_PATTERNS + _BILLING_PATTERNS
- )
- explicit_rate_limit = any(p in c.msg for p in _RATE_LIMIT_PATTERNS)
- if quota_wall and not explicit_rate_limit and not _has_usage_limit_transient_signal(c.msg, c.body, c.headers):
- return _V_BILLING
- # Carry the reset window so the terminal copy can name it instead of "wait a minute" (#89401).
- reset = _rate_limit_reset_seconds(c.msg, c.body, c.headers)
- if reset:
- return _v(_R.rate_limit, **_ROTATE_FALLBACK, error_context={"reset_at": time.time() + reset})
- return _V_RATE_LIMIT
- def _status_5xx(c: _Ctx) -> Verdict:
- # Request-validation errors as 5xx (codex.nekos.me) fail fast instead of
- # retry-flooding — unless the parameter was injected server-side.
- validation = any(p in c.msg for p in _REQUEST_VALIDATION_PATTERNS) or c.code in _5XX_VALIDATION_CODES
- if validation and not _is_server_injected_param_rejection(c.msg, c.provider_slug):
- return _V_FORMAT_ERROR
- return _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_SERVER_ERROR
- def _is_nous_route(c: _Ctx) -> bool:
- """The usage-window 402 code is Nous's: any other upstream sending it keeps main's text rules."""
- if (c.provider or "").strip().lower() == "nous":
- return True
- from utils import base_url_host_matches
- return bool(c.base_url) and base_url_host_matches(c.base_url, "nousresearch.com")
- def _classify_402(error_msg: str, result_fn: Callable[..., Any], code: str = "", body: Any = None) -> Any:
- """Disambiguate 402: "usage limit, try again in 5 minutes" is a periodic quota, not billing.
- The Nous usage-window code is decisive (its prose names the reset time, which must not
- demote it to a transient rate limit); every other 402 keeps the text rules. A usage-window 402's
- optional ``display_message`` (server-set wording) rides in ``error_context`` for the refusal line."""
- if code == _NOUS_USAGE_WINDOW_402_CODE:
- # Tagged so recovery skips the paid-entitlement refresh (none restores included usage before the reset).
- context: dict[str, Any] = {"usage_window": True}
- if display := _usage_window_display_message(body):
- context["display_message"] = display
- # No fallback: another model is the user's call (the limit card offers it), never automatic.
- return result_fn(**{**_V_BILLING, "should_fallback": False}, error_context=context)
- transient = any(p in error_msg for p in _USAGE_LIMIT_PATTERNS) and any(
- p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS
- )
- return result_fn(**(_V_RATE_LIMIT if transient else _V_BILLING))
- def _usage_window_display_message(body: Any) -> str:
- """The server's optional wording for a usage-window 402 (top level or under ``error``), trimmed and
- capped; "" when absent."""
- if not isinstance(body, dict):
- return ""
- for holder in (body, body.get("error")):
- value = holder.get("display_message") if isinstance(holder, dict) else None
- if isinstance(value, str) and value.strip():
- return value.strip()[:500]
- return ""
- def _has_large_inline_image(content: Any) -> bool:
- """True when a rejected ``content`` list carries a ``data:image/`` part the shrink pass would rewrite
- (over ``conversation_compression._IMAGE_SHRINK_TARGET_BYTES``; below it a shrink retry is a no-op)."""
- from agent.conversation_compression import _IMAGE_SHRINK_TARGET_BYTES
- for part in content if isinstance(content, list) else ():
- image = part.get("image_url") if isinstance(part, dict) else None
- url = image.get("url") if isinstance(image, dict) else image
- if isinstance(url, str) and url.startswith("data:image/") and len(url) > _IMAGE_SHRINK_TARGET_BYTES:
- return True
- return False
- def _oversized_message_content_rejection(body: Any) -> bool:
- """400 rejecting a *message* ``content`` field whose rejected value carries a large inline image.
- Nebius Token Factory caps a single image at 10 MiB and reports the violation through the field that
- failed to coerce — pydantic ``{"type": "string_type", "loc": ["body","messages",N,"content","str"],
- "msg": "Input should be a valid string", "input": [...]}`` — naming no size vocabulary, so the
- keyword multimodal *tool*-content rule (#104731) claimed it and spent its retry stripping tool images
- that were never there (#112473). The same list-shaped content with a small image succeeds, so the
- image bytes are the trigger. Tool-scoped locs (``messages.N.tool.content.str``) stay with #104731.
- """
- details = body.get("detail") if isinstance(body, dict) else None
- for detail in details if isinstance(details, list) else ():
- loc = detail.get("loc") if isinstance(detail, dict) else None
- if detail.get("type") != "string_type" or not isinstance(loc, list) or len(loc) < 2:
- continue
- parts = [str(x).lower() for x in loc]
- if parts[:2] == ["body", "messages"] and parts[-2:] == ["content", "str"] and not any(
- x.startswith("tool") for x in parts
- ) and _has_large_inline_image(detail.get("input")):
- return True
- return False
- def _classify_400(c: _Ctx) -> Verdict:
- """400 Bad Request — image/tool shapes, request-shape rejections, overflow, or generic."""
- msg, code = c.msg, c.code
- # A size cap reported *through* a message content field must beat the keyword
- # multimodal rule, which would otherwise claim "input should be a valid string".
- if _oversized_message_content_rejection(c.body):
- return _V_IMAGE_TOO_LARGE
- verdict = _first_match(msg, _IMAGE_TOOL_RULES)
- if verdict is not None:
- return verdict
- # Codex ChatGPT-account model rejection: exact normalized text only, so arbitrary 400s never
- # rotate. Before request-validation, whose "not supported" wording would abort as format_error (#71970).
- if CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER in msg:
- return _V_MODEL_ENTITLEMENT
- # Invalid encrypted reasoning replay blob (OpenAI Responses); before
- # overflow because "encrypted content … could not be verified" trips it.
- if code == "invalid_encrypted_content" or "invalid_encrypted_content" in msg or (
- "encrypted content for item" in msg and "could not be verified" in msg
- ) or "could not decrypt the provided encrypted_content" in msg or (
- # Custom Responses endpoints wrap a replay rejection in a generic bad_request (#95834).
- "encrypted content could not be decrypted or parsed" in msg
- ) or (
- # OpenCode Zen wraps this OpenAI replay rejection in ``invalid_request_error`` (#111309).
- "encrypted_content" in msg and "was not issued to this caller" in msg
- ) or (
- # Azure Foundry (gpt-6-astra) rejects replayed reasoning from several prior responses this way (#105369).
- "conflicting authenticated continuation identities" in msg
- ):
- return _V_INVALID_ENCRYPTED
- # Route rejecting a reasoning disable: a reasoning-mandatory route (GLM-5.3 on Nous Portal /
- # OpenRouter), a chat-only relay that does not accept ``reasoning_effort: none`` at all
- # (#114460), or an endpoint whose closed level set starts above the disable ("must be one of:
- # 'minimal', …" — Alibaba token-plan). Deterministic for the request shape, but the only bad
- # value is the disable — the loop drops it (or, when the disable is refused rather than the
- # field, steps the effort up to the floor) and retries once. Must precede request-validation,
- # which would abort as format_error.
- if (_REASONING_MANDATORY_PATTERN in msg or is_reasoning_field_rejection(msg)
- or is_reasoning_level_rejection(msg)):
- return _V_REASONING_MANDATORY
- # 400 blaming a field this route never sent (Codex OAuth injects then rejects
- # prompt_cache_retention ~20% of the time): transient, retry identical request.
- if _is_server_injected_param_rejection(msg, c.provider_slug):
- return _V_SERVER_ERROR
- if any(p in msg for p in _MALFORMED_TOOL_ARGS_PATTERNS):
- return _V_MALFORMED_TOOL_ARGS
- if any(p in msg for p in _ROLE_ALTERNATION_PATTERNS):
- return _V_ROLE_ALTERNATION
- # Body byte cap before the invalid-body and token-overflow rules: Novita tags it
- # INVALID_REQUEST_BODY and Z.AI's "request size exceeds the limit" trips "exceeds the limit".
- if _is_byte_size_limit_message(msg):
- return _V_PAYLOAD_TOO_LARGE
- # Before overflow: GPT-5's "Unsupported parameter: 'max_tokens'" contains it.
- if any(p in msg for p in _400_VALIDATION_PATTERNS) or code in _400_VALIDATION_CODES:
- return _V_FORMAT_ERROR
- # Malformed message array before overflow: input can be tiny and compression
- # cannot fix it. litellm/Bedrock proxies use errorCode=INVALID_REQUEST_BODY.
- if any(p in msg for p in _INVALID_MESSAGE_BODY_PATTERNS) or code == "invalid_request_body":
- logger.warning(
- "Malformed message array 400 (invalid request body) classified as format_error, NOT context "
- "overflow — failing fast + falling back instead of entering the compression loop. This usually "
- "means an empty-content assistant stub is in the transcript; num_messages=%s approx_tokens=%s. "
- "error=%.200s", c.num_messages, c.approx_tokens, msg,
- )
- return _V_FORMAT_ERROR
- # Memory ceiling by code: _by_status runs before _by_error_code, so a
- # 400 whose wording a proxy stripped would fall through to format_error.
- if code in _MEMORY_CEILING_ERROR_CODES:
- return _V_OVERLOADED
- verdict = _first_match(msg, _400_TAIL_RULES)
- if verdict is not None:
- return verdict
- # Generic 400 + large session → probable overflow (Anthropic can return a
- # bare "Error"); proxy shapes are read so a descriptive rejection isn't "bare".
- body_msg = next((m for m in (str(x or "").strip().lower() for x in _body_message_candidates(c.body)) if m), "")
- is_generic = len(body_msg) < 30 or body_msg in {"error", ""}
- if is_generic and c.large_session(0.4, 80000, 80):
- return _V_CONTEXT_OVERFLOW
- return _V_FORMAT_ERROR
- def _classify_image_tool_422(c: _Ctx) -> Verdict:
- """422: pydantic relays report the same content-field shapes as 400 (#104731, #112473)."""
- if _oversized_message_content_rejection(c.body):
- return _V_IMAGE_TOO_LARGE
- return _first_match(c.msg, _IMAGE_TOOL_RULES) or _V_FORMAT_ERROR
- # 401 not retryable on its own: rotation/refresh run before the retryability
- # check, then the client-error abort path (fallback first) is correct. 408 is
- # retry-safe (RFC 9110 §15.5.9; proxies emit it when generation outruns the
- # read window). Unlisted 4xx → format_error, 5xx → server_error.
- _STATUS_HANDLERS: dict[int, Callable[[_Ctx], Verdict]] = {
- 400: _classify_400, 401: lambda c: _V_AUTH_ROTATE,
- 402: lambda c: _classify_402(c.msg, dict, c.code if _is_nous_route(c) else "", c.body),
- 403: _status_403, 404: _status_404, 408: lambda c: _V_TIMEOUT, 413: lambda c: _V_PAYLOAD_TOO_LARGE,
- 422: lambda c: _classify_image_tool_422(c),
- 429: _status_429, 500: _status_5xx, 502: _status_5xx,
- 503: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
- 529: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
- }
- # ── Helpers ─────────────────────────────────────────────────────────────
- _RESET_FIELDS = ("resets_in_seconds", "resets_at", "reset_at", "retry_after")
- _RESET_HEADERS = ("retry-after", "Retry-After", "x-ratelimit-reset", "X-RateLimit-Reset")
- def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_headers) -> bool:
- """Whether a usage-limit response identifies a reset window (message, body fields, or headers)."""
- if any(pattern in error_msg for pattern in _USAGE_LIMIT_TRANSIENT_SIGNALS):
- return True
- payloads = [p for p in (body, _error_obj(body)) if isinstance(p, dict)]
- if any(payload.get(f) not in (None, "") for payload in payloads for f in _RESET_FIELDS):
- return True
- if response_headers and hasattr(response_headers, "get"):
- return any(response_headers.get(h) not in (None, "") for h in _RESET_HEADERS)
- return False
- def _rate_limit_reset_seconds(error_msg: str, body: dict, response_headers) -> Optional[float]:
- """Seconds until a 429's window reopens, from the body's reset fields, ``Retry-After`` or the
- message grammar (``retry after Ns`` / ``resets in 4hr``); None when the response names none."""
- from agent.retry_utils import parse_retry_after_seconds, reset_delay_from_message
- for payload in (p for p in (body, _error_obj(body)) if isinstance(p, dict)):
- for name in _RESET_FIELDS:
- value = payload.get(name)
- if value in (None, ""):
- continue
- if name.endswith("_at") and isinstance(value, (int, float)):
- return max(0.0, float(value) - time.time())
- if (seconds := parse_retry_after_seconds(value)) is not None:
- return seconds
- if (seconds := parse_retry_after_seconds(response_headers)) is not None:
- return seconds
- return reset_delay_from_message(error_msg)
- def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
- """True when a bare model id is only known to the provider as ``vendor/id``.
- Never guesses: an id absent from the curated catalogue returns False so real
- endpoint problems keep their retryable ``unknown`` classification.
- """
- name = (model or "").strip()
- if not name or "/" in name:
- return False
- try:
- from hermes_cli.model_normalize import suggest_prefixed_model_id
- return bool(suggest_prefixed_model_id((provider or "").strip(), name))
- except Exception:
- return False
- def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
- """True when a 400 blames a one-route-only parameter this route never sends.
- Conservative: known parameters only, and only when ``provider`` is not a
- sender, so a genuine bad parameter (``max_tokens`` on GPT-5) stays format_error.
- """
- provider_slug = (provider or "").strip().lower()
- for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
- if error_msg and param in error_msg and any(w in error_msg for w in _PARAM_REJECTION_WORDS):
- return not any(sender in provider_slug for sender in senders)
- return False
- _CODEX_MASKED_REPLAY_MESSAGE = "request blocked."
- _CODEX_UNSUPPORTED_CONTENT_DETAIL = "unsupported content type"
- def _is_codex_masked_replay_rejection(c: _Ctx) -> bool:
- """HTTP 400 / status-less ``{code: invalid_prompt, message: "Request blocked."}`` from
- ``openai-codex`` — as an SDK error body, a Responses ``error`` SSE frame, or the
- ``response.failed`` text ``"invalid_prompt: Request blocked."`` — or the bare
- ``{"detail": "Unsupported content type"}`` envelope the same backend returns for a rejected
- encrypted-reasoning replay (#51512). Both are exact envelopes, provider-gated."""
- if c.provider_slug != "openai-codex" or c.status_code not in (None, 400):
- return False
- body = c.body if isinstance(c.body, dict) else {}
- if str(body.get("detail") or "").strip().lower() == _CODEX_UNSUPPORTED_CONTENT_DETAIL or (
- not body and _CODEX_UNSUPPORTED_CONTENT_DETAIL in c.msg and "detail" in c.msg
- ):
- return True
- # The OpenAI SDK unwraps ``body["error"]`` on status errors; stream frames keep the envelope.
- body_msg = next((str(m).strip().lower() for m in _body_message_candidates(body) if m), "")
- return (c.code == "invalid_prompt" and body_msg == _CODEX_MASKED_REPLAY_MESSAGE) or (
- c.msg.strip() == f"invalid_prompt: {_CODEX_MASKED_REPLAY_MESSAGE}"
- )
- def _error_obj(body: Any) -> dict:
- """``body["error"]`` when it is a dict, else ``{}``."""
- err = body.get("error") if isinstance(body, dict) else None
- return err if isinstance(err, dict) else {}
- def _json_dict(text: Any) -> Optional[dict]:
- """Parse a JSON object string; None for non-strings, blanks, invalid JSON or non-objects."""
- if not (isinstance(text, str) and text.strip()):
- return None
- try:
- inner = json.loads(text)
- except (json.JSONDecodeError, TypeError):
- return None
- return inner if isinstance(inner, dict) else None
- def _openrouter_wrapped_message(err_obj: dict) -> str:
- """Lowercased inner message from OpenRouter's ``error.metadata.raw`` JSON wrapper."""
- metadata = err_obj.get("metadata", {})
- inner = _json_dict(metadata.get("raw")) if isinstance(metadata, dict) else None
- return str(_error_obj(inner).get("message") or "").lower() if inner else ""
- def _build_error_msg(error: Exception, body: Any) -> str:
- """Lowercased str(error) + body message + OpenRouter-wrapped upstream message
- (OpenAI SDK's APIStatusError.__str__ omits the body, so it is appended)."""
- raw_msg = str(error).lower()
- body_msg = metadata_msg = ""
- if isinstance(body, dict):
- err_obj = _error_obj(body)
- body_msg = str(err_obj.get("message") or "").lower() or str(body.get("message") or "").lower()
- metadata_msg = _openrouter_wrapped_message(err_obj) if err_obj else ""
- parts = [raw_msg]
- if body_msg and body_msg not in raw_msg:
- parts.append(body_msg)
- if metadata_msg and metadata_msg not in raw_msg and metadata_msg not in body_msg:
- parts.append(metadata_msg)
- return " ".join(parts)
- def _body_message_candidates(body: dict) -> Iterator[Any]:
- """Body message fields in priority order (OpenAI, flat, litellm/Bedrock proxy, FastAPI shapes)."""
- yield _error_obj(body).get("message")
- yield body.get("message")
- yield body.get("errorMessage")
- args = body.get("errorArgs")
- yield args.get("reason") if isinstance(args, dict) else None
- # FastAPI/Starlette relays and the Codex gateway answer {"detail": "..."} (or a nested
- # OpenAI-ish object); without it a descriptive rejection reads as a bare 400 and the
- # large-session heuristic sends it into compression (#81558). A list here is pydantic's
- # validation shape, read by _oversized_message_content_rejection.
- detail = body.get("detail")
- yield detail.get("message") if isinstance(detail, dict) else detail if isinstance(detail, str) else None
- def _from_cause_chain(error: Exception, pick: Callable[[Any], Any], default: Any) -> Any:
- """First non-None ``pick(exc)`` over the error and its __cause__/__context__ chain (max 5 deep)."""
- current = error
- for _ in range(5):
- found = pick(current)
- if found is not None:
- return found
- cause = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
- if cause is None or cause is current:
- break
- current = cause
- return default
- def _status_of(exc: Any) -> Optional[int]:
- code = getattr(exc, "status_code", None)
- if isinstance(code, int):
- return code
- code = getattr(exc, "status", None) # some SDKs use .status
- return code if isinstance(code, int) and 100 <= code < 600 else None
- def _body_of(exc: Any) -> Optional[dict]:
- body = getattr(exc, "body", None)
- if isinstance(body, dict):
- return body
- response = getattr(exc, "response", None)
- try:
- json_body = response.json() if response is not None else None
- except Exception:
- return None
- return json_body if isinstance(json_body, dict) else None
- def _headers_of(exc: Any) -> Any:
- headers = getattr(getattr(exc, "response", None), "headers", None)
- return headers if headers and hasattr(headers, "get") else None
- def _status_code_from_body(body: Any) -> Optional[int]:
- """Numeric HTTP error status (400-599) from ``error.code``/``code`` in a structured body.
- An aggregator/relay can deliver the upstream failure only this way — as an
- error object inside an HTTP-200 SSE stream — leaving the SDK to raise a
- status-less ``APIError`` whose ``body`` carries the status (#121270). String
- codes stay symbolic (``_code_from_payload``'s ``"400" is not a code``), unlike the
- string-parsing text-SSE sibling ``chat_completion_helpers._status_code_from_payload``."""
- if not isinstance(body, dict):
- return None
- error_obj = _error_obj(body)
- candidates = [error_obj.get(k) for k in ("status_code", "status", "http_status", "code")] + [body.get("code")]
- return next(
- (c for c in candidates if isinstance(c, int) and not isinstance(c, bool) and 400 <= c < 600),
- None,
- )
- def _extract_status_code(error: Exception) -> Optional[int]:
- """HTTP status code from the error or its cause chain; a body-carried numeric
- ``code`` counts when the exception itself carries no status (#121270)."""
- status = _from_cause_chain(error, _status_of, None)
- if status is None:
- status = _status_code_from_body(_from_cause_chain(error, _body_of, {}))
- return status
- def _extract_error_body(error: Exception) -> dict:
- """Structured error body from an SDK exception or its cause chain."""
- return _from_cause_chain(error, _body_of, {})
- def _code_from_payload(payload: Any, top_keys: Sequence[str], peek_message: bool) -> str:
- """Code/type from ``payload.error`` or a top-level key; ``"400"`` is not a code.
- ``peek_message`` also parses a JSON ``error.message`` for a nested code
- (Responses API surfaces ``invalid_encrypted_content`` this way). Gemini
- puts the HTTP status in ``error.code`` and the symbolic code
- (``UNAVAILABLE``) in ``error.status``, so a numeric code defers to it."""
- if not isinstance(payload, dict):
- return ""
- error_obj = payload.get("error", {})
- if isinstance(error_obj, dict):
- code = error_obj.get("code") or error_obj.get("type") or ""
- if not isinstance(code, str):
- code = error_obj.get("status") or code
- if isinstance(code, str) and code.strip() and code.strip() != "400":
- return code.strip()
- message = error_obj.get("message")
- if peek_message and isinstance(message, str) and message.strip().startswith("{"):
- nested_code = _code_from_payload(_json_dict(message), ("code", "error_code"), False)
- if nested_code:
- return nested_code
- code = next((payload.get(k) for k in top_keys if payload.get(k)), "")
- text = str(code).strip() if isinstance(code, (str, int)) else ""
- return text if text and text != "400" else ""
- def _extract_error_code(body: dict) -> str:
- """Extract an error code string from the response body."""
- return _code_from_payload(body, ("code", "error_code", "errorCode"), True) if body else ""
- def _extract_message(error: Exception, body: dict) -> str:
- """Extract the most informative error message (structured body first)."""
- msg = next((m for m in _body_message_candidates(body or {}) if isinstance(m, str) and m.strip()), None)
- return (msg.strip() if msg else str(error))[:500]
- def _is_openrouter_upstream_error(body: Any, provider: str) -> bool:
- """OpenRouter's "Provider returned error" wrapper: the key is healthy, the
- upstream failed, so credential rotation is the wrong recovery."""
- err = _error_obj(body)
- if str(err.get("message") or "").strip().lower() != "provider returned error":
- return False
- if (provider or "").strip().lower() == "openrouter":
- return True
- # Otherwise require the metadata shape only OpenRouter produces.
- metadata = err.get("metadata")
- return isinstance(metadata, dict) and ("raw" in metadata or "provider_name" in metadata)
- def _extract_upstream_provider_name(body: Any) -> Optional[str]:
- """Pull the upstream provider name out of OpenRouter's error metadata."""
- metadata = _error_obj(body).get("metadata")
- name = metadata.get("provider_name") if isinstance(metadata, dict) else None
- return name.strip() if isinstance(name, str) and name.strip() else None
|