turn_recovery.py 107 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989
  1. """Recovery-branch handlers for the conversation turn's inner retry loop.
  2. When the model call raises, one-shot recovery chains run before the generic retry/backoff
  3. path. Handlers return ``True`` (request repaired in place; loop ``continue``s with the same
  4. ``retry_count``) or ``False`` (fall through). Guards live on ``TurnRetryState``; handlers
  5. mutate ``agent`` / ``messages`` / ``api_messages`` in place. Logger name stays
  6. ``agent.conversation_loop`` (caplog pins); that module is only imported lazily (cycle + patch sites).
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import locale
  11. import math
  12. import re
  13. import time
  14. from dataclasses import dataclass
  15. from typing import Any, Dict, List, Optional, Tuple
  16. from agent.conversation_compression import COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE
  17. from agent.fast_mode import fast_mode_unprovisioned, mark_fast_mode_unavailable
  18. from agent.model_metadata import is_output_cap_error, parse_available_output_tokens_from_error
  19. from agent.retry_utils import is_zai_coding_overload_error, zai_coding_overload_retry_ceiling
  20. from agent.error_classifier import FailoverReason, classify_api_error
  21. from agent.message_sanitization import (
  22. _looks_like_corrupt_image_rejection, _looks_like_image_content_rejection, _messages_carry_images,
  23. _sanitize_messages_non_ascii,
  24. _sanitize_messages_surrogates, _sanitize_structure_non_ascii, _sanitize_structure_surrogates,
  25. _strip_images_from_messages, _strip_non_ascii, close_interrupted_tool_sequence, record_reasoning_field_rejection,
  26. )
  27. from agent.thinking_timeout_guidance import build_thinking_timeout_guidance, is_thinking_timeout
  28. from agent.vision_message_prep import _provider_model_key
  29. from agent.turn_failure_copy import (
  30. CONTENT_POLICY_NEXT_STEPS, content_policy_copy, exhausted_copy, limit_reset_copy, nonretryable_copy,
  31. provider_label_for, site_copy, stamp_failure,
  32. )
  33. from agent.turn_retry_state import TurnRetryState
  34. from agent.turn_recovery_codex import _is_codex_token_expired, _recover_stale_codex_reasoning
  35. from hermes_constants import display_hermes_home
  36. from utils import base_url_host_matches
  37. logger = logging.getLogger("agent.conversation_loop")
  38. def _runtime_uses_ascii_encoding() -> bool:
  39. """Return whether the process genuinely needs an ASCII-only request fallback."""
  40. encoding = locale.getpreferredencoding(False).strip().lower().replace("_", "-")
  41. return encoding in {"ascii", "us-ascii", "ansi-x3.4-1968"}
  42. def _vlines(agent: Any, *lines: str) -> None:
  43. """Force-``_vprint`` each line prefixed with ``agent.log_prefix``."""
  44. for line in lines:
  45. agent._vprint(f"{agent.log_prefix}{line}", force=True, diagnostic=True)
  46. def _plines(agent: Any, *lines: str) -> None:
  47. """``print`` each line prefixed with ``agent.log_prefix``."""
  48. from gateway.warning_notifications import render_notification
  49. render_notification(
  50. lambda: [print(f"{agent.log_prefix}{line}") for line in lines],
  51. platform=getattr(agent, "_notification_platform", getattr(agent, "platform", "cli")),
  52. user_config=getattr(agent, "_notification_config", None))
  53. def _blines(agent: Any, *lines: str) -> None:
  54. """``_buffer_vprint`` each line (surfaces only if every retry+fallback exhausts)."""
  55. for line in lines:
  56. agent._buffer_vprint(line)
  57. def _image_error_max_dimension(error: Exception) -> Optional[int]:
  58. """Extract a provider-reported image dimension ceiling, if present."""
  59. parts = []
  60. for value in (error, getattr(error, "message", None), getattr(error, "body", None)):
  61. if value:
  62. try:
  63. parts.append(str(value))
  64. except Exception:
  65. pass
  66. text = " ".join(parts).lower()
  67. # OpenAI Codex Responses reports a tile-patch budget (ceil(w/32)×ceil(h/32))
  68. # instead of a pixel ceiling. A square image is the worst case for the budget,
  69. # so a per-side cap of isqrt(limit)*32 px keeps isqrt(limit)² ≤ limit — for the
  70. # 30000-patch ceiling that is 5536 px. Without this the caller falls back to
  71. # 8000 px and a 6000 px image that already exceeds the budget is skipped (#106337).
  72. if "patches after processing" in text:
  73. match = re.search(r"exceeding the limit of\s*(\d{2,7})", text)
  74. if not match:
  75. return None
  76. max_dimension = math.isqrt(int(match.group(1))) * 32
  77. return max_dimension if 512 <= max_dimension <= 8000 else None
  78. if "image" not in text or "dimension" not in text or "max allowed size" not in text:
  79. return None
  80. match = re.search(r"max allowed size(?:\s+for [^:]+)?:\s*(\d{3,5})\s*pixels?", text)
  81. if not match:
  82. return None
  83. try:
  84. max_dimension = int(match.group(1))
  85. except ValueError:
  86. return None
  87. return max_dimension if 512 <= max_dimension <= 8000 else None
  88. def _try_refresh_nous_paid_entitlement_credentials(agent) -> bool:
  89. """Refresh Nous runtime credentials after a fresh paid-entitlement check."""
  90. try:
  91. from hermes_cli.nous_account import get_nous_portal_account_info
  92. if get_nous_portal_account_info(force_fresh=True).paid_service_access is not True:
  93. return False
  94. return agent._try_refresh_nous_client_credentials(force=True)
  95. except Exception:
  96. return False
  97. def _repair_transport_credentials(agent: Any) -> bool:
  98. """Strip non-ASCII from ``_client_kwargs["default_headers"]`` and the API key.
  99. Non-ASCII in the key makes httpx fail encoding the Authorization header — the usual
  100. persistent cause of UnicodeEncodeError that survives message/tool sanitization (#6843,
  101. e.g. ʋ instead of v from a bad copy-paste). Entra ID bearer providers are callables
  102. minting ASCII JWTs; skip them (``_strip_non_ascii`` would crash). Returns True when
  103. either the headers or the key were repaired.
  104. """
  105. _client_kwargs = getattr(agent, "_client_kwargs", None)
  106. _default_headers = _client_kwargs.get("default_headers") if isinstance(_client_kwargs, dict) else None
  107. _repaired = bool(isinstance(_default_headers, dict) and _sanitize_structure_non_ascii(_default_headers))
  108. _raw_key = getattr(agent, "api_key", None) or ""
  109. if isinstance(_raw_key, str) and _raw_key:
  110. _clean_key = _strip_non_ascii(_raw_key)
  111. if _clean_key != _raw_key:
  112. agent.api_key = _clean_key
  113. if isinstance(_client_kwargs, dict):
  114. _client_kwargs["api_key"] = _clean_key
  115. # The live client reads its own api_key copy on every request.
  116. if getattr(agent, "client", None) is not None and hasattr(agent.client, "api_key"):
  117. agent.client.api_key = _clean_key
  118. _repaired = True
  119. _vlines(
  120. agent,
  121. "⚠️ API key contained non-ASCII characters (bad copy-paste?) — stripped them. "
  122. "If auth fails, re-copy the key from your provider's dashboard.",
  123. )
  124. return _repaired
  125. def _recover_unicode_encode_error(
  126. agent: Any, api_error: Exception, messages: list[dict[str, Any]], api_messages: Any,
  127. api_kwargs: Any, active_system_prompt: Any,
  128. ) -> tuple[bool, Any]:
  129. """UnicodeEncodeError recovery: lone surrogates (clipboard paste) first, then an ASCII
  130. codec under a non-UTF-8 locale. Sanitizes in place; bounded by the caller's
  131. ``_unicode_sanitization_passes < 2`` guard (surrogate strip, then ASCII-only)."""
  132. _err_str = str(api_error).lower()
  133. _is_ascii_codec = "'ascii'" in _err_str or "ascii" in _err_str
  134. # utf-8 refusing U+D800..U+DFFF ("surrogates not allowed").
  135. _is_surrogate_error = "surrogate" in _err_str or ("'utf-8'" in _err_str and not _is_ascii_codec)
  136. # Sanitize canonical messages for surrogate recovery, but keep ASCII recovery
  137. # request-local: API copies may carry fields absent from the durable transcript.
  138. _surrogates_found = _sanitize_messages_surrogates(messages)
  139. _surrogates_found |= isinstance(api_messages, list) and _sanitize_messages_surrogates(api_messages)
  140. _surrogates_found |= isinstance(api_kwargs, dict) and _sanitize_structure_surrogates(api_kwargs)
  141. # Gate the retry on the error type, not on whether anything was found — a new
  142. # transformed field could slip through.
  143. if _surrogates_found or _is_surrogate_error:
  144. if _surrogates_found:
  145. # In-place rewrites may have popped _DB_PERSISTED_MARKER off stamped live dicts;
  146. # force a full flush scan so the repaired rows are rewritten.
  147. agent._db_flush_scan_prefix = None
  148. agent._unicode_sanitization_passes += 1
  149. agent._buffer_vprint(
  150. "⚠️ Stripped invalid surrogate characters from messages. Retrying..."
  151. if _surrogates_found else
  152. "⚠️ Surrogate encoding error — retrying after full-payload sanitization..."
  153. )
  154. return True, active_system_prompt
  155. if not _is_ascii_codec:
  156. return False, active_system_prompt
  157. # Error text is provider-controlled and can mention ``ascii`` even when the
  158. # process sends UTF-8. In that normal case, do not rewrite conversation,
  159. # tools, prompts, or prefill; only repair values that can poison an ASCII
  160. # transport header. If nothing was repaired, an identical retry cannot
  161. # succeed — return False so the error surfaces through the normal path
  162. # instead of burning both sanitization passes on unchanged requests.
  163. if not _runtime_uses_ascii_encoding():
  164. if not _repair_transport_credentials(agent):
  165. return False, active_system_prompt
  166. agent._unicode_sanitization_passes += 1
  167. _vlines(
  168. agent,
  169. "⚠️ Repaired non-ASCII request credentials/headers without changing conversation content. Retrying...",
  170. )
  171. return True, active_system_prompt
  172. agent._force_ascii_payload = True
  173. # Strip all non-ASCII from the request-local api_messages (reused across retries). The
  174. # failed attempt's api_kwargs is NOT touched: build_api_request rebuilds it from
  175. # ``agent.tools`` on the next iteration and ``sanitize_outbound_kwargs`` strips the whole
  176. # payload under ``_force_ascii_payload``. Canonical agent state stays byte-stable.
  177. _messages_sanitized = isinstance(api_messages, list) and _sanitize_messages_non_ascii(api_messages)
  178. _system_sanitized = False
  179. if isinstance(active_system_prompt, str):
  180. _sanitized_system = _strip_non_ascii(active_system_prompt)
  181. if _sanitized_system != active_system_prompt:
  182. active_system_prompt = _sanitized_system
  183. _system_sanitized = True
  184. _transport_repaired = _repair_transport_credentials(agent)
  185. # Always retry on ASCII codec detection: _force_ascii_payload sanitizes the full
  186. # api_kwargs next iteration even when the checks above find nothing.
  187. agent._unicode_sanitization_passes += 1
  188. _vlines(
  189. agent,
  190. "⚠️ System encoding is ASCII — stripped non-ASCII characters from request payload. Retrying..."
  191. if (_messages_sanitized or _system_sanitized or _transport_repaired) else
  192. "⚠️ System encoding is ASCII — enabling full-payload sanitization for retry...",
  193. )
  194. return True, active_system_prompt
  195. def _strip_request_images_and_retry(agent: Any, api_messages: Any) -> bool:
  196. """Strip image parts from the per-call ``api_messages`` copy; True if anything was removed.
  197. Shared by the corrupt-image recoveries: a bad payload says nothing about the model, so it
  198. is stripped for this attempt only and the model is never recorded as image-rejecting."""
  199. if isinstance(api_messages, list) and _strip_images_from_messages(api_messages):
  200. _vlines(agent, "⚠️ Provider rejected a corrupted image — stripped images from the retry payload and retrying...")
  201. return True
  202. return False
  203. def recover_before_classification(
  204. agent: Any, api_error: Exception, *, messages: list[dict[str, Any]], api_messages: Any,
  205. api_kwargs: Any, active_system_prompt: Any,
  206. ) -> tuple[bool, Any]:
  207. """Recovery branches that run BEFORE ``classify_api_error``: UnicodeEncodeError
  208. sanitization, Anthropic fast mode with no capacity (drop ``speed`` for that model),
  209. provider image-content rejection (record the (provider, model);
  210. build_api_request strips images from that model's requests only), an unknown-field
  211. rejection naming a replayed reasoning key (recorded per (provider, host, model),
  212. stripped on the retry), and the Bedrock
  213. AnthropicBedrock SDK streaming fallback. Returns ``(retry_now, active_system_prompt)``;
  214. the prompt may be ASCII-sanitized in place."""
  215. if isinstance(api_error, UnicodeEncodeError) and getattr(agent, '_unicode_sanitization_passes', 0) < 2:
  216. _recovered, active_system_prompt = _recover_unicode_encode_error(
  217. agent, api_error, messages, api_messages, api_kwargs, active_system_prompt
  218. )
  219. if _recovered:
  220. return True, active_system_prompt
  221. # Anthropic fast mode with no capacity: a 429 whose fast-mode limit header is 0 can never
  222. # succeed at fast speed, and it says nothing about the key's standard-speed limits. Stop
  223. # sending ``speed`` to this model and retry now, before credential rotation benches the key.
  224. if fast_mode_unprovisioned(api_error, api_kwargs) and mark_fast_mode_unavailable(agent):
  225. _vlines(agent, f"⚠️ Fast mode isn't available for {agent.model} on this Anthropic organization — using standard speed for this session, retrying...")
  226. logger.warning("%sFast mode: %s has a fast-mode limit of 0; standard speed for this session", agent.log_prefix, agent.model)
  227. return True, active_system_prompt
  228. # Some providers 4xx on image_url content: record the (provider, model) and retry;
  229. # build_api_request strips images from that model's requests only. English phrase
  230. # match; extend it.
  231. _err_body = ""
  232. try:
  233. _err_body = str(getattr(api_error, "body", None) or getattr(api_error, "message", None) or str(api_error))
  234. except Exception:
  235. pass
  236. _err_status = getattr(api_error, "status_code", None)
  237. # 4xx-only gate: 5xx/timeouts are transient and take the retry path.
  238. _status_ok = _err_status is None or (400 <= int(_err_status) < 500)
  239. # Guarded PER MODEL, not by a turn-global flag: in a fallback chain the next model can reject
  240. # images too, and a turn-wide flag would skip its recovery and fail the turn.
  241. _model_key = _provider_model_key(agent)
  242. _rejected = agent._image_rejecting_models
  243. _corrupt = _looks_like_corrupt_image_rejection(_err_body)
  244. if _status_ok and (_corrupt or (_model_key not in _rejected and _looks_like_image_content_rejection(_err_body))):
  245. # Send-path only. A rejection says what THIS model accepts, not what the conversation
  246. # holds: stripping ``messages`` (canonical history) and forcing a flush deleted every
  247. # image — and every image-only message — from state.db for good, so a later switch to a
  248. # vision model found them gone. Same failure as the ASCII strip in #117802.
  249. if _corrupt:
  250. # A bad payload says nothing about the model's capability: strip this attempt only
  251. # (like the image_corrupt branch below) and leave the model unmarked so a later good
  252. # image still reaches it. Retry only if something was stripped, or a text-only
  253. # request would loop on the same error.
  254. if _strip_request_images_and_retry(agent, api_messages):
  255. return True, active_system_prompt
  256. elif _messages_carry_images(api_messages or messages):
  257. # Record the model; the retry re-enters build_api_request with the same
  258. # api_messages and strip_images_for_rejecting_model strips them there.
  259. # A request with no image never marks: a 4xx body can echo the user's own text
  260. # ("X is not a multimodal model"), and a wrong mark blinds a vision model until /new.
  261. _rejected.add(_model_key)
  262. _vlines(
  263. agent,
  264. "⚠️ Server rejected image content — sending text only to this model; "
  265. "images stay in the session history.",
  266. )
  267. return True, active_system_prompt
  268. # A strict schema rejected a replayed reasoning key BY NAME: record it per (provider, host, model)
  269. # and retry once; build_api_request re-shapes api_messages without it. An unnamed upstream error
  270. # never counts, and a key already recorded falls through to normal handling (no loop).
  271. _sent = api_kwargs.get("messages") if isinstance(api_kwargs, dict) else None
  272. if _status_ok and (_new := record_reasoning_field_rejection(agent, _err_body, _sent or api_messages)):
  273. _vlines(agent, f"⚠️ {agent.model} rejected replayed {', '.join(sorted(_new))} — retrying without it for this model.")
  274. return True, active_system_prompt
  275. # AnthropicBedrock SDK raises "Unexpected event order" when Bedrock errors before
  276. # message_start; fall back to native Converse for this session.
  277. if (
  278. isinstance(api_error, RuntimeError)
  279. and "unexpected event order" in str(api_error).lower()
  280. and getattr(agent, "provider", "") == "bedrock"
  281. and agent.api_mode == "anthropic_messages"
  282. and not getattr(agent, "_bedrock_converse_fallback_attempted", False)
  283. ):
  284. agent._bedrock_converse_fallback_attempted = True
  285. agent.api_mode = "bedrock_converse"
  286. agent._bedrock_region = getattr(agent, "_bedrock_region", None) or "us-east-1"
  287. agent.client = None # Drop the AnthropicBedrock client
  288. agent._client_kwargs = {}
  289. _vlines(agent, "⚠️ AnthropicBedrock SDK streaming failed — falling back to native Converse API for this session.")
  290. return True, active_system_prompt
  291. return False, active_system_prompt
  292. def _print_nous_401_diagnostics(agent: Any, api_error: Exception) -> None:
  293. """Nous 401 that survived a credential refresh: likely Portal OAuth expired/revoked,
  294. no credits, or agent key blocked."""
  295. from agent.conversation_loop import _print_nous_entitlement_guidance
  296. from hermes_constants import display_hermes_home
  297. _body_text = ""
  298. try:
  299. _body = getattr(api_error, "body", None) or getattr(api_error, "response", None)
  300. if _body is not None:
  301. _body_text = str(_body)[:200]
  302. except Exception:
  303. pass
  304. _plines(agent, "🔐 Nous 401 — Portal authentication failed.")
  305. if _body_text:
  306. _plines(agent, f" Response: {_body_text}")
  307. try:
  308. from hermes_cli.anon_auth import is_anonymous_agent
  309. if is_anonymous_agent(agent):
  310. # The free tier has no credits, no agent key and no auth.json to inspect: its session
  311. # ended and could not be replaced. The two doors are a sign-in or another provider.
  312. _plines(agent, " Your session ended and Hermes couldn't start a new one.",
  313. " Sign in with a Nous account (it's free), or switch providers with /model.")
  314. return
  315. except Exception:
  316. pass
  317. if not _print_nous_entitlement_guidance(agent, "Nous model access"):
  318. _plines(agent, " Most likely: Portal OAuth expired, account out of credits, or agent key revoked.")
  319. _plines(
  320. agent,
  321. " Troubleshooting:",
  322. " • Re-authenticate: hermes auth add nous",
  323. " • Check credits / billing: https://portal.nousresearch.com",
  324. f" • Verify stored credentials: {display_hermes_home()}/auth.json",
  325. " • Switch providers temporarily: /model <model> --provider openrouter",
  326. )
  327. def _print_anthropic_401_diagnostics(agent: Any, key: Any) -> None:
  328. """Anthropic 401 that survived a credential refresh: show auth method + fixes."""
  329. from agent.anthropic_credentials import _is_oauth_token
  330. from agent.azure_identity_adapter import is_token_provider
  331. from hermes_constants import display_hermes_home
  332. _plines(agent, "🔐 Anthropic 401 — authentication failed.")
  333. if is_token_provider(key):
  334. # Azure Foundry Entra ID: JWT minted per-request by an httpx hook; 401 = Azure
  335. # rejected it (RBAC, az login, IMDS).
  336. _plines(
  337. agent,
  338. " Auth method: Microsoft Entra ID (httpx event hook)",
  339. " Run `hermes doctor` for credential-chain diagnostics, or",
  340. " `az login` if your developer session expired.",
  341. )
  342. else:
  343. auth_method = "Bearer (OAuth/setup-token)" if _is_oauth_token(key) else "x-api-key (API key)"
  344. _plines(
  345. agent,
  346. f" Auth method: {auth_method}",
  347. f" Token prefix: {key[:12]}..." if isinstance(key, str) and len(key) > 12 else " Token: (empty or short)",
  348. )
  349. _dhh = display_hermes_home()
  350. _plines(
  351. agent,
  352. " Troubleshooting:",
  353. f" • Check ANTHROPIC_TOKEN in {_dhh}/.env for Hermes-managed OAuth/setup tokens",
  354. f" • Check ANTHROPIC_API_KEY in {_dhh}/.env for API keys or legacy token values",
  355. " • For API keys: verify at https://platform.claude.com/settings/keys",
  356. " • Hermes login (OAuth): run 'hermes auth add anthropic' to sign in again, then retry",
  357. " • Inspect what Hermes holds: hermes auth list anthropic",
  358. " • Legacy cleanup: hermes config set ANTHROPIC_TOKEN \"\"",
  359. " • Clear stale keys: hermes config set ANTHROPIC_API_KEY \"\"",
  360. )
  361. def _refresh_credentials_after_401(
  362. agent: Any, api_error: Exception, _retry: TurnRetryState, status_code: Optional[int]
  363. ) -> bool:
  364. """Per-provider one-shot credential refresh on 401 (codex/xai, vertex, nous, a profile's
  365. ``refresh_runtime_credentials``, anthropic), printing user-facing diagnostics when the
  366. nous/anthropic refresh fails. Returns True when a refresh succeeded and the call should be retried."""
  367. from agent.conversation_loop import _credential_refresh_profile
  368. if status_code != 401:
  369. return False
  370. if (
  371. agent.api_mode == "codex_responses"
  372. and agent.provider in {"openai-codex", "xai-oauth"}
  373. and not _retry.codex_auth_retry_attempted
  374. ):
  375. _retry.codex_auth_retry_attempted = True
  376. if agent._try_refresh_codex_client_credentials(force=True):
  377. _label = "xAI OAuth" if agent.provider == "xai-oauth" else "Codex"
  378. agent._buffer_vprint(f"🔐 {_label} auth refreshed after 401. Retrying request...")
  379. return True
  380. if agent.api_mode == "chat_completions" and agent.provider == "vertex" and not _retry.vertex_auth_retry_attempted:
  381. _retry.vertex_auth_retry_attempted = True
  382. if agent._try_refresh_vertex_client_credentials():
  383. agent._buffer_vprint("🔐 Vertex AI token refreshed after 401. Retrying request...")
  384. return True
  385. if (
  386. agent.api_mode in ("chat_completions", "anthropic_messages")
  387. and agent.provider == "nous"
  388. and not _retry.nous_auth_retry_attempted
  389. ):
  390. _retry.nous_auth_retry_attempted = True
  391. from agent.credential_auth_recovery import nous_pool_refreshed_this_turn
  392. # The pool already spent this episode's one forced refresh (and its backoff retries):
  393. # a second force here would redeem the rotating grant again for a server-side 401.
  394. if not nous_pool_refreshed_this_turn(agent) and agent._try_refresh_nous_client_credentials(force=True):
  395. agent._buffer_vprint("🔐 Nous agent key refreshed after 401. Retrying request...")
  396. return True
  397. _print_nous_401_diagnostics(agent, api_error)
  398. if not _retry.provider_auth_refresh_attempted and (profile := _credential_refresh_profile(agent)) is not None:
  399. _retry.provider_auth_refresh_attempted = True
  400. if agent._try_refresh_provider_runtime_credentials(reason="unauthorized"):
  401. from agent.provider_route_hooks import ask_on_error_path
  402. agent._buffer_vprint("🔐 " + (ask_on_error_path(profile, "credential_refresh_notice", "", reason="unauthorized") or (
  403. f"{profile.display_name or profile.name} credentials refreshed after 401. Retrying request...")))
  404. return True
  405. if (
  406. agent.api_mode == "anthropic_messages"
  407. and hasattr(agent, '_anthropic_api_key')
  408. and not _retry.anthropic_auth_retry_attempted
  409. ):
  410. _retry.anthropic_auth_retry_attempted = True
  411. if agent._try_refresh_anthropic_client_credentials():
  412. _plines(agent, "🔐 Anthropic credentials refreshed after 401. Retrying request...")
  413. return True
  414. _print_anthropic_401_diagnostics(agent, agent._anthropic_api_key)
  415. return False
  416. def _recover_format_errors(
  417. agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState,
  418. messages: list[dict[str, Any]], api_messages: Any,
  419. ) -> bool:
  420. """One-shot format-recovery strips: thinking-signature → invalid-encrypted-content
  421. replay disable → native-compaction reject → llama.cpp grammar strip. Returns True when
  422. the request was repaired and should be retried."""
  423. # Upstream mutation can invalidate a thinking signature. Native Anthropic has multiple replay
  424. # carriers and preserved historical blocks, so suppress the rejected opaque blocks across all
  425. # carriers and persist that suppression. Other transports keep their established one-request
  426. # repair: drop reasoning_details only.
  427. if classified.reason == FailoverReason.thinking_signature and not _retry.thinking_sig_retry_attempted:
  428. _retry.thinking_sig_retry_attempted = True
  429. from agent.anthropic_thinking_replay import remember_rejected_thinking, tracks_rejected_thinking
  430. if tracks_rejected_thinking(agent):
  431. removed = remember_rejected_thinking(agent, api_messages)
  432. detail = "suppressed rejected Anthropic replay blocks"
  433. else:
  434. removed = 0
  435. for message in api_messages:
  436. if isinstance(message, dict) and "reasoning_details" in message:
  437. message.pop("reasoning_details", None)
  438. removed += 1
  439. detail = "stripped reasoning_details"
  440. _vlines(agent, f"⚠️ Thinking block signature invalid, {detail} and retrying...")
  441. logger.warning(
  442. "%sThinking block signature recovery: %s from %d carrier/message(s) "
  443. "(canonical messages unchanged)",
  444. agent.log_prefix, detail, removed,
  445. )
  446. return True
  447. # 400 ``invalid_encrypted_content`` on a stale ``codex_reasoning_items`` blob (the 401
  448. # ``token_expired`` twin is taken ahead of the credential pool in the caller).
  449. if classified.reason == FailoverReason.invalid_encrypted_content and _recover_stale_codex_reasoning(
  450. agent, _retry, messages, api_messages
  451. ):
  452. return True
  453. # Structured 400 naming ``context_management``: disable native compaction for the
  454. # session, retry once; local compression takes over.
  455. if (
  456. agent.api_mode == "codex_responses"
  457. and not _retry.native_compaction_reject_retry_attempted
  458. and bool(getattr(agent, "codex_responses_native_compaction", False))
  459. ):
  460. from agent.native_compaction import is_native_compaction_rejection
  461. if is_native_compaction_rejection(api_error, getattr(api_error, "status_code", None)):
  462. _retry.native_compaction_reject_retry_attempted = True
  463. agent.codex_responses_native_compaction = False
  464. _vlines(
  465. agent,
  466. "⚠️ Provider rejected native compaction (context_management) — disabled for this session, "
  467. "local compression stays active. Retrying...",
  468. )
  469. logger.warning(
  470. "%sNative compaction rejection recovery: disabled "
  471. "codex_responses_native for this session and retrying",
  472. agent.log_prefix,
  473. )
  474. return True
  475. # llama.cpp ``json-schema-to-grammar`` rejects regex escapes and most ``format``
  476. # values: strip ``pattern``/``format`` from ``agent.tools``, retry once.
  477. if classified.reason == FailoverReason.llama_cpp_grammar_pattern and not _retry.llama_cpp_grammar_retry_attempted:
  478. _retry.llama_cpp_grammar_retry_attempted = True
  479. try:
  480. from tools.schema_sanitizer import strip_pattern_and_format
  481. _, _stripped = strip_pattern_and_format(agent.tools)
  482. except Exception as _strip_exc: # pragma: no cover — defensive
  483. logger.warning("%sllama.cpp grammar recovery: strip helper failed: %s", agent.log_prefix, _strip_exc)
  484. _stripped = 0
  485. if _stripped:
  486. _vlines(agent, f"⚠️ llama.cpp rejected tool schema grammar — stripped {_stripped} pattern/format keyword(s), retrying...")
  487. logger.warning(
  488. "%sllama.cpp grammar recovery: stripped %d "
  489. "pattern/format keyword(s) from tool schemas",
  490. agent.log_prefix, _stripped,
  491. )
  492. return True
  493. # Nothing to strip — fall through to normal retry rather than loop on the same error.
  494. logger.warning(
  495. "%sllama.cpp grammar error but no pattern/format "
  496. "keywords to strip — falling through to normal retry",
  497. agent.log_prefix,
  498. )
  499. return False
  500. _WELCOME_ROUTE_HEAL_COPY = {
  501. "anon_on_paid_host": "Reconnected to the free model's own route.",
  502. "named_on_welcome_host": "Reconnected to your Nous account's own route.",
  503. "session_expired": "Renewed the connection to the free model.",
  504. }
  505. def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState) -> bool:
  506. """Two one-shot repairs for the Nous free tier, both silent on the wire and named once in chat.
  507. ``model_not_free``: the session asked the welcome host for a model it does not serve; move
  508. to the first alternate the gateway named (its own model) and retry, instead of failing the
  509. turn. ``anon_on_paid_host`` / ``named_on_welcome_host``: this process is pointed at the other
  510. identity's host (a stale route); re-read the credentials, which heals the URL, and retry. The
  511. refresh reports False when the store yields the same route, so a user-set
  512. ``NOUS_INFERENCE_BASE_URL`` falls straight through to the terminal copy. ``session_expired``
  513. (403 ``invalid jwt``): the same forced refresh re-exchanges the free credential.
  514. Reads the CLASSIFIER's context (``classified.error_context``): that is where
  515. ``_nous_welcome_tier`` parks ``welcome_refusal`` / ``welcome_route``. The turn's other context
  516. (``extract_api_error_context``) never carries them."""
  517. ctx = getattr(classified, "error_context", None) or {}
  518. refusal = ctx.get("welcome_refusal") if isinstance(ctx, dict) else None
  519. if isinstance(refusal, dict) and refusal.get("reason") == "model_not_free" and not _retry.welcome_model_switch_attempted:
  520. _retry.welcome_model_switch_attempted = True
  521. alternates = [a for a in (refusal.get("alternates") or []) if isinstance(a, str) and a]
  522. requested = str(getattr(agent, "model", "") or "")
  523. target = alternates[0] if alternates else None
  524. if target and target != requested:
  525. try:
  526. agent.model = target
  527. agent._nous_model_switch = (requested, target)
  528. except Exception:
  529. return False
  530. _vlines(agent, f"↪️ {requested} isn't available without signing in; using {target} for now. Retrying...")
  531. logger.info("%sNous free tier: moved %s -> %s after model_not_free", agent.log_prefix, requested, target)
  532. return True
  533. route = ctx.get("welcome_route") if isinstance(ctx, dict) else None
  534. if route in _WELCOME_ROUTE_HEAL_COPY and not _retry.welcome_route_heal_attempted:
  535. _retry.welcome_route_heal_attempted = True
  536. try:
  537. healed = bool(agent._try_refresh_nous_client_credentials(force=True))
  538. except Exception:
  539. healed = False
  540. if healed:
  541. _vlines(agent, f"🔐 {_WELCOME_ROUTE_HEAL_COPY[route]} Retrying request...")
  542. return True
  543. return False
  544. def _clamp_to_affordable_budget(agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState) -> bool:
  545. """A credit-limited 402 (OpenRouter: "...requested up to 65536 tokens, but can only afford
  546. 56272") means the account has credit and the output cap was too large: retry ONCE with the
  547. cap lowered to the affordable budget instead of abandoning the provider as billing (#49769).
  548. Same budget parse as the auxiliary client's retry. The 402 itself says the request's cap
  549. exceeded the budget (which may be a boosted continuation cap above ``agent.max_tokens``),
  550. so a second credit-limited 402 on this call is genuine exhaustion."""
  551. if classified.reason != FailoverReason.billing or _retry.affordable_402_clamp_attempted:
  552. return False
  553. from agent.auxiliary_client import _affordable_max_tokens_from_error
  554. affordable = _affordable_max_tokens_from_error(api_error)
  555. if affordable is None:
  556. return False
  557. _retry.affordable_402_clamp_attempted = True
  558. agent._ephemeral_max_output_tokens = affordable
  559. _vlines(agent, f"💳 Credit balance covers fewer output tokens — retrying with max_tokens={affordable:,}...")
  560. logger.info("%sCredit-limited 402: retrying once with max_tokens=%d", agent.log_prefix, affordable)
  561. return True
  562. def _reject_all_turns(agent: Any, api_error: Exception) -> bool:
  563. reject = getattr(agent._get_transport(), "reject_all_turns", None)
  564. return bool(reject(api_error)) if callable(reject) else False
  565. def _pool_error_context(agent, classified: Any, error_context: Any) -> Any:
  566. """What the credential pool sees: a plan limit (usage-window 402) benches only the refused model on the
  567. credential, until its reset; every other failure passes the error context through unchanged."""
  568. if not (getattr(classified, "error_context", None) or {}).get("usage_window"):
  569. return error_context
  570. return {**(error_context if isinstance(error_context, dict) else {}), "usage_window": True,
  571. **_usage_window_reset_context(agent)}
  572. def _usage_window_reset_context(agent) -> dict[str, float]:
  573. """``{"reset_at": epoch s}`` for a usage-window 402: when the refused allowance frees up, from the usage
  574. headers the 402 carried (the API's error body has no reset). Empty when the state names none, so the
  575. pool keeps its default bench."""
  576. try:
  577. from agent.credits_tracker import window_stop_resets_at_ms
  578. from agent.plan_limits import current_model_is_top_tier
  579. state = getattr(agent, "_credits_state", None)
  580. resets_ms = (window_stop_resets_at_ms(state, model_is_top_tier=current_model_is_top_tier(agent))
  581. if state is not None else None)
  582. except Exception:
  583. logger.debug("usage-window reset unavailable for the pool", exc_info=True)
  584. return {}
  585. return {"reset_at": resets_ms / 1000} if isinstance(resets_ms, (int, float)) and resets_ms > 0 else {}
  586. def recover_after_classification(
  587. agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState, *,
  588. status_code: Optional[int], error_context: Any, messages: list[dict[str, Any]],
  589. api_messages: Any,
  590. ) -> tuple[bool, bool]:
  591. """One-shot recovery chain that runs AFTER ``classify_api_error`` and before the
  592. generic retry path. Order is load-bearing (each branch may ``return`` early):
  593. welcome-tier repair → credit-limited 402 output-cap clamp → ``reasoning.context`` opt-out → Nous paid-entitlement refresh →
  594. Codex stale-reasoning strip on 401 ``token_expired`` → credential-pool rotation → image shrink → multimodal-tool-content strip → corrupt-image
  595. strip → Anthropic OAuth 1M-beta disable → per-provider 401 credential refresh →
  596. format-recovery strips.
  597. Returns ``(retry_now, recovered_with_pool)``; the latter feeds the Nous rate-limit guard."""
  598. from agent.conversation_loop import _is_nous_inference_route
  599. # The credit-limited 402 clamp runs before pool rotation, which would bench a credential
  600. # that still has credit. A ``reasoning.context`` 400 is classified reasoning_mandatory, whose rung
  601. # would drop the whole reasoning config; omit only the all_turns opt-in, ahead of it.
  602. if _recover_welcome_tier(agent, classified, _retry) or _clamp_to_affordable_budget(
  603. agent, api_error, classified, _retry
  604. ) or (agent.api_mode == "codex_responses" and _reject_all_turns(agent, api_error)):
  605. return True, False
  606. # 401 ``token_expired`` while the transcript still carries ``codex_reasoning_items`` is a
  607. # stale replayed blob far more often than a dead bearer (#88510): strip BEFORE the pool
  608. # refreshes/benches every healthy entry over a session-state problem. A real expiry pays
  609. # one extra round-trip and then takes the credential path below as before.
  610. if _is_codex_token_expired(agent, api_error) and _recover_stale_codex_reasoning(
  611. agent, _retry, messages, api_messages
  612. ):
  613. return True, False
  614. if (
  615. classified.reason == FailoverReason.billing
  616. # A usage-window 402 is the plan's own limit: refreshing credentials cannot lift it.
  617. and not (getattr(classified, "error_context", None) or {}).get("usage_window")
  618. and _is_nous_inference_route(
  619. getattr(agent, "provider", "") or "", getattr(agent, "base_url", "") or ""
  620. )
  621. and not _retry.nous_paid_entitlement_refresh_attempted
  622. ):
  623. _retry.nous_paid_entitlement_refresh_attempted = True
  624. if _try_refresh_nous_paid_entitlement_credentials(agent):
  625. _vlines(agent, "🔐 Nous paid access verified — refreshed runtime credentials and retrying request...")
  626. return True, False
  627. recovered_with_pool, _retry.has_retried_429 = agent._recover_with_credential_pool(
  628. status_code=status_code, has_retried_429=_retry.has_retried_429,
  629. classified_reason=classified.reason, error_context=_pool_error_context(agent, classified, error_context),
  630. billing_unverified=classified.billing_unverified,
  631. )
  632. if recovered_with_pool:
  633. return True, recovered_with_pool
  634. # Shrink oversized native image parts in-place and retry once.
  635. if classified.reason == FailoverReason.image_too_large and not _retry.image_shrink_retry_attempted:
  636. _retry.image_shrink_retry_attempted = True
  637. if agent._try_shrink_image_parts_in_messages(
  638. api_messages, max_dimension=_image_error_max_dimension(api_error) or 8000
  639. ):
  640. _vlines(agent, "📐 Image(s) exceeded provider size limit — shrank and retrying...")
  641. return True, recovered_with_pool
  642. logger.info(
  643. "image-shrink recovery: no data-URL image parts found "
  644. "or shrink didn't reduce size; surfacing original error."
  645. )
  646. # Strict OpenAI-spec providers 400 on list-type tool content: strip images, mark
  647. # (provider, model) no-list-tool-content for the session, retry once.
  648. if (
  649. classified.reason == FailoverReason.multimodal_tool_content_unsupported
  650. and not _retry.multimodal_tool_content_retry_attempted
  651. ):
  652. _retry.multimodal_tool_content_retry_attempted = True
  653. if agent._try_strip_image_parts_from_tool_messages(api_messages):
  654. _vlines(agent, "📐 Provider rejected list-type tool content — downgraded screenshots to text and retrying...")
  655. return True, recovered_with_pool
  656. logger.info(
  657. "multimodal-tool-content recovery: no list-type tool "
  658. "messages with image parts found; surfacing original error."
  659. )
  660. # Route rejecting a reasoning disable: a reasoning-mandatory route (Nous Portal / OpenRouter,
  661. # e.g. GLM-5.3) 400s on ``reasoning: {enabled: false}``; a chat-only OpenAI-compatible relay
  662. # 400s on the ``reasoning_effort: none`` the title/continuation disable projects (#114460).
  663. # The catalog guard in the provider profile normally swallows the first, but a process that
  664. # warmed its caps cache before the route flipped keeps sending it. One-shot: never send a
  665. # disable again this session (the wire builder omits it → route default), queue a catalog
  666. # refresh so the guard is right next time (no-op for providers without a catalog), retry.
  667. if (
  668. classified.reason == FailoverReason.reasoning_mandatory
  669. and not _retry.reasoning_mandatory_retry_attempted
  670. ):
  671. _retry.reasoning_mandatory_retry_attempted = True
  672. sent = getattr(agent, "_wire_reasoning_config", None)
  673. if isinstance(sent, dict) and sent.get("enabled") is not False and sent.get("effort") not in (None, "none"):
  674. # The rejected request carried an ENABLED config: the route refuses that reasoning
  675. # level (#100536: ``reasoning.effort: max`` on a Responses relay). Dropping a disable
  676. # would resend the identical request; omit the reasoning fields instead (route default).
  677. agent._reasoning_effort_rejected = True
  678. _vlines(agent, f"⚠️ {agent.model} rejects reasoning effort {sent['effort']} — using the route's default for this session, retrying...")
  679. logger.warning("%sReasoning-effort recovery: dropping reasoning config for %s", agent.log_prefix, agent.model)
  680. return True, recovered_with_pool
  681. agent._reasoning_disable_rejected = True
  682. # "Reasoning is mandatory ... cannot be disabled" understands the field and refuses only the
  683. # OFF: step up to the floor effort (the closest the route allows to what the user asked for)
  684. # rather than the route default. A relay that does not know the field at all keeps the
  685. # drop (a floor would 400 the same way).
  686. from agent.error_classifier import is_reasoning_required_rejection
  687. agent._reasoning_floor_required = is_reasoning_required_rejection(str(api_error))
  688. try:
  689. from hermes_cli.models_reasoning_caps import refresh_reasoning_caps_async
  690. refresh_reasoning_caps_async(agent.provider)
  691. except Exception:
  692. pass
  693. if agent._reasoning_floor_required:
  694. from agent.auxiliary_reasoning_floor import REASONING_FLOOR_EFFORT
  695. _vlines(agent, f"⚠️ {agent.model} cannot disable reasoning — using effort={REASONING_FLOOR_EFFORT} for this session, retrying...")
  696. logger.warning("%sReasoning-disable recovery: stepping reasoning up to %s for %s",
  697. agent.log_prefix, REASONING_FLOOR_EFFORT, agent.model)
  698. else:
  699. _vlines(agent, f"⚠️ {agent.model} rejects disabling reasoning — using the route's default for this session, retrying...")
  700. logger.warning("%sReasoning-disable recovery: dropping reasoning disable for %s", agent.log_prefix, agent.model)
  701. return True, recovered_with_pool
  702. # Provider rejected the image bytes; shrinking can't help, so strip image parts.
  703. # Strip ONLY the per-call copy: replacing msg["content"] on the shallow api_messages
  704. # rows keeps canonical history's images (transient rejection must not erase history).
  705. if classified.reason == FailoverReason.image_corrupt:
  706. if _strip_request_images_and_retry(agent, api_messages):
  707. return True, recovered_with_pool
  708. logger.info("image-corrupt recovery: no image parts found to strip; surfacing original error.")
  709. # Anthropic OAuth subscription rejected the 1M-context beta: disable it for this
  710. # session, rebuild the client, retry once. Reactive so capable subscriptions keep 1M.
  711. if (
  712. # See PR #17680 for the original report (we chose reactive recovery over the proposed unconditional
  713. # omit so capable subscriptions don't silently lose the capability).
  714. classified.reason == FailoverReason.oauth_long_context_beta_forbidden
  715. and agent.api_mode == "anthropic_messages"
  716. and agent._is_anthropic_oauth
  717. and not _retry.oauth_1m_beta_retry_attempted
  718. ):
  719. _retry.oauth_1m_beta_retry_attempted = True
  720. if not getattr(agent, "_oauth_1m_beta_disabled", False):
  721. agent._oauth_1m_beta_disabled = True
  722. try:
  723. agent._anthropic_client.close()
  724. except Exception:
  725. pass
  726. agent._rebuild_anthropic_client()
  727. _vlines(agent, "🔕 OAuth subscription doesn't support the 1M-context beta — disabled for this session and retrying...")
  728. return True, recovered_with_pool
  729. if _refresh_credentials_after_401(agent, api_error, _retry, status_code):
  730. return True, recovered_with_pool
  731. if _recover_format_errors(agent, api_error, classified, _retry, messages, api_messages):
  732. return True, recovered_with_pool
  733. return False, recovered_with_pool
  734. def _failed_turn_result(final_response: str, messages: Any, api_call_count: int, error: str) -> dict[str, Any]:
  735. """Base failed-turn result dict shared by the two terminal paths."""
  736. return {
  737. "final_response": final_response, "messages": messages, "api_calls": api_call_count,
  738. "completed": False, "failed": True, "error": error,
  739. }
  740. def settle_delivered_partial(agent: Any, messages: Any, current_turn_user_idx: Any) -> str:
  741. """Visible text already delivered this turn ("" when none), collapsing any continuation
  742. trail first so the terminal persist never keeps a dangling synthetic nudge (#119001).
  743. ``build_api_request`` resets ``_current_streamed_assistant_text`` per attempt, so after a
  744. mid-stream death + continuation + pre-stream error the live accumulator is empty and the
  745. fragment rows (now collapsed into one assistant row) are the only record.
  746. """
  747. from agent.turn_truncation import collapse_continuation_trail
  748. collapsed = collapse_continuation_trail(
  749. agent, messages, current_turn_user_idx, finish_reason="error",
  750. )
  751. live = getattr(agent, "_current_streamed_assistant_text", "")
  752. if isinstance(live, str) and live.strip():
  753. from agent.agent_runtime_helpers import strip_think_blocks
  754. return strip_think_blocks(agent, live).strip() or collapsed
  755. return collapsed
  756. def _with_delivered_partial(final_response: str, error_summary: str, delivered: str) -> tuple:
  757. """Prepend delivered partial text to a terminal error body ("" unchanged).
  758. Returns ``(final_response, keep_partial)``; callers set ``result["partial"]``
  759. when ``keep_partial`` so the gateway emits ``payload.partial`` and surfaces
  760. retain the bubble instead of clearing it. ``final_response`` must stay
  761. distinct from ``error`` — that inequality is the retention contract.
  762. """
  763. _delivered = (delivered or "").strip()
  764. if not _delivered or _delivered == (error_summary or "").strip():
  765. return final_response, False
  766. return f"{_delivered}\n\n{final_response}", True
  767. def limit_reset_epoch(agent: Any, api_error: Exception) -> Optional[float]:
  768. """Epoch seconds when the provider says its limit lifts (Retry-After header, ``resets_at`` /
  769. ``retry_after`` body fields, "try again in N" text) — the same datum the backoff honours."""
  770. from agent.credential_pool import _parse_absolute_timestamp
  771. try:
  772. return _parse_absolute_timestamp(agent._extract_api_error_context(api_error).get("reset_at"))
  773. except Exception: # advisory only — never break the error path
  774. return None
  775. # Responses for other failures can still carry rate-limit headers; a reset time only explains a limit.
  776. _LIMIT_REASONS = frozenset({FailoverReason.rate_limit, FailoverReason.upstream_rate_limit, FailoverReason.billing})
  777. def _stamp_limit_reset(result: dict[str, Any], agent: Any, api_error: Exception, classified: Any) -> None:
  778. """``failure_resets_at`` for structured clients (Desktop card: "Limit resets at HH:mm") and the
  779. same sentence appended to the chat text every plain surface (CLI/TUI/gateway) renders (#98852)."""
  780. if classified.reason not in _LIMIT_REASONS:
  781. return
  782. resets_at = limit_reset_epoch(agent, api_error)
  783. if resets_at is None:
  784. return
  785. result["failure_resets_at"] = resets_at
  786. if line := limit_reset_copy(resets_at):
  787. result["final_response"] = f"{result['final_response']}\n\n{line}"
  788. def _print_nonretryable_auth_guidance(
  789. agent: Any, classified: Any, *, status_code: Optional[int], provider: Any, base_url: Any, model: Any,
  790. ) -> None:
  791. """Actionable guidance for a terminal auth / billing error."""
  792. if (getattr(classified, "error_context", None) or {}).get("usage_window"):
  793. return # a plan limit's refusal line and limit panel are its guidance: no second copy, no portal fetch
  794. from agent.conversation_loop import _print_billing_or_entitlement_guidance, _print_nous_entitlement_guidance
  795. if classified.reason == FailoverReason.billing and _print_billing_or_entitlement_guidance(
  796. agent, capability="model access", provider=provider, base_url=str(base_url),
  797. model=model, unverified=classified.billing_unverified,
  798. ):
  799. return
  800. if provider == "nous" and _print_nous_entitlement_guidance(agent, "Nous model access"):
  801. return
  802. if provider in {"openai-codex", "xai-oauth", "nous"} and status_code == 401:
  803. if provider == "openai-codex":
  804. from agent.turn_failure_copy import oauth_relogin_command
  805. _vlines(
  806. agent,
  807. " 💡 Codex OAuth token was rejected (HTTP 401). Your token may have been",
  808. " refreshed by another client (Codex CLI, VS Code) or another Hermes profile.",
  809. f" Sign this profile in again: `{oauth_relogin_command(provider)}`",
  810. )
  811. elif provider == "xai-oauth":
  812. _vlines(
  813. agent,
  814. " 💡 xAI OAuth token was rejected (HTTP 401). To fix:",
  815. " re-authenticate with xAI Grok OAuth (SuperGrok / Premium+) from `hermes model`.",
  816. )
  817. else: # nous
  818. _vlines(
  819. agent,
  820. " 💡 Nous Portal OAuth token was rejected (HTTP 401). Your token may be",
  821. " expired, revoked, or your account may be out of credits. To fix:",
  822. " 1. Re-authenticate: hermes portal",
  823. " 2. Check your portal account: https://portal.nousresearch.com",
  824. )
  825. # ``:free`` is OpenRouter slug syntax; Nous Portal will reject the model
  826. # name even after a successful re-auth.
  827. if isinstance(model, str) and model.endswith(":free"):
  828. _vlines(
  829. agent,
  830. f" ⚠️ Note: `{model}` looks like an OpenRouter slug (`:free` suffix).",
  831. " Nous Portal won't recognize that model name. Either switch to a",
  832. f" Nous catalog model, or run `/model openrouter:{model}` to use OpenRouter.",
  833. )
  834. return
  835. from hermes_cli.auth_plugin_providers import plugin_refresh_hook
  836. if status_code == 401 and plugin_refresh_hook(str(provider or "")) is not None:
  837. # A plugin OAuth grant the pool could not refresh: the fix is a fresh sign-in, not a key.
  838. from agent.turn_failure_copy import oauth_relogin_command
  839. _vlines(agent, f" 💡 {provider} sign-in was rejected (HTTP 401). Sign in again: `{oauth_relogin_command(provider)}`")
  840. return
  841. _vlines(
  842. agent,
  843. " 💡 Your API key was rejected by the provider. Check:",
  844. " • Is the key valid? Run: hermes setup",
  845. f" • Does your account have access to {model}?",
  846. )
  847. if base_url_host_matches(str(base_url), "openrouter.ai"):
  848. _vlines(agent, " • Check credits: https://openrouter.ai/settings/credits")
  849. def _welcome_tier_guidance(classified: Any, *, model: Any, in_chat: bool, door: bool = True) -> str:
  850. """Copy for a Nous free-tier refusal the classifier parsed (``welcome_refusal`` /
  851. ``welcome_route`` in ``error_context``); empty for every other error."""
  852. ctx = getattr(classified, "error_context", None) or {}
  853. refusal, route = ctx.get("welcome_refusal"), ctx.get("welcome_route")
  854. if not refusal and not route:
  855. return ""
  856. from hermes_cli.anon_auth import welcome_refusal_copy, welcome_route_refusal_copy
  857. if refusal:
  858. return welcome_refusal_copy(refusal, model=str(model or ""), in_chat=in_chat, door=door)
  859. return welcome_route_refusal_copy(str(route), in_chat=in_chat, door=door)
  860. # Closed table: every card kind the desktop has copy for. An unknown gateway reason lands on
  861. # "refused" (generic card, sentence kept) rather than a code the desktop cannot key on.
  862. _WELCOME_SURFACE_KINDS = {
  863. "rate_limited": "rate_limited", "at_capacity": "at_capacity", "admission_closed": "at_capacity",
  864. "model_not_free": "model_not_free", "feature_not_free": "model_not_free",
  865. }
  866. def _welcome_surface_kind(classified: Any) -> str:
  867. """The free-tier failure kind a client renders its card from (``error_surface`` code
  868. ``free_tier_<kind>``): the welcome refusal's reason, or the route refusal; "" otherwise."""
  869. ctx = getattr(classified, "error_context", None) or {}
  870. refusal = ctx.get("welcome_refusal") if isinstance(ctx, dict) else None
  871. if isinstance(refusal, dict):
  872. return _WELCOME_SURFACE_KINDS.get(str(refusal.get("reason") or ""), "refused")
  873. route = ctx.get("welcome_route") if isinstance(ctx, dict) else None
  874. if route == "tier_disabled":
  875. return "disabled"
  876. # The renewal failed but the tier is up: a plain Retry mints a new credential.
  877. if route == "session_expired":
  878. return "outage"
  879. # A named account on the welcome host has already signed in: no sign-in card, copy only.
  880. if route == "named_on_welcome_host":
  881. return ""
  882. return "route" if route else ""
  883. def _stamp_free_tier(result: dict[str, Any], kind: str, message: str) -> dict[str, Any]:
  884. """Structured free-tier failure block: ``error_surface`` keys its code on ``kind`` and a client
  885. shows ``message`` (the chat sentence) as the card body instead of its own generic copy."""
  886. result["free_tier"] = {"kind": kind or "refused", "message": message}
  887. return result
  888. def _welcome_outage_copy(agent: Any, base_url: Any, classified: Any) -> str:
  889. """On the Nous free tier, a transport / server failure that outlived every retry reads as one
  890. plain sentence (the free model is having trouble) rather than the technical summary. Empty
  891. for every other route and for rate limits / billing, which have their own copy."""
  892. from hermes_cli.anon_auth import FREE_TIER_OUTAGE_COPY, on_free_model
  893. # Not ``unknown``: that is the classifier's catch-all for status-less local failures, which
  894. # are not the free model's trouble.
  895. if on_free_model(agent, base_url) and classified.reason in (
  896. FailoverReason.timeout, FailoverReason.overloaded, FailoverReason.server_error):
  897. return FREE_TIER_OUTAGE_COPY
  898. return ""
  899. # Terminal status label per non-retryable reason (default names the HTTP status).
  900. _NONRETRYABLE_LABELS = {
  901. FailoverReason.content_policy_blocked: "The provider's safety filter refused this request",
  902. FailoverReason.upstream_blocked: "A firewall/CDN in front of the provider blocked this request",
  903. FailoverReason.ssl_cert_verification: "The provider's security certificate could not be verified",
  904. # Only reached after the one-shot image shrink ran (recover_after_classification sets the flag first).
  905. FailoverReason.image_too_large: "Request still exceeded the provider's size limit after shrinking images",
  906. }
  907. def _missing_vendor_prefix_suggestion(api_error: Exception, provider: Any, model: Any) -> Optional[str]:
  908. """Prefixed catalogue id when a bare 404 most likely means ``vendor/model`` lost its prefix."""
  909. if getattr(api_error, "status_code", None) != 404:
  910. return None
  911. try:
  912. from hermes_cli.model_normalize import suggest_prefixed_model_id
  913. return suggest_prefixed_model_id(str(provider or ""), str(model or ""))
  914. except Exception:
  915. return None
  916. def nonretryable_client_error_result(
  917. agent: Any, api_error: Exception, classified: Any, *, status_code: Optional[int],
  918. api_kwargs: Any, api_messages: Any, messages: list[dict[str, Any]], conversation_history: Any,
  919. api_call_count: int, approx_tokens: int, provider: Any, base_url: Any, model: Any,
  920. delivered: str = "",
  921. ) -> dict[str, Any]:
  922. """Terminal path for a non-retryable 4xx once fallback is exhausted: debug dump, flush
  923. the retry trace, print auth / billing / content-policy / TLS guidance, persist (skipped
  924. for likely context-overflow 400s so the failure does not grow the session), build result."""
  925. # Result/guidance helpers stay in the loop module (tests import + patch them there).
  926. from agent.conversation_loop import (
  927. _billing_failure_result, _content_policy_blocked_result, _usage_window_refusal,
  928. )
  929. if api_kwargs is not None:
  930. agent._dump_api_request_debug(api_kwargs, reason="non_retryable_client_error", error=api_error)
  931. # Terminal — flush buffered context so the user sees what was tried before the abort.
  932. agent._flush_status_buffer()
  933. # Summarize once: Cloudflare/proxy HTML pages and raw provider bodies must be
  934. # collapsed here or they leak verbatim via the ``error`` field.
  935. _nonretryable_summary = agent._summarize_api_error(api_error)
  936. _plabel = provider_label_for(provider)
  937. _label = _NONRETRYABLE_LABELS.get(classified.reason, f"{_plabel} rejected the request and retrying won't help")
  938. agent._emit_diagnostic_status(f"❌ {_label}: {_nonretryable_summary}")
  939. # The endpoint/status trace is developer detail: verbose only (the log has it always).
  940. if getattr(agent, "verbose_logging", False):
  941. _vlines(
  942. agent,
  943. f" 🔌 Provider: {provider} Model: {model} (HTTP {status_code})",
  944. f" 🌐 Endpoint: {base_url}",
  945. )
  946. _welcome_hint = _welcome_tier_guidance(classified, model=model, in_chat=False)
  947. _prefix_suggestion = _missing_vendor_prefix_suggestion(api_error, provider, model)
  948. from hermes_cli.left_core_migration import host_install_hint
  949. if _moved_hint := host_install_hint(base_url): # a host whose plugin left core: install it first
  950. _vlines(agent, f" 💡 {_moved_hint}")
  951. if _welcome_hint:
  952. # A free-tier gate or a wrong-host refusal: the way forward is a sign-in or another
  953. # provider, never the key/credits advice below.
  954. _vlines(agent, f" 💡 {_welcome_hint}")
  955. elif classified.is_auth or classified.reason == FailoverReason.billing:
  956. _print_nonretryable_auth_guidance(
  957. agent, classified, status_code=status_code, provider=provider, base_url=base_url, model=model
  958. )
  959. elif classified.reason == FailoverReason.model_not_found:
  960. _vlines(agent, f" 💡 Model '{model}' isn't available on {_plabel}. Pick another with /model.")
  961. if _prefix_suggestion:
  962. _vlines(agent, f" Did you mean '{_prefix_suggestion}'? It looks like the vendor prefix is missing.")
  963. elif classified.reason not in _NONRETRYABLE_LABELS:
  964. _vlines(agent, f" 💡 Fix: pick another model (/model), or check `{display_hermes_home()}/logs/agent.log`.")
  965. # A WAF/CDN block (#53099, #70566): the key never reached the provider; the usual cause
  966. # is the SDK User-Agent, which the per-provider extra_headers override.
  967. if classified.reason == FailoverReason.upstream_blocked:
  968. _vlines(
  969. agent,
  970. " 💡 The endpoint's firewall/CDN blocked the request before it reached the model — your key",
  971. " and model access are probably fine. Relays often reject the SDK's default User-Agent:",
  972. " set `extra_headers: {User-Agent: HermesAgent/1.0}` on the custom_providers entry,",
  973. " or check the proxy/WAF rules and your network.",
  974. )
  975. # Content-policy blocks: the provider refused this prompt, so recovery is a rephrase
  976. # or another model, not key/retry advice.
  977. if classified.reason == FailoverReason.content_policy_blocked:
  978. _vlines(
  979. agent,
  980. f" 💡 {CONTENT_POLICY_NEXT_STEPS}",
  981. " To route future blocks to another provider automatically: hermes fallback add",
  982. )
  983. # TLS certificate failures are environment problems — name the knobs for each cause.
  984. if classified.reason == FailoverReason.ssl_cert_verification:
  985. _vlines(
  986. agent,
  987. " 💡 Hermes couldn't verify the provider's security certificate. This fails the same",
  988. " way on every retry — fix the environment, then try again:",
  989. " • Corporate TLS-inspecting proxy? Ask your administrator to install",
  990. " its root certificate in the operating system trust store.",
  991. " • Missing/stale system CA store? Refresh the OS certificate store.",
  992. " A provider-specific CA can also be configured with ssl_ca_cert.",
  993. " • Self-signed local endpoint (llama.cpp, LM Studio, vLLM)? Use http://",
  994. " for localhost, or add the server's cert to your trust store.",
  995. )
  996. logger.error("%sNon-retryable client error: %s", agent.log_prefix, api_error)
  997. # Skip persistence on likely context-overflow (400 + large session): persisting the
  998. # failed message grows the session and repeats the failure.
  999. # Persisting the failed user message would make the session even larger, causing the same failure on the
  1000. # next attempt. (#1630)
  1001. if status_code == 400 and (approx_tokens > 50000 or len(api_messages) > 80):
  1002. _vlines(agent, "⚠️ Skipping session persistence for large failed session to prevent growth loop.")
  1003. else:
  1004. agent._persist_session(messages, conversation_history)
  1005. if classified.reason == FailoverReason.content_policy_blocked:
  1006. return _content_policy_blocked_result(
  1007. messages, api_call_count,
  1008. final_response="⚠️ " + content_policy_copy(label=_plabel, summary=_nonretryable_summary),
  1009. error_detail=_nonretryable_summary,
  1010. )
  1011. # Billing walls get the same structured recovery descriptor as the max-retries path
  1012. # so every surface renders one consistent signal.
  1013. if classified.reason == FailoverReason.billing:
  1014. return _billing_failure_result(
  1015. classified=classified, summary=_nonretryable_summary, messages=messages,
  1016. api_call_count=api_call_count, provider=provider, base_url=base_url, model=model,
  1017. usage_window=_usage_window_refusal(agent, classified, api_messages),
  1018. )
  1019. if _welcome_hint:
  1020. # A free-tier refusal is fully explained by its own sentence; the raw provider summary
  1021. # (status codes, JSON) is for the log, not for a first-time user's chat.
  1022. _final_response = _welcome_tier_guidance(classified, model=model, in_chat=True)
  1023. else:
  1024. # Every surface reads final_response; the CLI hint lines above never reach chat.
  1025. _final_response = nonretryable_copy(
  1026. classified, provider=provider, model=model, summary=_nonretryable_summary,
  1027. prefix_suggestion=_prefix_suggestion,
  1028. )
  1029. # Same verdict fields as the max-retries path: without them the UI descriptor
  1030. # (agent/error_surface.py) reads a rejected OAuth token as a retryable
  1031. # "Provider error" and offers Retry instead of a re-login.
  1032. _final_response, _keep_partial = _with_delivered_partial(
  1033. _final_response, _nonretryable_summary, delivered,
  1034. )
  1035. result = _failed_turn_result(_final_response, messages, api_call_count, _nonretryable_summary)
  1036. result.update({
  1037. "failure_reason": classified.reason.value,
  1038. "failure_retryable": bool(classified.retryable),
  1039. })
  1040. if _keep_partial:
  1041. result["partial"] = True
  1042. _stamp_limit_reset(result, agent, api_error, classified)
  1043. if _welcome_hint and (_kind := _welcome_surface_kind(classified)):
  1044. # The card form: the desktop renders the sign-in as a button, so no "To sign in" tail.
  1045. _stamp_free_tier(result, _kind,
  1046. _welcome_tier_guidance(classified, model=model, in_chat=True, door=False))
  1047. return result
  1048. _STREAM_DROP_MARKERS = (
  1049. "connection lost", "connection reset", "connection closed", "network connection",
  1050. "network error", "terminated",
  1051. )
  1052. def max_retries_exhausted_result(
  1053. agent: Any, api_error: Exception, classified: Any, *, attempts: int, is_rate_limited: bool,
  1054. error_msg: str, api_kwargs: Any, api_messages: Any, messages: list[dict[str, Any]],
  1055. conversation_history: Any, api_call_count: int, approx_tokens: int, provider: Any,
  1056. base_url: Any, model: Any, delivered: str = "",
  1057. ) -> dict[str, Any]:
  1058. """Terminal path once retries, transport recovery and fallback all failed: flush the
  1059. trace, emit the billing / rate-limit / generic status, print stream-drop or thinking-timeout
  1060. guidance (the latter wins), persist, build the result with ``failure_reason`` /
  1061. ``failure_retryable`` / ``billing_block``."""
  1062. # Result/guidance helpers stay in the loop module (tests import + patch them there).
  1063. from agent.conversation_loop import (
  1064. _billing_block_dict, _billing_or_entitlement_message, _billing_terminal_label,
  1065. _print_billing_or_entitlement_guidance,
  1066. )
  1067. agent._flush_status_buffer()
  1068. _final_summary = agent._summarize_api_error(api_error)
  1069. _billing_guidance = ""
  1070. _is_billing = classified.reason == FailoverReason.billing
  1071. if _is_billing:
  1072. if classified.billing_unverified:
  1073. # Ambiguous body — hedge the terminal line.
  1074. agent._emit_diagnostic_status(
  1075. "❌ Provider reported usage/credit exhaustion "
  1076. f"(unverified — may be a content-filter rejection) — {_final_summary}"
  1077. )
  1078. else:
  1079. agent._emit_diagnostic_status(f"❌ Billing or credits exhausted — {_final_summary}")
  1080. _billing_kw = dict(
  1081. capability="model access", provider=provider, base_url=str(base_url), model=model,
  1082. unverified=classified.billing_unverified,
  1083. )
  1084. # A plan limit (usage-window 402) carries its own refusal line: no portal fetch, no second copy.
  1085. if not (classified.error_context or {}).get("usage_window"):
  1086. _billing_guidance = _billing_or_entitlement_message(**_billing_kw)
  1087. _print_billing_or_entitlement_guidance(agent, **_billing_kw)
  1088. elif is_rate_limited:
  1089. _reset = reset_hint(api_error)
  1090. agent._emit_diagnostic_status(
  1091. f"❌ Rate limited after {attempts} {'attempt' if attempts == 1 else 'retries'} — {_final_summary}"
  1092. f"{f' (resets in {_reset})' if _reset else ''}"
  1093. )
  1094. else:
  1095. agent._emit_diagnostic_status(f"❌ API failed after {attempts} retries — {_final_summary}")
  1096. _vlines(agent, f" 💀 Final error: {_final_summary}")
  1097. _welcome_hint = _welcome_tier_guidance(classified, model=model, in_chat=False)
  1098. if _welcome_hint:
  1099. _vlines(agent, f" 💡 {_welcome_hint}")
  1100. # SSE stream-drop (e.g. "Network connection lost"): usually a proxy/CDN cutting a very
  1101. # large tool call mid-response.
  1102. _is_stream_drop = (
  1103. not getattr(api_error, "status_code", None)
  1104. and any(p in error_msg for p in _STREAM_DROP_MARKERS)
  1105. )
  1106. if _is_stream_drop:
  1107. _vlines(
  1108. agent,
  1109. " 💡 The provider's stream connection keeps dropping. This often happens "
  1110. "when the model tries to write a very large file in a single tool call.",
  1111. " Try asking the model to use execute_code with Python's open() for "
  1112. "large files, or to write the file in smaller sections.",
  1113. )
  1114. # A known reasoning model hit a transport error before the first content token.
  1115. # Distinct from _is_stream_drop; detection lives in agent.thinking_timeout_guidance.
  1116. _is_thinking_timeout = is_thinking_timeout(classified, model, error_msg)
  1117. if _is_thinking_timeout:
  1118. _vlines(agent, f" 💡 {build_thinking_timeout_guidance(provider=provider, model=model).strip()}")
  1119. logger.error(
  1120. "%sAPI call failed after %s retries. %s | provider=%s model=%s msgs=%s tokens=~%s",
  1121. agent.log_prefix, attempts, _final_summary,
  1122. provider, model, len(api_messages), f"{approx_tokens:,}",
  1123. )
  1124. if api_kwargs is not None:
  1125. agent._dump_api_request_debug(api_kwargs, reason="max_retries_exhausted", error=api_error)
  1126. agent._persist_session(messages, conversation_history)
  1127. _billing_block = None
  1128. _billing_unverified = False
  1129. _free_tier_kind = ""
  1130. if _is_billing:
  1131. _billing_unverified = classified.billing_unverified
  1132. _final_response = _billing_terminal_label(_final_summary, _billing_unverified)
  1133. if _billing_guidance:
  1134. _final_response += f"\n\n{_billing_guidance}"
  1135. # Structured recovery descriptor so every surface renders the same link + label.
  1136. _billing_block = _billing_block_dict(
  1137. provider, base_url, model, _billing_guidance, unverified=_billing_unverified
  1138. )
  1139. else:
  1140. # Every surface reads final_response (the 💡 lines above are CLI-only), so the chat
  1141. # text carries the plain what-happened + next step itself.
  1142. _reset_at = classified.error_context.get("reset_at")
  1143. _final_response = exhausted_copy(
  1144. classified.reason.value, label=provider_label_for(provider), attempts=attempts, error=api_error,
  1145. summary=_final_summary, reset_seconds=_reset_at - time.time() if _reset_at else None,
  1146. )
  1147. if _welcome_hint:
  1148. _final_response = _welcome_tier_guidance(classified, model=model, in_chat=True)
  1149. _free_tier_kind = _welcome_surface_kind(classified)
  1150. elif _outage := _welcome_outage_copy(agent, base_url, classified):
  1151. _final_response, _free_tier_kind = _outage, "outage"
  1152. if _is_thinking_timeout:
  1153. # Thinking-timeout guidance overrides stream-drop guidance, which would wrongly
  1154. # suggest splitting large file writes.
  1155. _final_response += "\n\n" + build_thinking_timeout_guidance(provider=provider, model=model)
  1156. elif _is_stream_drop:
  1157. _final_response += (
  1158. "\n\nThe connection kept dropping while the model was writing — this often "
  1159. "happens when it writes a very large file in one go. Ask me to write the file in "
  1160. "smaller sections (or via execute_code with Python's open())."
  1161. )
  1162. result = _failed_turn_result(_final_response, messages, api_call_count, _final_summary)
  1163. result.update({
  1164. # Classified reason so callers (kanban worker in cli.py) can tell a quota wall
  1165. # (``rate_limit`` / ``billing``) from a task failure.
  1166. "failure_reason": classified.reason.value,
  1167. # The classifier's own retry verdict — UI surfaces use this, not the reason string.
  1168. "failure_retryable": bool(classified.retryable),
  1169. # True when the billing verdict rests on an ambiguous body.
  1170. "billing_unverified": _billing_unverified,
  1171. # Present only for billing walls: (provider, billing_url, is_nous, message).
  1172. "billing_block": _billing_block,
  1173. })
  1174. # Retry-exhaustion after partial delivery (#119001): the text was already
  1175. # shown, so keep it as the reply (marked failed) instead of an error-only
  1176. # turn — the gateway flags ``partial`` and surfaces retain the bubble.
  1177. _final_response, _keep_partial = _with_delivered_partial(
  1178. _final_response, _final_summary, delivered,
  1179. )
  1180. if _keep_partial:
  1181. result["final_response"] = _final_response
  1182. result["partial"] = True
  1183. _stamp_limit_reset(result, agent, api_error, classified)
  1184. if _free_tier_kind:
  1185. _stamp_free_tier(result, _free_tier_kind, (
  1186. _welcome_tier_guidance(classified, model=model, in_chat=True, door=False)
  1187. if _welcome_hint else _final_response))
  1188. return result
  1189. def log_api_error_attempt(
  1190. agent: Any, api_error: Exception, *, retry_count: int, max_retries: int,
  1191. status_code: Optional[int], elapsed_time: float, api_messages: Any, approx_tokens: int,
  1192. retryable: bool = True,
  1193. ) -> tuple[str, str, Any, Any, Any]:
  1194. """Log one failed API attempt (warning + buffered retry trace, OpenRouter "no tool
  1195. endpoints" hint, bare-404 missing-vendor-prefix hint); the buffer only surfaces if every
  1196. retry+fallback exhausts. Returns ``(error_type, error_msg, provider, base_url, model)``.
  1197. ``retryable=False`` (the classifier's verdict, e.g. a 401 on a static-key route) is
  1198. named on the line: a bare ``attempt 1/3`` promises a second attempt that never comes
  1199. and sends readers hunting for a retry bug (#73237)."""
  1200. error_type = type(api_error).__name__
  1201. error_msg = str(api_error).lower()
  1202. _error_summary = agent._summarize_api_error(api_error)
  1203. _attempt = f"attempt {retry_count}/{max_retries}" + ("" if retryable else ", not retryable")
  1204. logger.warning(
  1205. "API call failed (%s) error_type=%s %s summary=%s",
  1206. _attempt, error_type, agent._client_log_context(), _error_summary,
  1207. )
  1208. _provider = getattr(agent, "provider", "unknown")
  1209. _base = getattr(agent, "base_url", "unknown")
  1210. _model = getattr(agent, "model", "unknown")
  1211. _blines(agent, f"⚠️ {_attempt[0].upper()}{_attempt[1:]} failed: {_error_summary}")
  1212. # Exception class, endpoint, raw body and token counts are developer detail: verbose only.
  1213. if getattr(agent, "verbose_logging", False):
  1214. _status_code_str = f" [HTTP {status_code}]" if status_code else ""
  1215. _blines(
  1216. agent,
  1217. f" 🔌 {error_type}{_status_code_str} Provider: {_provider} Model: {_model}",
  1218. f" 🌐 Endpoint: {_base}",
  1219. )
  1220. if status_code and status_code < 500:
  1221. _err_body = getattr(api_error, "body", None)
  1222. _err_body_str = str(_err_body)[:300] if _err_body else None
  1223. if _err_body_str:
  1224. _blines(agent, f" 📋 Details: {_err_body_str}")
  1225. _blines(agent, f" ⏱️ Elapsed: {elapsed_time:.2f}s Context: {len(api_messages)} msgs, ~{approx_tokens:,} tokens")
  1226. if agent._is_openrouter_url() and "support tool use" in error_msg:
  1227. _blines(agent, f" 💡 No OpenRouter providers for {_model} support tool calling with your current settings.")
  1228. from agent.chat_completion_helpers import _provider_preferences_for_agent
  1229. if _provider_preferences_for_agent(agent).get("only"):
  1230. _blines(
  1231. agent,
  1232. " Your provider_routing.only restriction is filtering out tool-capable providers.",
  1233. " Try removing the restriction or adding providers that support tools for this model.",
  1234. )
  1235. _blines(agent, f" Check which providers support tools: https://openrouter.ai/models/{_model}")
  1236. # Bare 404 on a ``vendor/model`` catalogue usually means the id lost its prefix; the
  1237. # provider never names the model, so we do.
  1238. _suggestion = _missing_vendor_prefix_suggestion(api_error, _provider, _model)
  1239. if _suggestion:
  1240. _blines(
  1241. agent,
  1242. f" 💡 Model '{_model}' is not a valid id for provider {_provider} — it is missing its vendor prefix.",
  1243. f" Did you mean '{_suggestion}'? Re-pick it with /model.",
  1244. )
  1245. return error_type, error_msg, _provider, _base, _model
  1246. def abort_turn_on_interrupt(
  1247. agent: Any, messages: list[dict[str, Any]], conversation_history: Any, api_call_count: int, *,
  1248. abort_message: str, interrupt_text: str,
  1249. ) -> dict[str, Any]:
  1250. """Announce ``abort_message``, close any open tool sequence with ``interrupt_text``,
  1251. persist, clear the interrupt and return the ``interrupted`` result dict."""
  1252. _vlines(agent, f"⚡ {abort_message}")
  1253. # Empty-response recovery can leave a synthetic assistant+nudge pair after an
  1254. # already-executed tool result. Strip only that request-local scaffold before
  1255. # closing, so this exit owner can persist its specific interrupt reason.
  1256. agent._drop_trailing_empty_response_scaffolding(messages)
  1257. close_interrupted_tool_sequence(messages, interrupt_text, stopped=True)
  1258. agent._persist_session(messages, conversation_history)
  1259. # The turn was stopped, not rebuilt: a pending steer was aimed at this turn's next
  1260. # tool iteration, which will no longer happen — drop it (hard-cancel semantics).
  1261. agent.clear_interrupt(hard_cancel=True)
  1262. return {
  1263. "final_response": interrupt_text, "messages": messages, "api_calls": api_call_count,
  1264. "completed": False, "interrupted": True,
  1265. }
  1266. def interruptible_backoff_sleep(
  1267. agent: Any, wait_time: float, _retry: Optional[TurnRetryState], *,
  1268. messages: list[dict[str, Any]], conversation_history: Any, api_call_count: int,
  1269. abort_message: str, interrupt_text: str, activity_label: str,
  1270. ) -> Optional[dict[str, Any]]:
  1271. """Sleep ``wait_time`` in 200 ms slices so interrupts are honoured promptly, touching
  1272. activity every ~30 s so the gateway's inactivity monitor knows we are alive.
  1273. On interrupt with ``_retry`` given and a redirect pending: preserve the redirect, arm
  1274. ``_retry.restart_with_redirected_messages`` and return ``None`` (caller rebuilds the
  1275. turn). Otherwise return the ``interrupted`` result dict. ``None`` when the wait completed."""
  1276. sleep_end = time.time() + wait_time
  1277. _touch_counter = 0
  1278. while time.time() < sleep_end:
  1279. if agent._interrupt_requested:
  1280. if _retry is not None and agent.clear_interrupt(preserve_redirect=True):
  1281. _retry.restart_with_redirected_messages = True
  1282. return None
  1283. return abort_turn_on_interrupt(
  1284. agent, messages, conversation_history, api_call_count,
  1285. abort_message=abort_message, interrupt_text=interrupt_text,
  1286. )
  1287. time.sleep(0.2)
  1288. _touch_counter += 1
  1289. if _touch_counter % 150 == 0: # 150 × 0.2s = 30s
  1290. agent._touch_activity(f"{activity_label}, {int(sleep_end - time.time())}s remaining")
  1291. # A long wait can outlive a short-lived credential (the free tier's JWT lives 15 minutes), and
  1292. # the per-iteration pre-expiry adoption does not run between retries.
  1293. agent._adopt_nous_key_before_expiry()
  1294. return None
  1295. _ZAI_POLICY_NOTES = {
  1296. "zai_coding_overload_long": " (Z.AI Coding overload adaptive long backoff)",
  1297. "zai_coding_overload_short": " (Z.AI Coding overload short retry)",
  1298. }
  1299. def reset_hint(api_error: Exception) -> str:
  1300. """``"~13m"`` until the ``reset_at`` parsed from *api_error* (epoch s/ms or ISO-8601), else ``""``.
  1301. A bare "Rate limited. Waiting 60s" hides the one fact that decides whether to wait or switch
  1302. models (#26889): a per-minute throttle and a 13-minute plan window look identical without it."""
  1303. from agent.agent_runtime_helpers import extract_api_error_context
  1304. from agent.credential_pool import _parse_absolute_timestamp
  1305. from agent.usage_pricing import format_duration_compact
  1306. reset_at = extract_api_error_context(api_error).get("reset_at")
  1307. if reset_at is None:
  1308. return ""
  1309. remaining = (_parse_absolute_timestamp(reset_at) or 0.0) - time.time()
  1310. return f"~{format_duration_compact(remaining)}" if remaining >= 1 else ""
  1311. def compute_error_backoff(
  1312. agent: Any, api_error: Exception, *, retry_count: int, max_retries: int, is_rate_limited: bool,
  1313. is_zai_coding_overload: bool, base_url: Any, model: Any,
  1314. ) -> float:
  1315. """Pick the wait before the next API retry and announce it. Retry-After wins for
  1316. rate limits and any other retryable error (capped at ``RETRY_AFTER_CAP_S``); otherwise
  1317. jittered backoff, replaced by the adaptive policy for 429s / Z.AI overloads. Retries are
  1318. buffered (replayed only if every retry fails); a long Z.AI Coding wait or a non-rate-limit
  1319. ``Retry-After`` over ``LIVE_RETRY_WAIT_CAP_S`` is announced when it starts."""
  1320. # Imported lazily so tests that patch ``agent.retry_utils.jittered_backoff`` /
  1321. # ``adaptive_rate_limit_backoff`` (incl. the run_agent conftest fast-backoff fixture) intercept.
  1322. from agent.retry_utils import (
  1323. LIVE_RETRY_WAIT_CAP_S, RETRY_AFTER_CAP_S, adaptive_rate_limit_backoff, jittered_backoff,
  1324. provider_retry_after_seconds,
  1325. )
  1326. from hermes_cli.anon_auth import on_free_model
  1327. # Respect Retry-After on every retryable provider error, not just 429s. Retryable
  1328. # 5xx responses (e.g. Cloudflare 520/524) also carry the header or a structured
  1329. # ``retry_after`` problem-detail body field; ignoring either turns an origin
  1330. # outage into a retry storm.
  1331. _retry_after = provider_retry_after_seconds(api_error)
  1332. if _retry_after is not None:
  1333. _retry_after = min(_retry_after, RETRY_AFTER_CAP_S)
  1334. wait_time = _retry_after if _retry_after is not None else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0)
  1335. _backoff_policy = None
  1336. _adaptive = is_rate_limited or is_zai_coding_overload
  1337. if _adaptive and _retry_after is None:
  1338. wait_time, _backoff_policy = adaptive_rate_limit_backoff(
  1339. retry_count, base_url=str(base_url), model=model, error=api_error, default_wait=wait_time,
  1340. )
  1341. _reset = reset_hint(api_error) if _adaptive else ""
  1342. _free_busy = is_rate_limited and on_free_model(agent, base_url)
  1343. _wait_reason = ("The free model is busy" if _free_busy
  1344. else "Provider overloaded" if is_zai_coding_overload and not is_rate_limited else "Rate limited")
  1345. if _adaptive:
  1346. _policy_note = _ZAI_POLICY_NOTES.get(_backoff_policy or "", "")
  1347. _status = (f"⏱️ {_wait_reason}.{f' Resets in {_reset}.' if _reset else ''} Waiting {wait_time:.1f}s "
  1348. f"(attempt {retry_count + 1}/{max_retries}){_policy_note}...")
  1349. _announce_now = _backoff_policy == "zai_coding_overload_long"
  1350. else:
  1351. _status = f"⏳ Retrying in {wait_time:.1f}s (attempt {retry_count}/{max_retries})..."
  1352. # A 5xx Retry-After can reach the cap; buffering it would leave the user silent for minutes.
  1353. _announce_now = _retry_after is not None and _retry_after > LIVE_RETRY_WAIT_CAP_S
  1354. if _announce_now:
  1355. agent._emit_diagnostic_status(_status)
  1356. else:
  1357. agent._buffer_diagnostic_status(_status)
  1358. # The buffered line replays only if every retry fails; the live status line is transient
  1359. # (rewritten by the next frame, cleared on recovery), so it names the wait on every retry
  1360. # without adding transcript chatter. The reset window belongs here too: during the wait this
  1361. # line is where the user learns whether to sit it out or switch models.
  1362. if _reset:
  1363. _live_reason = f"{_wait_reason.lower()} — resets in {_reset},"
  1364. else:
  1365. _live_reason = "the free model is busy —" if _free_busy else "waiting on provider —"
  1366. agent._emit_diagnostic_wait(
  1367. f"⏳ {_live_reason} retrying in {wait_time:.0f}s (attempt {retry_count}/{max_retries})"
  1368. )
  1369. logger.warning(
  1370. "Retrying API call in %ss (attempt %s/%s) %s policy=%s error=%s",
  1371. wait_time, retry_count, max_retries, agent._client_log_context(),
  1372. _backoff_policy or "default", api_error,
  1373. )
  1374. return wait_time
  1375. def free_tier_cooldown_ends_turn(agent: Any, api_error: Exception, base_url: Any) -> bool:
  1376. """True when an attended session on the free model meets a cooldown longer than a person
  1377. sits through. The free tier's pause is service-wide: a 600 s retry met a second 429 while its
  1378. 1069 s window ran, so the caller ends the attempt cycle now (fallback, or the reset time and
  1379. the sign-in / model doors). Every other provider, and every unattended run (cron, messaging,
  1380. ``hermes chat -q``), waits out its ``Retry-After``: a paid bucket reopens on time, and
  1381. ``fallback.min_switch_reset_seconds`` relies on that wait."""
  1382. from agent.retry_utils import LIVE_RETRY_WAIT_CAP_S, provider_retry_after_seconds
  1383. from agent.session_source import is_attended
  1384. from hermes_cli.anon_auth import on_free_model
  1385. if not (on_free_model(agent, base_url) and is_attended(agent)):
  1386. return False
  1387. cooldown = provider_retry_after_seconds(api_error)
  1388. return cooldown is not None and cooldown > LIVE_RETRY_WAIT_CAP_S
  1389. def _codex_soft_failure_error(response: Any) -> dict[str, Any]:
  1390. """``response.error`` of a Codex ``failed``/``cancelled`` Response as ``{"code", "message"}``
  1391. (the SDK types it as ``ResponseError``; the raw-SSE assembler keeps the dict); ``{}`` when absent."""
  1392. error_obj = getattr(response, "error", None)
  1393. if not error_obj:
  1394. return {}
  1395. if isinstance(error_obj, dict):
  1396. fields = error_obj
  1397. elif hasattr(error_obj, "code") or hasattr(error_obj, "message"):
  1398. fields = {"code": getattr(error_obj, "code", None), "message": getattr(error_obj, "message", None)}
  1399. else:
  1400. fields = {"message": str(error_obj)}
  1401. return {k: v for k, v in fields.items() if isinstance(v, str) and v.strip()}
  1402. class _CodexSoftFailure(Exception):
  1403. """A Codex HTTP-200 ``status=failed`` Response reshaped so ``classify_api_error`` and
  1404. ``extract_api_error_context`` read ``response.error`` exactly like an SDK error body."""
  1405. def __init__(self, error: dict[str, Any]) -> None:
  1406. super().__init__(error.get("message") or "")
  1407. self.body = {"error": error}
  1408. def classify_codex_soft_failure(agent: Any, response: Any) -> tuple[Any, dict[str, Any]]:
  1409. """``(classified, error_context)`` for a Codex ``failed``/``cancelled`` Response, or
  1410. ``(None, {})`` when it is not one. The SDK never raises on these HTTP-200 soft failures,
  1411. so this is the only place their quota/billing/auth semantics reach the credential pool."""
  1412. if agent.api_mode != "codex_responses":
  1413. return None, {}
  1414. if str(getattr(response, "status", "") or "").strip().lower() not in {"failed", "cancelled"}:
  1415. return None, {}
  1416. exc = _CodexSoftFailure(_codex_soft_failure_error(response))
  1417. classified = classify_api_error(
  1418. exc, provider=getattr(agent, "provider", "") or "", model=getattr(agent, "model", "") or "",
  1419. base_url=str(getattr(agent, "base_url", "") or ""), api_key=getattr(agent, "api_key", None),
  1420. )
  1421. return classified, agent._extract_api_error_context(exc)
  1422. def validate_response_shape(agent: Any, response: Any) -> tuple[bool, list[str]]:
  1423. """Validate the raw provider response via the transport; ``(response_invalid,
  1424. error_details)``. A Codex ``failed``/``cancelled`` status (e.g. quota exhaustion) is
  1425. invalid so the fallback chain triggers; an empty Codex ``output`` with non-empty
  1426. ``output_text`` is deferred to normalization."""
  1427. if agent._get_transport().validate_response(response):
  1428. return False, []
  1429. if response is None:
  1430. return True, ["response is None"]
  1431. if agent.api_mode == "codex_responses":
  1432. _codex_resp_status = str(getattr(response, "status", "") or "").strip().lower()
  1433. if _codex_resp_status in {"failed", "cancelled"}:
  1434. _codex_error_msg = (
  1435. _codex_soft_failure_error(response).get("message")
  1436. or f"Responses API returned status '{_codex_resp_status}'"
  1437. )
  1438. logger.warning(
  1439. "Codex response status='%s' (error=%s). Routing to fallback. %s",
  1440. _codex_resp_status, _codex_error_msg, agent._client_log_context(),
  1441. )
  1442. return True, [f"response.status={_codex_resp_status}: {_codex_error_msg}"]
  1443. # Stream backfill may have failed but normalize can still recover from output_text.
  1444. _out_text = getattr(response, "output_text", None)
  1445. _out_text_stripped = _out_text.strip() if isinstance(_out_text, str) else ""
  1446. if _out_text_stripped:
  1447. logger.debug(
  1448. "Codex response.output is empty but output_text is present "
  1449. "(%d chars); deferring to normalization.",
  1450. len(_out_text_stripped),
  1451. )
  1452. return False, []
  1453. logger.warning(
  1454. "Codex response.output is empty after stream backfill "
  1455. "(status=%s, incomplete_details=%s, model=%s). %s",
  1456. getattr(response, "status", None), getattr(response, "incomplete_details", None),
  1457. getattr(response, "model", None),
  1458. f"api_mode={agent.api_mode} provider={agent.provider}",
  1459. )
  1460. return True, ["response.output is empty"]
  1461. if agent.api_mode == "anthropic_messages":
  1462. detail = "response.content invalid (not a non-empty list)"
  1463. elif agent.api_mode == "bedrock_converse":
  1464. detail = "Bedrock response invalid (no output or choices)"
  1465. elif not hasattr(response, 'choices'):
  1466. detail = "response has no 'choices' attribute"
  1467. elif response.choices is None:
  1468. detail = "response.choices is None"
  1469. else:
  1470. detail = "response.choices is empty"
  1471. return True, [detail]
  1472. def describe_invalid_response(agent: Any, response: Any, api_duration: float) -> tuple[str, str, str]:
  1473. """Diagnostics for an empty/malformed response: ``(error_msg, provider_name,
  1474. failure_hint)``. The hint is derived from the provider error code (524/504/429/
  1475. 5xx) and the response time, instead of always assuming rate limiting."""
  1476. error_msg = "Unknown"
  1477. provider_name = "Unknown"
  1478. _has_error = bool(response and hasattr(response, 'error') and response.error)
  1479. if _has_error:
  1480. # A typed ``ResponseError`` stringifies as its repr; show the provider's message.
  1481. error_msg = _codex_soft_failure_error(response).get("message") or str(response.error)
  1482. if hasattr(response.error, 'metadata') and response.error.metadata:
  1483. provider_name = response.error.metadata.get('provider_name', 'Unknown')
  1484. elif response and hasattr(response, 'message') and response.message:
  1485. error_msg = str(response.message)
  1486. # OpenRouter often returns the actual model used.
  1487. if provider_name == "Unknown" and response and hasattr(response, 'model') and response.model:
  1488. provider_name = f"model={response.model}"
  1489. if provider_name == "Unknown" and response:
  1490. resp_attrs = {k: str(v)[:100] for k, v in vars(response).items() if not k.startswith('_')}
  1491. if agent.verbose_logging:
  1492. logging.debug(f"Response attributes for invalid response: {resp_attrs}")
  1493. _resp_error_code = None
  1494. if _has_error:
  1495. _code_raw = getattr(response.error, 'code', None)
  1496. if _code_raw is None and isinstance(response.error, dict):
  1497. _code_raw = response.error.get('code')
  1498. if _code_raw is not None:
  1499. try:
  1500. _resp_error_code = int(_code_raw)
  1501. except (TypeError, ValueError):
  1502. pass
  1503. return error_msg, provider_name, _failure_hint_for(_resp_error_code, api_duration)
  1504. def _failure_hint_for(code: Optional[int], api_duration: float) -> str:
  1505. """Human-readable hint from the provider error code and response time."""
  1506. if code == 524:
  1507. return f"upstream provider timed out (Cloudflare 524, {api_duration:.0f}s)"
  1508. if code == 504:
  1509. return f"upstream gateway timeout (504, {api_duration:.0f}s)"
  1510. if code == 429:
  1511. return "rate limited by upstream provider (429)"
  1512. if code in {500, 502}:
  1513. return f"upstream server error ({code}, {api_duration:.0f}s)"
  1514. if code in {503, 529}:
  1515. return f"upstream provider overloaded ({code})"
  1516. if code is not None:
  1517. return f"upstream error (code {code}, {api_duration:.0f}s)"
  1518. if api_duration < 10:
  1519. return f"fast response ({api_duration:.1f}s) — likely rate limited"
  1520. if api_duration > 60:
  1521. return f"slow response ({api_duration:.0f}s) — likely upstream timeout"
  1522. return f"response time {api_duration:.1f}s"
  1523. @dataclass
  1524. class ClassifiedErrorVerdict:
  1525. """Outcome of ``route_classified_error``. ``action``: ``"return"`` (terminal result),
  1526. ``"break"`` (restart armed on ``_retry``), ``"continue"`` (re-enter the retry loop; Nous
  1527. guard re-check) or ``"fallthrough"`` (proceed to overflow / client-error / backoff
  1528. handling). The remaining fields are loop locals the router rebound or computed."""
  1529. action: str
  1530. result: Optional[dict[str, Any]]
  1531. status_code: Optional[int]
  1532. messages: list[dict[str, Any]]
  1533. active_system_prompt: Any
  1534. conversation_history: Any
  1535. retry_count: int
  1536. max_retries: int
  1537. compression_attempts: int
  1538. provider_overflow_recovery_pending: bool
  1539. is_rate_limited: bool
  1540. wrapped_output_cap_budget: Optional[int]
  1541. is_zai_coding_overload: bool
  1542. _OVERFLOW_REASONS = frozenset({
  1543. FailoverReason.long_context_tier, FailoverReason.payload_too_large, FailoverReason.context_overflow,
  1544. })
  1545. _RATE_LIMIT_REASONS = frozenset({
  1546. FailoverReason.rate_limit, FailoverReason.billing, FailoverReason.upstream_rate_limit,
  1547. })
  1548. _TRANSPORT_FAILURE_REASONS = frozenset({FailoverReason.timeout, FailoverReason.overloaded})
  1549. _LONG_CONTEXT_TIER_CAP = 200000
  1550. def _cap_long_context_tier(agent: Any) -> int:
  1551. """Cap the compressor's context window at the long-context tier limit; returns the
  1552. previous ``context_length``."""
  1553. compressor = agent.context_compressor
  1554. old_ctx = compressor.context_length
  1555. if old_ctx > _LONG_CONTEXT_TIER_CAP:
  1556. compressor.update_model(
  1557. model=agent.model, context_length=_LONG_CONTEXT_TIER_CAP, base_url=agent.base_url,
  1558. api_key=getattr(agent, "api_key", ""), provider=agent.provider, api_mode=agent.api_mode,
  1559. )
  1560. # Context probing flags exist only on the built-in compressor (plugin engines
  1561. # manage their own). Don't persist — a tier limit, not a model capability;
  1562. # 1M should return if extra usage is enabled.
  1563. if hasattr(compressor, "_context_probed"):
  1564. compressor._context_probed = True
  1565. compressor._context_probe_persistable = False
  1566. agent._buffer_vprint(
  1567. f"⚠️ Anthropic long-context tier "
  1568. f"requires extra usage — reducing context: "
  1569. f"{old_ctx:,} → {_LONG_CONTEXT_TIER_CAP:,} tokens"
  1570. )
  1571. return old_ctx
  1572. def _eager_fallback_status(classified: Any, is_upstream: bool, is_transport_failure: bool) -> str:
  1573. """Status line announcing an eager fallback switch."""
  1574. if is_upstream:
  1575. _upstream_name = (classified.error_context or {}).get("upstream_provider", "aggregator")
  1576. return f"⚠️ Upstream {_upstream_name} rate-limited — switching to fallback model..."
  1577. if classified.reason == FailoverReason.billing:
  1578. if classified.billing_unverified:
  1579. # Ambiguous body — don't assert billing.
  1580. return (
  1581. "⚠️ Provider reported usage/credit exhaustion "
  1582. "(unverified — may be a content-filter rejection) "
  1583. "— switching to fallback provider..."
  1584. )
  1585. return "⚠️ Billing or credits exhausted — switching to fallback provider..."
  1586. if is_transport_failure:
  1587. return "⚠️ Provider unreachable — switching to fallback provider..."
  1588. return "⚠️ Rate limited — switching to fallback provider..."
  1589. def activate_codex_app_server_fallback(agent: Any, result: dict[str, Any]) -> bool:
  1590. """The codex app-server runtime reports a failed turn as ``result["error"]`` text instead of
  1591. raising, so the generic classify -> ``fallback_providers`` chain never saw it (#71633).
  1592. Classify that text; on a billing / rate-limit verdict activate the configured fallback and
  1593. return True so the caller re-runs the same user turn on the generic loop."""
  1594. error = result.get("error")
  1595. if not error or result.get("interrupted") or not agent._has_pending_fallback():
  1596. return False
  1597. classified = classify_api_error(
  1598. RuntimeError(str(error)), provider=getattr(agent, "provider", "") or "", model=getattr(agent, "model", "") or "",
  1599. )
  1600. if classified.reason not in _RATE_LIMIT_REASONS:
  1601. return False
  1602. agent._buffer_diagnostic_status(
  1603. _eager_fallback_status(classified, classified.reason == FailoverReason.upstream_rate_limit, False))
  1604. return bool(agent._try_activate_fallback(reason=classified.reason))
  1605. def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: Any, classified: Any = None) -> bool:
  1606. """Record a genuine account-level Nous 429 to the cross-session breaker; upstream
  1607. capacity 429s (no exhausted bucket in headers or last-known state) are left alone.
  1608. *error_context* is the turn's (``extract_api_error_context``); *classified* brings the
  1609. classifier's own context, where a welcome-tier ``rate_limited`` refusal and its ``reset_at``
  1610. live. A long welcome reset is an exhausted allowance whatever the headers say, and the one
  1611. place the user is told that signing in lifts it."""
  1612. _genuine = False
  1613. try:
  1614. from agent.nous_rate_guard import (
  1615. is_genuine_nous_rate_limit, is_long_welcome_rate_limit, record_nous_rate_limit)
  1616. _err_resp = getattr(api_error, "response", None)
  1617. _err_hdrs = getattr(_err_resp, "headers", None) if _err_resp else None
  1618. from hermes_cli.anon_auth import is_anonymous_agent
  1619. anonymous = is_anonymous_agent(agent)
  1620. _classified_ctx = getattr(classified, "error_context", None) or {}
  1621. # Only an anonymous request's fairshare body is an allowance verdict; named
  1622. # requests keep the exhausted-bucket rule, whatever their host or body says.
  1623. _genuine = (
  1624. (anonymous and is_long_welcome_rate_limit(_classified_ctx))
  1625. or is_genuine_nous_rate_limit(headers=_err_hdrs, last_known_state=agent._rate_limit_state))
  1626. if _genuine:
  1627. _merged = {**(error_context if isinstance(error_context, dict) else {}), **_classified_ctx}
  1628. record_nous_rate_limit(headers=_err_hdrs, error_context=_merged, anonymous=anonymous)
  1629. else:
  1630. logger.info(
  1631. "Nous 429 looks like upstream capacity "
  1632. "(no exhausted bucket in headers or "
  1633. "last-known state) -- not tripping "
  1634. "cross-session breaker."
  1635. )
  1636. except Exception:
  1637. pass
  1638. return _genuine
  1639. def route_classified_error(
  1640. agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState, *, error_msg: str,
  1641. error_context: Any, recovered_with_pool: bool, base_url: Any, model: Any,
  1642. messages: list[dict[str, Any]], api_messages: Any, system_message: Any,
  1643. active_system_prompt: Any, conversation_history: Any, retry_count: int, max_retries: int,
  1644. compression_attempts: int, max_compression_attempts: int, api_call_count: int,
  1645. effective_task_id: Any,
  1646. ) -> ClassifiedErrorVerdict:
  1647. """Ordered (load-bearing) recovery steps between classification and overflow handling:
  1648. compaction-disabled overflow → terminal error (output-cap errors exempt); Anthropic
  1649. long-context tier 429 → cap at 200k and compress; eager fallback for rate-limit/billing
  1650. (immediately) and transport failures (after 1 retry) unless credential-pool rotation may
  1651. still recover (upstream-aggregator 429s always fall back); persistent 401/403 → fallback
  1652. chain once; genuine Nous 429 → cross-session breaker + re-enter the loop exactly once."""
  1653. from agent.conversation_compression import conversation_history_after_compression
  1654. from agent.conversation_loop import _arm_fallback_restart, _ra
  1655. from agent.model_metadata import estimate_request_tokens_rough
  1656. _provider_overflow_recovery_pending = False
  1657. is_rate_limited = False
  1658. _wrapped_output_cap_budget = None
  1659. _is_zai_coding_overload = False
  1660. status_code = getattr(api_error, "status_code", None)
  1661. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ClassifiedErrorVerdict:
  1662. return ClassifiedErrorVerdict(
  1663. action=action, result=result, status_code=status_code, messages=messages,
  1664. active_system_prompt=active_system_prompt, conversation_history=conversation_history,
  1665. retry_count=retry_count, max_retries=max_retries,
  1666. compression_attempts=compression_attempts,
  1667. provider_overflow_recovery_pending=_provider_overflow_recovery_pending,
  1668. is_rate_limited=is_rate_limited, wrapped_output_cap_budget=_wrapped_output_cap_budget,
  1669. is_zai_coding_overload=_is_zai_coding_overload,
  1670. )
  1671. def _fallback_break() -> ClassifiedErrorVerdict:
  1672. nonlocal active_system_prompt, retry_count, compression_attempts
  1673. active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
  1674. retry_count = 0
  1675. compression_attempts = 0
  1676. return _verdict("break")
  1677. # ``compression.enabled: false`` forbids every automatic trigger, incl. these
  1678. # overflow recovery paths; error out. Output-cap errors exempt.
  1679. _is_output_cap_error = (
  1680. is_output_cap_error(error_msg) or parse_available_output_tokens_from_error(error_msg) is not None
  1681. )
  1682. if (
  1683. classified.reason in _OVERFLOW_REASONS
  1684. and not getattr(agent, "compression_enabled", True)
  1685. and not _is_output_cap_error
  1686. ):
  1687. agent._flush_status_buffer()
  1688. _vlines(
  1689. agent,
  1690. "❌ The conversation is too long for the model and automatic shrinking is off (compression.enabled: false).",
  1691. " 💡 Run /compress to shrink it now, /new to start fresh, "
  1692. "pick a model with a bigger context window, or remove attachments.",
  1693. )
  1694. logger.error(
  1695. f"{agent.log_prefix}Context overflow ({classified.reason.value}) with "
  1696. f"auto-compaction disabled — not compressing."
  1697. )
  1698. agent._persist_session(messages, conversation_history)
  1699. _final_response = site_copy("compression_disabled", model=agent.model)
  1700. return _verdict("return", stamp_failure({
  1701. "final_response": _final_response, "messages": messages, "completed": False,
  1702. "api_calls": api_call_count, "error": _final_response, "partial": True, "failed": True,
  1703. "compaction_disabled": True,
  1704. }, "context_overflow", False))
  1705. # Anthropic 429 "Extra usage is required for long context requests" is a
  1706. # subscription-tier limit, not transient: cap at 200k and compress.
  1707. if classified.reason == FailoverReason.long_context_tier:
  1708. old_ctx = _cap_long_context_tier(agent)
  1709. compression_attempts += 1
  1710. if compression_attempts <= max_compression_attempts:
  1711. original_len = len(messages)
  1712. # Overhead-aware request size so recovery arms on the true request
  1713. # (msgs + tools + system), not the tool-blind message count.
  1714. messages, active_system_prompt = agent._compress_context(
  1715. # Route the overhead-aware _real_tokens (computed above) into compression, not the bare
  1716. # last_prompt_tokens — which is 0 in the no-usage fallback, hiding the true request size
  1717. # from the engine's overflow guard (upstream PR #77169 review).
  1718. messages, system_message,
  1719. approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None),
  1720. task_id=effective_task_id, trigger="overflow", overflow_reason=classified.reason.value,
  1721. )
  1722. conversation_history = conversation_history_after_compression(agent, messages, conversation_history)
  1723. if len(messages) < original_len or old_ctx > _LONG_CONTEXT_TIER_CAP:
  1724. agent._buffer_diagnostic_status(
  1725. COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE.format(
  1726. new_ctx=_LONG_CONTEXT_TIER_CAP, old_ctx=old_ctx
  1727. )
  1728. )
  1729. time.sleep(2)
  1730. # Provider proved the request doesn't fit the reduced window; row count
  1731. # isn't proof the rebuilt one does. Recheck before the next call.
  1732. _provider_overflow_recovery_pending = True
  1733. _retry.restart_with_compressed_messages = True
  1734. return _verdict("break")
  1735. # Compression exhausted or didn't help: fall through to normal error handling.
  1736. # Eager fallback: rate-limit/billing switch immediately (primary won't recover in
  1737. # the retry window); transport errors get 1 retry first.
  1738. is_rate_limited = classified.reason in _RATE_LIMIT_REASONS
  1739. # Some relays wrap upstream output-cap 400s as 429 (rate_limit). Only the max_tokens
  1740. # clamp fixes it. Parsed once; gates the eager-fallback exemption and overflow entry.
  1741. # Relay-wrapped output-cap errors: some gateways wrap an upstream "[400]: max_tokens (...) exceeds
  1742. # model's maximum output tokens (...)" as HTTP 429, which classifies as rate_limit. The failure is a
  1743. # deterministic request-shape problem — falling back to another provider (or burning generic retries)
  1744. # can't fix it, but the output-cap clamp below can, in one retry (#72281). Parse once here; the result
  1745. # gates both the eager-fallback exemption and the widened is_context_length_error entry, and is reused
  1746. # as available_out inside the handler.
  1747. _wrapped_output_cap_budget = (
  1748. parse_available_output_tokens_from_error(error_msg)
  1749. if classified.reason == FailoverReason.rate_limit else None
  1750. )
  1751. _is_transport_failure = classified.reason in _TRANSPORT_FAILURE_REASONS
  1752. # Z.AI overload 429s classify `overloaded`, which `is_rate_limited` excludes. Detect
  1753. # directly so the long backoff runs, and raise the ceiling to reach it.
  1754. _is_zai_coding_overload = is_zai_coding_overload_error(base_url=str(base_url), model=model, error=api_error)
  1755. if _is_zai_coding_overload:
  1756. max_retries = max(max_retries, zai_coding_overload_retry_ceiling())
  1757. _should_fallback = (
  1758. (is_rate_limited and _wrapped_output_cap_budget is None)
  1759. or (_is_transport_failure and retry_count >= 2)
  1760. ) and not (getattr(classified, "error_context", None) or {}).get("usage_window") # never switch models on a plan limit
  1761. if _should_fallback and agent._fallback_index < len(agent._fallback_chain):
  1762. # No eager fallback while credential pool rotation may recover. Exception: an
  1763. # upstream-aggregator 429 — the pool can't help, always fall back.
  1764. # Fixes #11314.
  1765. _is_upstream = classified.reason == FailoverReason.upstream_rate_limit
  1766. pool_may_recover = (
  1767. False if _is_upstream else _ra()._pool_may_recover_from_rate_limit(agent._credential_pool)
  1768. )
  1769. if not pool_may_recover:
  1770. agent._buffer_diagnostic_status(_eager_fallback_status(classified, _is_upstream, _is_transport_failure))
  1771. reset_at = error_context.get("reset_at") if isinstance(error_context, dict) else None
  1772. if agent._try_activate_fallback(reason=classified.reason, reset_at=reset_at):
  1773. return _fallback_break()
  1774. # A 401/403 surviving credential refresh means a broken credential or endpoint:
  1775. # escalate to the fallback chain once; False -> terminal handling.
  1776. if (
  1777. classified.is_auth
  1778. and not _retry.auth_failover_attempted
  1779. and agent._fallback_index < len(agent._fallback_chain)
  1780. ):
  1781. _retry.auth_failover_attempted = True
  1782. agent._buffer_diagnostic_status(
  1783. "🔐 Authentication failed and could not be refreshed — "
  1784. "switching to fallback provider..."
  1785. )
  1786. if agent._try_activate_fallback(reason=classified.reason):
  1787. return _fallback_break()
  1788. # Nous Portal: a genuine account-level 429 is recorded to a shared file so ALL
  1789. # sessions back off; is_genuine_nous_rate_limit excludes upstream 429s.
  1790. if (
  1791. is_rate_limited
  1792. and agent.provider == "nous"
  1793. and classified.reason == FailoverReason.rate_limit
  1794. and not recovered_with_pool
  1795. and _is_genuine_nous_rate_limit(agent, api_error, error_context, classified)
  1796. ):
  1797. # Re-enter the loop exactly once so the top-of-loop Nous guard runs
  1798. # (retry_count = max_retries would skip it entirely).
  1799. retry_count = max(0, max_retries - 1)
  1800. return _verdict("continue")
  1801. # Upstream capacity 429: normal retry logic will typically succeed.
  1802. return _verdict("fallthrough")