| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438 |
- """API-call exception handler for the conversation turn's retry loop: pre-/post-classification
- recovery, interpreter-shutdown abandon, classified-error routing, overflow recovery, the
- non-retryable client-error exit, max-retries exhaustion (primary transport recovery →
- fallback → terminal result) and the interruptible backoff. Nothing here imports
- ``agent.conversation_loop`` at module level (cycle) — loop-internal helpers resolve lazily so
- ``patch("agent.conversation_loop.X")`` sites keep intercepting.
- """
- from __future__ import annotations
- from dataclasses import dataclass
- import json
- import logging
- import ssl
- import time
- from typing import Any, Dict, Optional
- from agent.api_error_summary import is_provider_stream_parse_error
- from agent.error_classifier import RETRYABLE_CLIENT_REASONS, FailoverReason, classify_api_error
- from agent.turn_overflow import recover_from_overflow
- from agent.turn_recovery import (
- _NONRETRYABLE_LABELS, abort_turn_on_interrupt, compute_error_backoff, free_tier_cooldown_ends_turn,
- interruptible_backoff_sleep, log_api_error_attempt,
- max_retries_exhausted_result, nonretryable_client_error_result, recover_after_classification,
- recover_before_classification, route_classified_error, settle_delivered_partial,
- )
- logger = logging.getLogger("agent.conversation_loop")
- @dataclass
- class ApiErrorVerdict:
- """``action``: ``"continue"`` (retry the API call), ``"break"`` (leave the retry loop:
- fallback armed / redirect pending) or ``"return"`` (``result`` is the turn's result dict);
- ``"fallthrough"`` never happens — the handler always ends in an exit. The other fields
- are the retry-loop locals the handler rebinds; ``_provider_overflow_recovery_pending`` is
- merge-only (caller sets True when set)."""
- action: str
- thinking_spinner: Any
- messages: Any
- active_system_prompt: Any
- conversation_history: Any
- approx_tokens: Any
- retry_count: Any
- max_retries: Any
- compression_attempts: Any
- _provider_overflow_recovery_pending: Any
- result: Optional[dict[str, Any]] = None
- def handle_api_error(
- agent: Any, *, api_error: Any, _retry: Any, thinking_spinner: Any, messages: Any,
- api_messages: Any, api_kwargs: Any, system_message: Any, active_system_prompt: Any,
- conversation_history: Any, approx_tokens: Any, retry_count: Any, max_retries: Any,
- compression_attempts: Any, max_compression_attempts: Any, api_call_count: Any,
- api_request_id: Any, api_start_time: Any, effective_task_id: Any, turn_id: Any,
- current_turn_user_idx: Any = None,
- ) -> ApiErrorVerdict:
- """Recover from ``api_error`` in the original order. Every fallback activation must leave
- the retry loop with ``restart_with_rebuilt_messages`` armed (``"break"``) so the pre-API
- preflight re-runs against the fallback's context window (#84733)."""
- _provider_overflow_recovery_pending = False
- def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ApiErrorVerdict:
- return ApiErrorVerdict(
- action=action, thinking_spinner=thinking_spinner, messages=messages,
- active_system_prompt=active_system_prompt, conversation_history=conversation_history,
- approx_tokens=approx_tokens, retry_count=retry_count, max_retries=max_retries,
- compression_attempts=compression_attempts,
- _provider_overflow_recovery_pending=_provider_overflow_recovery_pending, result=result,
- )
- # Stop spinner silently — retry status is buffered and only flushed when every
- # retry+fallback is exhausted.
- if thinking_spinner:
- thinking_spinner.stop("")
- thinking_spinner = None
- if agent.thinking_callback:
- agent.thinking_callback("")
- _recovered, active_system_prompt = recover_before_classification(
- agent, api_error, messages=messages, api_messages=api_messages, api_kwargs=api_kwargs,
- active_system_prompt=active_system_prompt,
- )
- if _recovered:
- return _verdict("continue")
- status_code = getattr(api_error, "status_code", None)
- error_context = agent._extract_api_error_context(api_error)
- # A Nous 402/403 carries the same x-nous-credits-* headers as a success, and a usage-window plan's
- # "ran out for this model" / "payment paused" state is only visible on the blocking response itself
- # (the stream path captures headers after the stream opens, which a 402 never does). Fail-open.
- # Usage-window responses only: every other account keeps main's behaviour (no capture here).
- if status_code in (402, 403) and getattr(agent, "provider", "") == "nous":
- try:
- from agent.credits_tracker import response_is_usage_window
- response = getattr(api_error, "response", None)
- if response_is_usage_window(response):
- agent._capture_credits(response, refused=True)
- except Exception:
- logger.debug("credits ▸ header capture on %s failed (fail-open)", status_code, exc_info=True)
- # Process is exiting mid-flight: retries/rotation/fallbacks are futile and the
- # retry trace spams the shell. One log line.
- from tools.interpreter_shutdown import interpreter_shutting_down
- if interpreter_shutting_down(api_error):
- logger.warning(
- "%sInterpreter is shutting down — abandoning turn "
- "during API call #%d (%s)",
- agent.log_prefix, api_call_count, api_error,
- )
- _shutdown_summary = "Turn abandoned: the process was shutting down before the model call could complete."
- return _verdict("return", {
- "final_response": _shutdown_summary, "messages": messages, "api_calls": api_call_count,
- "completed": False, "failed": True, "error": _shutdown_summary,
- "failure_reason": "interpreter_shutdown", "failure_retryable": False,
- })
- _compressor = getattr(agent, "context_compressor", None)
- _ctx_len = getattr(_compressor, "context_length", 200000) if _compressor else 200000
- classified = classify_api_error(
- api_error, provider=getattr(agent, "provider", "") or "",
- model=getattr(agent, "model", "") or "", approx_tokens=approx_tokens,
- context_length=_ctx_len, num_messages=len(api_messages) if api_messages else 0,
- base_url=str(getattr(agent, "base_url", "") or ""),
- api_key=getattr(agent, "api_key", None),
- )
- logger.debug(
- "Error classified: reason=%s status=%s retryable=%s compress=%s rotate=%s fallback=%s",
- classified.reason.value, classified.status_code,
- classified.retryable, classified.should_compress,
- classified.should_rotate_credential, classified.should_fallback,
- )
- agent._invoke_api_request_error_hook(
- task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
- api_call_count=api_call_count, api_start_time=api_start_time, api_kwargs=api_kwargs,
- error_type=type(api_error).__name__, error_message=str(api_error), status_code=status_code,
- retry_count=retry_count, max_retries=max_retries, retryable=classified.retryable,
- reason=classified.reason.value,
- )
- _recovered, recovered_with_pool = recover_after_classification(
- agent, api_error, classified, _retry, status_code=status_code, error_context=error_context,
- messages=messages, api_messages=api_messages,
- )
- if _recovered:
- return _verdict("continue")
- retry_count += 1
- elapsed_time = time.time() - api_start_time
- # Liveness/watchdog label only (never shown in chat), so the classifier's
- # "not retryable" verdict is named on the logged attempt line below instead.
- agent._touch_activity(f"API error recovery (attempt {retry_count}/{max_retries})")
- error_type, error_msg, _provider, _base, _model = log_api_error_attempt(
- agent, api_error, retry_count=retry_count, max_retries=max_retries, status_code=status_code,
- elapsed_time=elapsed_time, api_messages=api_messages, approx_tokens=approx_tokens,
- retryable=bool(classified.retryable),
- )
- if agent._interrupt_requested:
- # Preserve a pending redirect: the user is steering, not stopping — rebuild the
- # turn from the correction instead of aborting.
- if agent.clear_interrupt(preserve_redirect=True):
- _retry.restart_with_redirected_messages = True
- return _verdict("break")
- return _verdict("return", abort_turn_on_interrupt(
- agent, messages, conversation_history, api_call_count,
- abort_message="Interrupt detected during error handling, aborting retries.",
- interrupt_text=f"Operation interrupted: handling API error ({error_type}: {agent._clean_error_message(str(api_error))}).",
- ))
- _ce = route_classified_error(
- agent, api_error, classified, _retry, error_msg=error_msg, error_context=error_context,
- recovered_with_pool=recovered_with_pool, base_url=_base, model=_model, messages=messages,
- api_messages=api_messages, system_message=system_message,
- active_system_prompt=active_system_prompt, conversation_history=conversation_history,
- retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts,
- max_compression_attempts=max_compression_attempts, api_call_count=api_call_count,
- effective_task_id=effective_task_id,
- )
- status_code = _ce.status_code
- messages = _ce.messages
- active_system_prompt = _ce.active_system_prompt
- conversation_history = _ce.conversation_history
- retry_count = _ce.retry_count
- max_retries = _ce.max_retries
- compression_attempts = _ce.compression_attempts
- is_rate_limited = _ce.is_rate_limited
- _wrapped_output_cap_budget = _ce.wrapped_output_cap_budget
- _is_zai_coding_overload = _ce.is_zai_coding_overload
- if _ce.provider_overflow_recovery_pending:
- _provider_overflow_recovery_pending = True
- if _ce.action != "fallthrough":
- return _verdict(_ce.action, _ce.result)
- _ov = recover_from_overflow(
- agent, api_error, classified, _retry, status_code=status_code, error_msg=error_msg,
- wrapped_output_cap_budget=_wrapped_output_cap_budget, messages=messages,
- api_messages=api_messages, system_message=system_message,
- active_system_prompt=active_system_prompt, conversation_history=conversation_history,
- approx_tokens=approx_tokens, compression_attempts=compression_attempts,
- max_compression_attempts=max_compression_attempts, api_call_count=api_call_count,
- effective_task_id=effective_task_id,
- )
- messages = _ov.messages
- active_system_prompt = _ov.active_system_prompt
- conversation_history = _ov.conversation_history
- approx_tokens = _ov.approx_tokens
- compression_attempts = _ov.compression_attempts
- is_context_length_error = _ov.is_context_length_error
- if _ov.provider_overflow_recovery_pending:
- _provider_overflow_recovery_pending = True
- if _ov.action != "fallthrough":
- return _verdict(_ov.action, _ov.result)
- _ue = settle_unrecovered_error(
- agent, api_error=api_error, classified=classified, _retry=_retry, status_code=status_code,
- error_msg=error_msg, error_context=error_context,
- is_context_length_error=is_context_length_error,
- is_rate_limited=is_rate_limited, _is_zai_coding_overload=_is_zai_coding_overload,
- _provider=_provider, _base=_base, _model=_model, messages=messages,
- api_messages=api_messages, api_kwargs=api_kwargs, active_system_prompt=active_system_prompt,
- conversation_history=conversation_history, approx_tokens=approx_tokens,
- retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts,
- api_call_count=api_call_count, current_turn_user_idx=current_turn_user_idx,
- )
- active_system_prompt = _ue.active_system_prompt
- retry_count = _ue.retry_count
- compression_attempts = _ue.compression_attempts
- return _verdict(_ue.action, _ue.result)
- def _is_local_validation_error(api_error: Any) -> bool:
- """ValueError/TypeError are local bugs, except: UnicodeEncodeError (surrogate recovery
- path), json.JSONDecodeError / provider stream-parse ValueErrors (transient provider/network failure, must retry),
- ssl.SSLError (inherits OSError *and* ValueError — a TLS failure is not a local bug)
- and "NoneType is not iterable" TypeErrors (upstream shape mismatches, e.g. Codex
- response.completed.output=null — retryable so the fallback path runs)."""
- if not isinstance(api_error, (ValueError, TypeError)):
- return False
- if isinstance(api_error, (UnicodeEncodeError, json.JSONDecodeError, ssl.SSLError)):
- return False
- _text = str(api_error).lower()
- if is_provider_stream_parse_error(api_error):
- return False # corrupted provider SSE chunk (jiter), transient — not a local bug (#65147)
- return not (isinstance(api_error, TypeError) and "nonetype" in _text and "not iterable" in _text)
- @dataclass
- class UnrecoveredErrorVerdict:
- """``action``: ``"continue"`` (retry), ``"break"`` (fallback armed / redirect pending) or
- ``"return"`` (``result`` is the terminal result dict). Rebinds ``active_system_prompt``,
- ``retry_count`` and ``compression_attempts``."""
- action: str
- active_system_prompt: Any
- retry_count: Any
- compression_attempts: Any
- result: Optional[dict[str, Any]] = None
- def settle_unrecovered_error(
- agent: Any, *, api_error: Any, classified: Any, _retry: Any, status_code: Any, error_msg: Any,
- is_context_length_error: Any, is_rate_limited: Any, _is_zai_coding_overload: Any,
- _provider: Any, _base: Any, _model: Any, messages: Any, api_messages: Any, api_kwargs: Any,
- active_system_prompt: Any, conversation_history: Any, approx_tokens: Any, retry_count: Any,
- max_retries: Any, compression_attempts: Any, api_call_count: Any, error_context: Any = None,
- current_turn_user_idx: Any = None,
- ) -> UnrecoveredErrorVerdict:
- """Decide the fate of an API error that every recovery chain declined: local validation /
- non-retryable client errors (provider stale-credential self-heal first, then fallback, then a
- terminal result), max-retries exhaustion (primary transport recovery -> fallback -> terminal
- result), else the interruptible error backoff. ``FailoverReason.billing`` (402) is deliberately
- treated as non-retryable (#31273)."""
- from agent.conversation_loop import _arm_fallback_restart, _credential_refresh_profile
- from agent.provider_route_hooks import ask_on_error_path
- def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> UnrecoveredErrorVerdict:
- return UnrecoveredErrorVerdict(
- action=action, active_system_prompt=active_system_prompt, retry_count=retry_count,
- compression_attempts=compression_attempts, result=result,
- )
- # ``FailoverReason.billing`` (402) is deliberately NOT excluded: pool rotation and
- # eager fallback already gave up, so retrying only burns paid requests on a depleted
- # balance. Mirrors 401/403.
- is_local_validation_error = _is_local_validation_error(api_error)
- # ``recover_after_classification`` sets ``image_shrink_retry_attempted`` BEFORE it runs the shrink,
- # so an image-size rejection reaching this point with the flag set had nothing left to shrink (the
- # excess is text on a host whose cap is payload-scoped, or an unshrinkable image). Re-sending the
- # byte-identical body ``max_retries`` times changes nothing: treat it as a client error and try
- # the fallback chain now, as the format_error verdict these 400s carried before did (#112473).
- shrink_spent = classified.reason == FailoverReason.image_too_large and bool(
- getattr(_retry, "image_shrink_retry_attempted", False)
- )
- # Same shape for the reasoning-disable rung: the retry already went out without the disable,
- # so a second reasoning-field rejection means the route refuses the configured reasoning
- # controls themselves — nothing left to drop, so take the fallback chain now instead of
- # replaying the identical request ``max_retries`` times (#114460).
- reasoning_spent = classified.reason == FailoverReason.reasoning_mandatory and bool(
- getattr(_retry, "reasoning_mandatory_retry_attempted", False)
- )
- is_client_error = (
- is_local_validation_error
- or shrink_spent
- or reasoning_spent
- or (
- not classified.retryable
- and not classified.should_compress
- and classified.reason not in RETRYABLE_CLIENT_REASONS
- )
- ) and not is_context_length_error
- if is_client_error:
- # A Codex ChatGPT-account entitlement 400 names the model: with nothing to rotate the
- # slug is dead for this account, so record it before the fallback walk runs (#106475).
- from agent.fallback_cooldown import _mark_entitlement_rejected_model
- _mark_entitlement_rejected_model(agent, api_error)
- # Provider self-heal BEFORE fallback: some providers surface a stale credential as a
- # non-auth error (the profile's ``is_stale_credential_error``), not a 401. Fresh
- # credential + client rebuild, one retry, SAME provider.
- if (
- not _retry.provider_stale_refresh_attempted
- and (profile := _credential_refresh_profile(agent)) is not None
- and ask_on_error_path(
- profile, "is_stale_credential_error", False,
- status_code=status_code, message=str(getattr(api_error, "message", "") or api_error),
- )
- ):
- _retry.provider_stale_refresh_attempted = True
- if agent._try_refresh_provider_runtime_credentials(reason="stale"):
- agent._buffer_vprint("🔐 " + (ask_on_error_path(profile, "credential_refresh_notice", "", reason="stale") or (
- f"{profile.display_name or profile.name} credential renewed after a "
- "stale-credential error. Retrying request...")))
- retry_count = 0
- return _verdict("continue")
- # ``should_fallback=False`` marks a deterministic failure no other provider can fix (the
- # model's own malformed tool-call JSON, #12770; MoA preset/adapter faults, #55933): skip
- # the cascade. An UNCLASSIFIED local ValueError/TypeError keeps its historical fallback;
- # a recognised verdict that opts out wins even when the exception is a ValueError subclass.
- _unclassified_local = is_local_validation_error and classified.reason == FailoverReason.unknown
- if classified.should_fallback or _unclassified_local or shrink_spent or reasoning_spent:
- # Announce the fallback only when a chain exists, else "trying fallback..." lies
- # before a silent abort.
- if agent._has_pending_fallback():
- _label = _NONRETRYABLE_LABELS.get(classified.reason, f"Non-retryable error (HTTP {status_code})")
- agent._buffer_diagnostic_status(f"⚠️ {_label} — trying fallback...")
- reset_at = error_context.get("reset_at") if isinstance(error_context, dict) else None
- if agent._try_activate_fallback(reason=classified.reason, reset_at=reset_at):
- # Direct ``return _verdict("break")`` is load-bearing: the restart handler
- # re-runs the pre-API preflight against the fallback's context window.
- active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
- retry_count = compression_attempts = 0
- return _verdict("break")
- # Terminal from here: collapse the continuation trail once, before the persist.
- _delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
- return _verdict("return", nonretryable_client_error_result(
- agent, api_error, classified, status_code=status_code, api_kwargs=api_kwargs,
- api_messages=api_messages, messages=messages, conversation_history=conversation_history,
- api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
- base_url=_base, model=_model, delivered=_delivered,
- ))
- # An attended session on the free model does not sit through a long cooldown: end the attempt
- # cycle now (fallback, else the reset time and the ways forward); the copy counts attempts made.
- attempts_made = min(retry_count, max_retries)
- if is_rate_limited and retry_count < max_retries and free_tier_cooldown_ends_turn(agent, api_error, _base):
- logger.info("%sFree-tier cooldown outlasts the attended wait — ending retries after attempt %s",
- agent.log_prefix, retry_count)
- retry_count = max_retries
- if retry_count >= max_retries:
- # Before fallback, rebuild the primary client once per API call block for
- # transient transport errors (stale pool, TCP reset).
- if not _retry.primary_recovery_attempted and agent._try_recover_primary_transport(
- api_error, retry_count=retry_count, max_retries=max_retries,
- ):
- _retry.primary_recovery_attempted = True
- retry_count = 0
- # Fresh attempt cycle: re-open fallback state so a follow-on 429 can still
- # activate fallback_providers.
- _retry.has_retried_429 = False
- agent._fallback_index = 0
- agent._fallback_activated = False
- return _verdict("continue")
- if agent._has_pending_fallback():
- agent._buffer_diagnostic_status(f"⚠️ Max retries ({max_retries}) exhausted — trying fallback...")
- reset_at = error_context.get("reset_at") if isinstance(error_context, dict) else None
- if agent._try_activate_fallback(reason=classified.reason, reset_at=reset_at):
- # Direct ``return _verdict("break")`` is load-bearing: the restart handler
- # re-runs the pre-API preflight against the fallback's context window.
- active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
- retry_count = compression_attempts = 0
- return _verdict("break")
- # Fallback first (above); only with nothing left to move to does the bounded auto-recovery
- # ladder park the turn on a transient outage instead of ending it (#85426, #107307).
- from agent.turn_recovery_autorecover import auto_recover_after_exhaustion
- _ladder = auto_recover_after_exhaustion(
- agent, api_error, classified, _retry, messages=messages,
- conversation_history=conversation_history, api_call_count=api_call_count,
- )
- if _ladder is not None:
- if _ladder["action"] == "continue":
- retry_count = 0
- return _verdict(_ladder["action"], _ladder.get("result"))
- _delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
- return _verdict("return", max_retries_exhausted_result(
- agent, api_error, classified, attempts=attempts_made, is_rate_limited=is_rate_limited,
- error_msg=error_msg, api_kwargs=api_kwargs, api_messages=api_messages,
- messages=messages, conversation_history=conversation_history,
- api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
- base_url=_base, model=_model, delivered=_delivered,
- ))
- wait_time = compute_error_backoff(
- agent, api_error, retry_count=retry_count, max_retries=max_retries,
- is_rate_limited=is_rate_limited, is_zai_coding_overload=_is_zai_coding_overload,
- base_url=_base, model=_model,
- )
- # Same preserve-redirect rule as the invalid-response wait: a steering correction
- # must survive backoff, not die as "Operation interrupted".
- _interrupted = interruptible_backoff_sleep(
- agent, wait_time, _retry, messages=messages, conversation_history=conversation_history,
- api_call_count=api_call_count,
- abort_message="Interrupt detected during retry wait, aborting.",
- interrupt_text=f"Operation interrupted: retrying API call after error (retry {retry_count}/{max_retries}).",
- activity_label=f"error retry backoff ({retry_count}/{max_retries})",
- )
- if _interrupted is not None:
- return _verdict("return", _interrupted)
- if _retry.restart_with_redirected_messages:
- # Leave the retry loop — the caller rebuilds this iteration from the correction
- # instead of re-firing the stale request.
- return _verdict("break")
- return _verdict("fallthrough")
|