turn_recovery_autorecover.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116
  1. """Bounded post-exhaustion auto-recovery ladder for a pre-delivery turn (#85426, #107307).
  2. Runs from ``settle_unrecovered_error`` once ``max_retries`` is spent AND the fallback chain has
  3. nothing left to move to (fallback stays first). While the provider is only *temporarily* away
  4. (5xx, overloaded/529, connect/read timeouts) and no answer text has reached the user yet, the
  5. turn parks with a visible countdown instead of ending in "API failed after N retries", then
  6. re-enters the ordinary retry loop. Cycles: ``agent.auto_recovery_cycles`` (default 5) with a
  7. jittered 15/30/60/60/60 s schedule; a provider ``Retry-After`` wins up to 120 s. Non-retryable
  8. classes (auth, format, content policy, billing, entitlement) never enter, and an interrupt
  9. cancels the wait cleanly. Overload-class errors use this same schedule — there is no separate
  10. overload backoff path.
  11. """
  12. from __future__ import annotations
  13. import logging
  14. from typing import Any, Dict, Optional
  15. from agent.error_classifier import FailoverReason
  16. logger = logging.getLogger("agent.conversation_loop")
  17. # Transient transport verdicts: the provider is expected back. Everything else (auth, format,
  18. # content policy, billing, entitlement, overflow, model-not-found ...) is deterministic for this
  19. # request and stays out.
  20. _LADDER_REASONS = frozenset({FailoverReason.overloaded, FailoverReason.server_error, FailoverReason.timeout})
  21. _LADDER_BASE_DELAY_S = 15.0
  22. _LADDER_CAP_S = 60.0
  23. # A provider that names its own cooldown knows better than the schedule, within reason.
  24. _RETRY_AFTER_CAP_S = 120.0
  25. # How the user stops the wait on this surface; the ladder text must say so plainly.
  26. _STOP_HINTS = {
  27. "cli": "press Esc to stop",
  28. "tui": "press Esc to stop",
  29. "desktop": "press Esc to stop",
  30. "api_server": "cancel the request to stop",
  31. "cron": "",
  32. }
  33. _DEFAULT_STOP_HINT = "send /stop to cancel"
  34. def auto_recovery_cycles(agent: Any) -> int:
  35. """Configured ``agent.auto_recovery_cycles`` (0 disables the ladder)."""
  36. return max(int(getattr(agent, "_auto_recovery_cycles", 0) or 0), 0)
  37. def ladder_wait_seconds(cycle: int, api_error: Any) -> float:
  38. """Wait before recovery ``cycle`` (1-based): jittered 15/30/60/60/60 s, or the provider's
  39. ``Retry-After`` when present (honoured past the 60 s cap, up to 120 s)."""
  40. from agent.retry_utils import jittered_backoff, provider_retry_after_seconds
  41. retry_after = provider_retry_after_seconds(api_error)
  42. if retry_after is not None:
  43. return min(retry_after, _RETRY_AFTER_CAP_S)
  44. return jittered_backoff(cycle, base_delay=_LADDER_BASE_DELAY_S, max_delay=_LADDER_CAP_S, jitter_ratio=0.2)
  45. def ladder_eligible(agent: Any, classified: Any) -> bool:
  46. """True when the ladder may engage: cycles configured, a transient transport verdict, and no
  47. answer text streamed to the user yet (delivered text is never replayed by this path)."""
  48. if auto_recovery_cycles(agent) <= 0 or classified.reason not in _LADDER_REASONS:
  49. return False
  50. streamed = getattr(agent, "_current_streamed_assistant_text", "") or ""
  51. return not agent._has_content_after_think_block(streamed)
  52. def ladder_notice(agent: Any, *, wait_s: float, cycle: int, total: int) -> str:
  53. hint = _STOP_HINTS.get(str(getattr(agent, "platform", "") or "").lower(), _DEFAULT_STOP_HINT)
  54. text = f"⏳ Provider temporarily unavailable — retrying automatically in {wait_s:.0f}s (cycle {cycle}/{total})"
  55. return f"{text}; {hint}" if hint else text
  56. def auto_recover_after_exhaustion(
  57. agent: Any, api_error: Any, classified: Any, _retry: Any, *, messages: Any,
  58. conversation_history: Any, api_call_count: int,
  59. ) -> Optional[dict[str, Any]]:
  60. """Run one recovery cycle after retries + fallback exhausted. Returns ``{"action": "continue"}``
  61. when the wait completed (caller zeroes ``retry_count`` and re-enters the retry loop),
  62. ``{"action": "break"}`` when a steering correction arrived mid-wait, ``{"action": "return",
  63. "result": …}`` when the user interrupted, and ``None`` when the ladder does not apply or is spent."""
  64. from agent.turn_recovery import interruptible_backoff_sleep
  65. if not ladder_eligible(agent, classified):
  66. return None
  67. total = auto_recovery_cycles(agent)
  68. used = int(getattr(_retry, "auto_recovery_cycles_used", 0) or 0)
  69. if used >= total:
  70. agent._emit_diagnostic_status(
  71. f"⏳ Automatic recovery gave up after {total} cycles — the provider is still unavailable."
  72. )
  73. return None
  74. cycle = used + 1
  75. _retry.auto_recovery_cycles_used = cycle
  76. wait_s = ladder_wait_seconds(cycle, api_error)
  77. notice = ladder_notice(agent, wait_s=wait_s, cycle=cycle, total=total)
  78. # Durable line on every surface (CLI print, TUI status.update, gateway bubble, api_server SSE)
  79. # plus the live wait line (spinner / thinking.delta / activity heartbeat).
  80. agent._emit_diagnostic_status(notice)
  81. agent._emit_diagnostic_wait(notice)
  82. logger.warning(
  83. "%sProvider unavailable (%s) — auto-recovery cycle %d/%d, retrying in %.0fs %s",
  84. agent.log_prefix, classified.reason.value, cycle, total, wait_s, agent._client_log_context(),
  85. )
  86. interrupted = interruptible_backoff_sleep(
  87. agent, wait_s, _retry, messages=messages, conversation_history=conversation_history,
  88. api_call_count=api_call_count,
  89. abort_message="Interrupt detected during automatic recovery wait, aborting.",
  90. interrupt_text=f"Operation interrupted: waiting for the provider to recover (cycle {cycle}/{total}).",
  91. activity_label=f"auto-recovery wait ({cycle}/{total})",
  92. )
  93. if interrupted is not None:
  94. return {"action": "return", "result": interrupted}
  95. if _retry.restart_with_redirected_messages:
  96. return {"action": "break"}
  97. return {"action": "continue"}