auxiliary_rate_limit_backoff.py 3.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071
  1. """Same-credential backoff for an auxiliary call refused by a rate limit (429, Bedrock throttle).
  2. Auxiliary clients make one attempt per call (OpenAI SDK ``max_retries=0``, botocore
  3. ``total_max_attempts: 1``), so the recovery ladder owns the wait. An immediate re-send lands in the
  4. same throttle window and spends the retry, then credential rotation and the provider fallback chain
  5. see a provider that would have answered a moment later. The ladder yields a ``wait`` step; the sync
  6. and async ladder drivers sleep it, so both caller kinds share one schedule.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. from typing import Any, Optional
  11. from agent import retry_utils
  12. logger = logging.getLogger(__name__)
  13. # One same-credential retry, then credential-pool rotation and the provider fallback chain take over.
  14. RATE_LIMIT_RETRIES = 1
  15. # Jittered exponential backoff (seconds): 1-1.5 before the first retry, doubling up to the cap.
  16. RATE_LIMIT_BACKOFF_BASE = 1.0
  17. RATE_LIMIT_BACKOFF_CAP = 4.0
  18. # A provider-declared cooldown longer than this is not worth sitting through inside an auxiliary
  19. # call (compression, titles, summaries): rotate or fall back instead.
  20. RATE_LIMIT_RETRY_AFTER_CAP = 5.0
  21. def rate_limit_retry_after(exc: Exception) -> Optional[float]:
  22. """Provider-declared cooldown in seconds: ``Retry-After`` / a ``retry_after`` body field, or the
  23. ``retry-after`` header botocore keeps in ``ResponseMetadata.HTTPHeaders``; None when absent."""
  24. value = retry_utils.provider_retry_after_seconds(exc)
  25. response = getattr(exc, "response", None)
  26. if value is None and isinstance(response, dict): # botocore ClientError: the parsed response dict
  27. value = retry_utils.parse_retry_after_seconds(response.get("ResponseMetadata", {}).get("HTTPHeaders"))
  28. return value if value is not None and value > 0 else None
  29. def rate_limit_wait(exc: Exception, attempt: int) -> Optional[float]:
  30. """Seconds to wait before same-credential retry ``attempt`` (1-based), or None when the provider
  31. declared a cooldown longer than ``RATE_LIMIT_RETRY_AFTER_CAP``."""
  32. declared = rate_limit_retry_after(exc)
  33. if declared is not None:
  34. return declared if declared <= RATE_LIMIT_RETRY_AFTER_CAP else None
  35. return retry_utils.jittered_backoff(
  36. attempt, base_delay=RATE_LIMIT_BACKOFF_BASE, max_delay=RATE_LIMIT_BACKOFF_CAP)
  37. def rate_limit_retry_rungs(first_err: Exception, client: Any, kwargs: dict[str, Any], label: str):
  38. """Ladder rungs: wait, then re-send on the same client, up to ``RATE_LIMIT_RETRIES`` times while
  39. the refusal stays a rate limit. Returns ``(response, None)`` or ``(None, last_err)``."""
  40. from agent.auxiliary_client import (
  41. _LadderStep, _credential_rung_accepts, _is_payment_error, _is_rate_limit_error, _rung,
  42. )
  43. err = first_err
  44. for attempt in range(1, RATE_LIMIT_RETRIES + 1):
  45. wait = rate_limit_wait(err, attempt)
  46. if wait is None:
  47. logger.info("Auxiliary %s: provider asked for a %.0fs cooldown; not waiting it out",
  48. label, rate_limit_retry_after(err) or 0.0)
  49. break
  50. logger.info("Auxiliary %s: rate limited (attempt %d/%d); retrying after %.2fs",
  51. label, attempt, RATE_LIMIT_RETRIES, wait)
  52. yield _LadderStep("wait", (wait,))
  53. resp, err = yield from _rung(_LadderStep("call", (client, kwargs)), _credential_rung_accepts)
  54. if err is None:
  55. return resp, None
  56. if not _is_rate_limit_error(err) or _is_payment_error(err):
  57. break
  58. return None, err