retry_utils.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195
  1. """Retry utilities — jittered backoff for decorrelated retries.
  2. Jittered delays (vs. fixed exponential) prevent thundering-herd retry spikes
  3. when many sessions hit the same rate-limited provider concurrently.
  4. """
  5. import random
  6. import re
  7. import threading
  8. import time
  9. from datetime import datetime, timezone, UTC
  10. from email.utils import parsedate_to_datetime
  11. from typing import Any, Optional
  12. # Monotonic counter for jitter-seed uniqueness within a process; locked
  13. # because concurrent gateway sessions retry simultaneously.
  14. _jitter_counter = 0
  15. _jitter_lock = threading.Lock()
  16. # Z.AI Coding Plan's GLM-5.2 endpoint often returns 429 code 1305 ("service may be
  17. # temporarily overloaded"). Short retries hammer the same window, so after
  18. # ``_ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS`` normal retries the wait widens progressively;
  19. # the cap stays interactive-friendly (a TUI message should fail visibly in minutes).
  20. # The short count is shared by ``adaptive_rate_limit_backoff`` and
  21. # ``zai_coding_overload_retry_ceiling`` so the two cannot silently desync.
  22. _ZAI_CODING_OVERLOAD_LONG_BACKOFF = (30.0, 60.0, 90.0, 120.0)
  23. _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS = 3
  24. def parse_retry_after_seconds(value_or_headers: Any) -> Optional[float]:
  25. """Parse a ``Retry-After`` value (numeric / HTTP-date) or a headers mapping (both casings tried) into
  26. seconds, clamped at 0.0; None when absent / unparseable."""
  27. raw = value_or_headers
  28. if raw is not None and not isinstance(raw, (str, int, float)):
  29. getter = getattr(raw, "get", None)
  30. if not callable(getter):
  31. return None
  32. try:
  33. raw = getter("Retry-After")
  34. if raw is None:
  35. raw = getter("retry-after")
  36. except Exception:
  37. return None
  38. if raw is None or isinstance(raw, bool):
  39. return None
  40. if isinstance(raw, (int, float)):
  41. return max(0.0, float(raw))
  42. text = str(raw).strip()
  43. if not text:
  44. return None
  45. try:
  46. return max(0.0, float(text))
  47. except (TypeError, ValueError):
  48. pass
  49. # HTTP-date form (RFC 7231): seconds until that instant, clamped at 0.
  50. try:
  51. when = parsedate_to_datetime(text)
  52. except (TypeError, ValueError):
  53. return None
  54. if when is None: # older stdlib returns None instead of raising
  55. return None
  56. if when.tzinfo is None:
  57. when = when.replace(tzinfo=UTC)
  58. return max(0.0, (when - datetime.now(UTC)).total_seconds())
  59. # Free-text "reset" grammars providers put in error bodies, tried in order. One table so the
  60. # conversation loop's error context and the credential pool's cooldown agree on the same wait.
  61. _QUOTA_RESET_DELAY_RE = re.compile(r"quotaResetDelay[:\s\"]+(\d+(?:\.\d+)?)(ms|s)", re.IGNORECASE)
  62. # "Resets in 4hr 5min" (weekly usage limits), "resets in 2 hours 5 minutes", "resets in 30s".
  63. _RESETS_IN_RE = re.compile(
  64. r"resets?\s+in\s+"
  65. r"(?:(\d+(?:\.\d+)?)\s*(?:h|hr|hrs|hour|hours)\b\s*)?"
  66. r"(?:(\d+(?:\.\d+)?)\s*(?:m|min|mins|minute|minutes)\b\s*)?"
  67. r"(?:(\d+(?:\.\d+)?)\s*(?:s|sec|secs|second|seconds)\b)?", re.IGNORECASE,
  68. )
  69. _RETRY_AFTER_SECONDS_RE = re.compile(r"retry\s+(?:after\s+)?(\d+(?:\.\d+)?)\s*(?:sec|secs|seconds|s\b)", re.IGNORECASE)
  70. # The plan usage-limit body field as it appears once stringified: ``'resets_in_seconds': 30995``.
  71. _RESETS_IN_SECONDS_FIELD_RE = re.compile(r"resets_in_seconds\W{1,4}(\d+(?:\.\d+)?)", re.IGNORECASE)
  72. def _quota_reset_seconds(m: re.Match[str]) -> float:
  73. value = float(m.group(1))
  74. return value / 1000.0 if m.group(2).lower() == "ms" else value
  75. def _resets_in_seconds(m: re.Match[str]) -> Optional[float]:
  76. if not any(m.groups()): # "resets in" with no unit-bearing number: not this grammar
  77. return None
  78. return float(m.group(1) or 0) * 3600 + float(m.group(2) or 0) * 60 + float(m.group(3) or 0)
  79. # An explicit "retry after N s" wins over "resets in ..." (the credential pool's precedence):
  80. # a body carrying both describes a short throttle inside a long quota window, and the
  81. # shorter explicit wait is the one the provider actually asks for.
  82. RETRY_DELAY_PATTERNS = (
  83. (_QUOTA_RESET_DELAY_RE, _quota_reset_seconds),
  84. (_RETRY_AFTER_SECONDS_RE, lambda m: float(m.group(1))),
  85. (_RESETS_IN_SECONDS_FIELD_RE, lambda m: float(m.group(1))),
  86. (_RESETS_IN_RE, _resets_in_seconds),
  87. )
  88. def format_reset_window(seconds: float) -> str:
  89. """``~9h`` / ``~45 min`` for chat copy naming when a quota window reopens (ceilinged)."""
  90. seconds = int(seconds)
  91. return f"~{-(-seconds // 3600)}h" if seconds >= 3600 else f"~{-(-seconds // 60)} min"
  92. def reset_delay_from_message(message: str) -> Optional[float]:
  93. """Seconds-until-reset parsed from free-text provider error messages, or None."""
  94. if not message:
  95. return None
  96. for pattern, to_seconds in RETRY_DELAY_PATTERNS:
  97. m = pattern.search(message)
  98. if m and (seconds := to_seconds(m)) is not None:
  99. return seconds
  100. return None
  101. def jittered_backoff(attempt: int, *, base_delay: float = 5.0, max_delay: float = 120.0, jitter_ratio: float = 0.5) -> float:
  102. """min(base * 2^(attempt-1), max_delay) + uniform jitter in
  103. [0, jitter_ratio * delay]. ``attempt`` is 1-based."""
  104. global _jitter_counter
  105. with _jitter_lock:
  106. _jitter_counter += 1
  107. tick = _jitter_counter
  108. exponent = max(0, attempt - 1)
  109. delay = max_delay if (exponent >= 63 or base_delay <= 0) else min(base_delay * (2 ** exponent), max_delay)
  110. # Seed from time + counter so coarse clocks still decorrelate.
  111. seed = (time.time_ns() ^ (tick * 0x9E3779B9)) & 0xFFFFFFFF
  112. return delay + random.Random(seed).uniform(0, jitter_ratio * delay)
  113. def _error_text(error: Any) -> str:
  114. """Best-effort flattened provider error text for retry classification."""
  115. parts = [error, getattr(error, "message", None), getattr(error, "body", None), getattr(error, "response", None)]
  116. return " ".join(str(part) for part in parts if part is not None).lower()
  117. def is_zai_coding_overload_error(*, base_url: str | None, model: str | None, error: Any) -> bool:
  118. """True only for the narrow Z.AI Coding Plan overload shape (429 + code
  119. 1305 / "temporarily overloaded"), so ordinary quota 429s still fail fast."""
  120. text = _error_text(error)
  121. return (
  122. getattr(error, "status_code", None) == 429
  123. and "api.z.ai/api/coding/paas/v4" in (base_url or "").lower()
  124. and "glm-5.2" in (model or "").lower()
  125. and ("1305" in text or "temporarily overloaded" in text)
  126. )
  127. def adaptive_rate_limit_backoff(
  128. attempt: int, *, base_url: str | None, model: str | None, error: Any, default_wait: float,
  129. short_attempts: int = _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS,
  130. ) -> tuple[float, str | None]:
  131. """``(wait_seconds, reason_label)``: ``default_wait`` for most providers; Z.AI Coding GLM-5.2 overloads keep
  132. ``short_attempts`` short retries, then 30→60→90→120s with light jitter. ``attempt`` is 1-based."""
  133. if not is_zai_coding_overload_error(base_url=base_url, model=model, error=error):
  134. return default_wait, None
  135. if attempt <= short_attempts:
  136. return default_wait, "zai_coding_overload_short"
  137. idx = min(attempt - short_attempts - 1, len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) - 1)
  138. base_delay = _ZAI_CODING_OVERLOAD_LONG_BACKOFF[idx]
  139. return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), "zai_coding_overload_long"
  140. def zai_coding_overload_retry_ceiling(short_attempts: int = _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS) -> int:
  141. """Retry-loop ceiling for the full Z.AI overload schedule: one past the last long entry,
  142. because the loop gives up when ``retry_count >= ceiling`` BEFORE computing the attempt's
  143. backoff (the default ``api_max_retries`` of 3 equals ``short_attempts``)."""
  144. return short_attempts + len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) + 1
  145. # A wait longer than this is one a person feels: a non-rate-limit Retry-After this long is announced
  146. # when it starts, and on the Nous free tier an attended session ends the turn instead of sitting through it.
  147. LIVE_RETRY_WAIT_CAP_S = 60.0
  148. # Anthropic Tier 1 input-token buckets reset in ~171s, so a 120s cap re-tripped the limit; 600s
  149. # covers realistic provider windows while still rejecting pathological values (#26293).
  150. RETRY_AFTER_CAP_S = 600.0
  151. def provider_retry_after_seconds(error: Any) -> Optional[float]:
  152. """Provider-declared cooldown: the ``Retry-After`` header, else a ``retry_after`` body field
  153. (top level or nested under ``error``). None when absent, unparseable or zero: a zero or expired
  154. cooldown carries no usable wait, and treating it as one would hot-loop the provider."""
  155. value = parse_retry_after_seconds(getattr(getattr(error, "response", None), "headers", None))
  156. if value is None:
  157. body = getattr(error, "body", None)
  158. if isinstance(body, dict):
  159. nested = body.get("error")
  160. value = parse_retry_after_seconds((nested if isinstance(nested, dict) else body).get("retry_after"))
  161. return value if value is not None and value > 0 else None