credential_pool_model_cooldowns.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. """Model-scoped rate-limit cooldowns for pooled credentials.
  2. Anthropic enforces its API rate limits (requests / tokens per minute) per
  3. model, so a generic 429 for one Claude model says nothing about the same
  4. credential's standing for its sibling models. Such a 429 is recorded as a
  5. cooldown on the requested model only, beside the credential-wide status that
  6. auth, billing and payment failures keep benching the whole credential with.
  7. """
  8. from __future__ import annotations
  9. import time
  10. from typing import Any, Dict, Iterable, Optional, TYPE_CHECKING
  11. if TYPE_CHECKING:
  12. from agent.credential_pool import PooledCredential
  13. # A Codex ChatGPT-account model entitlement 400 is a plan property, not a window: bench the
  14. # (credential, model) pair until an explicit ``hermes auth reset`` clears model_cooldowns (#71970).
  15. MODEL_ENTITLEMENT_BENCH_SECONDS = 365 * 24 * 60 * 60
  16. # A usage-window plan limit: this model is refused until the allowance resets; every other model
  17. # (a Free or non-Large one) stays usable on the same credential.
  18. FAILURE_REASON_USAGE_WINDOW = "usage_window"
  19. def pool_failure_reason(reason: Any, error_context: Optional[dict[str, Any]], billing_unverified: bool) -> str:
  20. """The pool's failure reason for a rotation (*reason* is a ``FailoverReason``). A usage-window 402
  21. benches only the refused model; an ambiguous billing body is sized as transient, not a 1-hour bench."""
  22. from agent.error_classifier import FailoverReason
  23. if reason == FailoverReason.billing and (error_context or {}).get("usage_window"):
  24. return FAILURE_REASON_USAGE_WINDOW
  25. if reason == FailoverReason.billing and billing_unverified:
  26. from agent.credential_pool import FAILURE_REASON_BILLING_UNVERIFIED
  27. return FAILURE_REASON_BILLING_UNVERIFIED
  28. return reason.value
  29. def model_cooldown_until(entry: PooledCredential, model: Optional[str]) -> Optional[float]:
  30. """Active cooldown blocking *entry* for *model*, or ``None``.
  31. Callers that do not know the model stay conservative: any active model
  32. cooldown blocks them, so an unscoped route cannot reuse the credential.
  33. """
  34. cooldowns = entry.model_cooldowns or {}
  35. values = cooldowns.values() if not model else (cooldowns.get(model),)
  36. now = time.time()
  37. active = [float(until) for until in values if isinstance(until, (int, float)) and until > now]
  38. return max(active) if active else None
  39. def merge_model_cooldowns(*maps: Any) -> dict[str, float]:
  40. """Latest reset per model across snapshots — each writer only observed its own model."""
  41. merged: dict[str, float] = {}
  42. for cooldowns in maps:
  43. if not isinstance(cooldowns, dict):
  44. continue
  45. for model, until in cooldowns.items():
  46. if isinstance(until, (int, float)):
  47. merged[model] = max(float(until), merged.get(model, 0.0))
  48. return merged
  49. class CredentialPoolModelCooldownMixin:
  50. def token_is_blocked(self, token: str, *, model: Optional[str] = None) -> bool:
  51. """Whether a pool cooldown blocks *token* for *model*.
  52. Closes the paths that hand out a native Anthropic token without
  53. selecting it from the pool (env / borrowed credentials). Tokens the
  54. pool does not know fail open: no row can attribute a cooldown to them.
  55. """
  56. with self._lock:
  57. return any(
  58. entry.runtime_api_key == token and model_cooldown_until(entry, model) is not None
  59. for entry in self._entries
  60. )
  61. def _is_model_scoped_failure(
  62. self, status_code: Optional[int], model: Optional[str], failure_reason: Optional[str],
  63. ) -> bool:
  64. """Anthropic per-model 429s, and a Codex ChatGPT-account model entitlement 400: the
  65. account cannot use *model*, but the credential stays valid for every other model (#71970)."""
  66. from agent.credential_pool import FAILURE_REASON_BILLING, FAILURE_REASON_BILLING_UNVERIFIED
  67. if not model:
  68. return False
  69. if failure_reason in ("model_entitlement", FAILURE_REASON_USAGE_WINDOW):
  70. return True
  71. return (
  72. self.provider == "anthropic" and status_code == 429
  73. and failure_reason not in (FAILURE_REASON_BILLING, FAILURE_REASON_BILLING_UNVERIFIED)
  74. )
  75. def _cool_down_model(
  76. self, entry: PooledCredential, model: str, error_context: Optional[dict[str, Any]],
  77. failure_reason: Optional[str] = None,
  78. ) -> None:
  79. """Record a cooldown for *model* on *entry* and every sibling sharing its key.
  80. Same TTL policy as a credential-wide 429 (provider ``reset_at`` wins, a
  81. sole credential keeps its short bench), except a ``model_entitlement``
  82. rejection, which stays benched until the explicit reset path clears it.
  83. Siblings matter because a ``model_config`` twin seeded from the same key
  84. would otherwise be re-selected for the very model that just failed.
  85. Caller holds the lock.
  86. """
  87. from agent.credential_pool import _exhausted_ttl, _normalize_error_context
  88. if failure_reason == "model_entitlement":
  89. until = time.time() + MODEL_ENTITLEMENT_BENCH_SECONDS
  90. else:
  91. until = _normalize_error_context(error_context).get("reset_at") or (
  92. time.time() + _exhausted_ttl(429, sole_credential=self._is_sole_credential())
  93. )
  94. failed_key = entry.runtime_api_key
  95. for scoped in list(self._entries):
  96. if scoped.id != entry.id and not (failed_key and scoped.runtime_api_key == failed_key):
  97. continue
  98. cooldowns = merge_model_cooldowns(scoped.model_cooldowns, {model: until})
  99. self._adopt(scoped, persist=False, model_cooldowns=cooldowns)
  100. self._persist()
  101. def limit_state(self, models: Iterable[str]) -> Optional[dict[str, Any]]:
  102. """What a picker should say about this pool's rate limits, or ``None`` when nothing is limited.
  103. ``{"scope": "account", "resets_at": epoch}`` when every live entry is benched credential-wide
  104. (the whole login is out; another model won't help), else ``{"scope": "models", "models":
  105. {model: epoch}}`` for the given *models* no usable entry can serve until a model cooldown
  106. ends. Entitlement benches (a year: the plan lacks the model) aren't windows that reset, so
  107. they're left out. Read-only: never clears or persists a cooldown.
  108. """
  109. from agent.credential_pool import STATUS_DEAD, _exhausted_until
  110. now = time.time()
  111. with self._lock:
  112. live = [entry for entry in self._entries if entry.last_status != STATUS_DEAD]
  113. sole = len(live) <= 1
  114. benched = {entry.id: until for entry in live
  115. if (until := _exhausted_until(entry, sole_credential=sole)) and until > now}
  116. if live and len(benched) == len(live):
  117. return {"scope": "account", "resets_at": min(benched.values())}
  118. usable = [entry for entry in live if entry.id not in benched]
  119. cooled: dict[str, float] = {}
  120. for model in models:
  121. waits = [model_cooldown_until(entry, model) for entry in usable]
  122. if waits and all(waits) and min(waits) - now < MODEL_ENTITLEMENT_BENCH_SECONDS / 2:
  123. cooled[model] = min(waits)
  124. return {"scope": "models", "models": cooled} if cooled else None