fallback_cooldown.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. """Primary rate-limit cooldown arming and per-session model rejection markers, shared by the
  2. fallback walk (chat_completion_helpers) and restore_primary_runtime (agent_runtime_helpers)."""
  3. import logging
  4. import math
  5. import time
  6. from agent.error_classifier import FailoverReason
  7. logger = logging.getLogger(__name__)
  8. _RATE_LIMIT_FAILOVER_REASONS = frozenset({FailoverReason.rate_limit, FailoverReason.billing, FailoverReason.upstream_rate_limit})
  9. def _provider_reset_delay(reset_at) -> float | None:
  10. """Seconds until the provider-declared reset, or None when missing/invalid/expired."""
  11. from agent.credential_pool import _parse_absolute_timestamp
  12. parsed = _parse_absolute_timestamp(reset_at)
  13. delay = parsed - time.time() if parsed is not None else None
  14. if delay is not None and math.isfinite(delay) and delay > 0:
  15. return delay
  16. return None
  17. def switch_deferred_by_reset(agent, reason: FailoverReason | None, reset_at) -> bool:
  18. """Opt-in ``fallback.min_switch_reset_seconds`` (default 0 = off, #117484): when the primary's
  19. rate limit reopens sooner than N seconds, switching model mid-task costs more than waiting, so
  20. the fallback walk is skipped and the retry loop's own backoff rides out the window. Only for
  21. rate-limit failovers leaving the primary with a valid future ``reset_at``."""
  22. if reason not in _RATE_LIMIT_FAILOVER_REASONS or getattr(agent, "_fallback_activated", False):
  23. return False
  24. try:
  25. from hermes_cli.config import load_config
  26. threshold = float((load_config() or {}).get("fallback", {}).get("min_switch_reset_seconds") or 0)
  27. except Exception:
  28. return False
  29. if threshold <= 0:
  30. return False
  31. delay = _provider_reset_delay(reset_at)
  32. if delay is None or delay >= threshold:
  33. return False
  34. logging.info("Rate limit resets in %.0f s (< fallback.min_switch_reset_seconds=%.0f): staying on the primary", delay, threshold)
  35. return True
  36. def _arm_rate_limit_cooldown(
  37. agent, reason: FailoverReason | None, reset_at=None,
  38. ) -> int | None:
  39. """Arm the primary cooldown until the provider reset, or use exponential backoff.
  40. ``reset_at`` is an absolute wall-clock timestamp while ``_rate_limited_until`` is monotonic;
  41. convert through a duration so wall-clock epoch values never enter the monotonic comparison.
  42. Missing, invalid, or expired provider resets retain the 60s → 2m → ... → 4h fallback.
  43. Only arm when leaving the primary: chain-switching from an active fallback means the primary
  44. was not the failing source. Return the armed cooldown in seconds, or None when not armed.
  45. """
  46. if reason not in _RATE_LIMIT_FAILOVER_REASONS:
  47. return None
  48. current_provider = (getattr(agent, "provider", "") or "").strip().lower()
  49. primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower()
  50. if getattr(agent, "_fallback_activated", False) and not (primary_provider and current_provider == primary_provider):
  51. return None
  52. backoff_count = getattr(agent, "_rate_limit_backoff_count", 0)
  53. agent._rate_limit_backoff_count = backoff_count + 1
  54. provider_delay = _provider_reset_delay(reset_at)
  55. if provider_delay is not None:
  56. backoff_seconds = math.ceil(provider_delay)
  57. source = "provider reset"
  58. else:
  59. backoff_seconds = min(60 * (2 ** backoff_count), 14400)
  60. source = "exponential fallback"
  61. agent._rate_limited_until = time.monotonic() + backoff_seconds
  62. logging.info(
  63. "Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d, %s)",
  64. backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1, source,
  65. )
  66. return backoff_seconds
  67. def _mark_entitlement_rejected_model(agent, api_error) -> bool:
  68. """Record a Codex ChatGPT-account 400 that rejects the current model for this account.
  69. Pool rotation runs first (recover_with_credential_pool benches (credential, model) and
  70. moves to the next entitled entry, #71970); this runs only once no pool entry is left for
  71. the model, so the (provider, model) pair is treated as dead for the session: the fallback
  72. walk skips it and restore_primary_runtime stops switching back — otherwise every turn
  73. re-fails on the primary, announces an unverified "Primary model restored", and oscillates
  74. forever (#106475).
  75. """
  76. if getattr(api_error, "status_code", None) != 400:
  77. return False
  78. from agent.error_classifier import CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER
  79. haystack = str(getattr(api_error, "message", "") or api_error).lower()
  80. if CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER not in haystack:
  81. return False
  82. provider = str(getattr(agent, "provider", "") or "").strip().lower()
  83. model = str(getattr(agent, "model", "") or "").strip()
  84. if not provider or not model:
  85. return False
  86. pool = getattr(agent, "_credential_pool", None)
  87. if pool is not None and pool.has_available(model=model):
  88. return False # another pool entry is still eligible for this model; rotation owns it
  89. rejected = getattr(agent, "_entitlement_rejected_models", None)
  90. if rejected is None:
  91. rejected = agent._entitlement_rejected_models = set()
  92. if (provider, model) in rejected:
  93. return True
  94. rejected.add((provider, model))
  95. logger.warning(
  96. "Model entitlement rejection: this account is not entitled to %s via %s; "
  97. "treating it as unavailable for this session",
  98. model, provider,
  99. )
  100. agent._buffer_diagnostic_status(
  101. f"🚫 This account is not entitled to {model} via {provider}; it will be skipped "
  102. "until restart. Switch to an entitled model via /model or `hermes model`."
  103. )
  104. return True
  105. def _is_entitlement_rejected(agent, provider: str, model: str) -> bool:
  106. """True when (provider, model) — as configured or normalized — was rejected as unentitled
  107. for this account (see _mark_entitlement_rejected_model)."""
  108. rejected = getattr(agent, "_entitlement_rejected_models", None) or ()
  109. if not rejected:
  110. return False
  111. if (provider, model) in rejected:
  112. return True
  113. from hermes_cli.model_normalize import normalize_model_for_provider
  114. return (provider, normalize_model_for_provider(model, provider)) in rejected