| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129 |
- """Primary rate-limit cooldown arming and per-session model rejection markers, shared by the
- fallback walk (chat_completion_helpers) and restore_primary_runtime (agent_runtime_helpers)."""
- import logging
- import math
- import time
- from agent.error_classifier import FailoverReason
- logger = logging.getLogger(__name__)
- _RATE_LIMIT_FAILOVER_REASONS = frozenset({FailoverReason.rate_limit, FailoverReason.billing, FailoverReason.upstream_rate_limit})
- def _provider_reset_delay(reset_at) -> float | None:
- """Seconds until the provider-declared reset, or None when missing/invalid/expired."""
- from agent.credential_pool import _parse_absolute_timestamp
- parsed = _parse_absolute_timestamp(reset_at)
- delay = parsed - time.time() if parsed is not None else None
- if delay is not None and math.isfinite(delay) and delay > 0:
- return delay
- return None
- def switch_deferred_by_reset(agent, reason: FailoverReason | None, reset_at) -> bool:
- """Opt-in ``fallback.min_switch_reset_seconds`` (default 0 = off, #117484): when the primary's
- rate limit reopens sooner than N seconds, switching model mid-task costs more than waiting, so
- the fallback walk is skipped and the retry loop's own backoff rides out the window. Only for
- rate-limit failovers leaving the primary with a valid future ``reset_at``."""
- if reason not in _RATE_LIMIT_FAILOVER_REASONS or getattr(agent, "_fallback_activated", False):
- return False
- try:
- from hermes_cli.config import load_config
- threshold = float((load_config() or {}).get("fallback", {}).get("min_switch_reset_seconds") or 0)
- except Exception:
- return False
- if threshold <= 0:
- return False
- delay = _provider_reset_delay(reset_at)
- if delay is None or delay >= threshold:
- return False
- logging.info("Rate limit resets in %.0f s (< fallback.min_switch_reset_seconds=%.0f): staying on the primary", delay, threshold)
- return True
- def _arm_rate_limit_cooldown(
- agent, reason: FailoverReason | None, reset_at=None,
- ) -> int | None:
- """Arm the primary cooldown until the provider reset, or use exponential backoff.
- ``reset_at`` is an absolute wall-clock timestamp while ``_rate_limited_until`` is monotonic;
- convert through a duration so wall-clock epoch values never enter the monotonic comparison.
- Missing, invalid, or expired provider resets retain the 60s → 2m → ... → 4h fallback.
- Only arm when leaving the primary: chain-switching from an active fallback means the primary
- was not the failing source. Return the armed cooldown in seconds, or None when not armed.
- """
- if reason not in _RATE_LIMIT_FAILOVER_REASONS:
- return None
- current_provider = (getattr(agent, "provider", "") or "").strip().lower()
- primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower()
- if getattr(agent, "_fallback_activated", False) and not (primary_provider and current_provider == primary_provider):
- return None
- backoff_count = getattr(agent, "_rate_limit_backoff_count", 0)
- agent._rate_limit_backoff_count = backoff_count + 1
- provider_delay = _provider_reset_delay(reset_at)
- if provider_delay is not None:
- backoff_seconds = math.ceil(provider_delay)
- source = "provider reset"
- else:
- backoff_seconds = min(60 * (2 ** backoff_count), 14400)
- source = "exponential fallback"
- agent._rate_limited_until = time.monotonic() + backoff_seconds
- logging.info(
- "Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d, %s)",
- backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1, source,
- )
- return backoff_seconds
- def _mark_entitlement_rejected_model(agent, api_error) -> bool:
- """Record a Codex ChatGPT-account 400 that rejects the current model for this account.
- Pool rotation runs first (recover_with_credential_pool benches (credential, model) and
- moves to the next entitled entry, #71970); this runs only once no pool entry is left for
- the model, so the (provider, model) pair is treated as dead for the session: the fallback
- walk skips it and restore_primary_runtime stops switching back — otherwise every turn
- re-fails on the primary, announces an unverified "Primary model restored", and oscillates
- forever (#106475).
- """
- if getattr(api_error, "status_code", None) != 400:
- return False
- from agent.error_classifier import CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER
- haystack = str(getattr(api_error, "message", "") or api_error).lower()
- if CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER not in haystack:
- return False
- provider = str(getattr(agent, "provider", "") or "").strip().lower()
- model = str(getattr(agent, "model", "") or "").strip()
- if not provider or not model:
- return False
- pool = getattr(agent, "_credential_pool", None)
- if pool is not None and pool.has_available(model=model):
- return False # another pool entry is still eligible for this model; rotation owns it
- rejected = getattr(agent, "_entitlement_rejected_models", None)
- if rejected is None:
- rejected = agent._entitlement_rejected_models = set()
- if (provider, model) in rejected:
- return True
- rejected.add((provider, model))
- logger.warning(
- "Model entitlement rejection: this account is not entitled to %s via %s; "
- "treating it as unavailable for this session",
- model, provider,
- )
- agent._buffer_diagnostic_status(
- f"🚫 This account is not entitled to {model} via {provider}; it will be skipped "
- "until restart. Switch to an entitled model via /model or `hermes model`."
- )
- return True
- def _is_entitlement_rejected(agent, provider: str, model: str) -> bool:
- """True when (provider, model) — as configured or normalized — was rejected as unentitled
- for this account (see _mark_entitlement_rejected_model)."""
- rejected = getattr(agent, "_entitlement_rejected_models", None) or ()
- if not rejected:
- return False
- if (provider, model) in rejected:
- return True
- from hermes_cli.model_normalize import normalize_model_for_provider
- return (provider, normalize_model_for_provider(model, provider)) in rejected
|