| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165 |
- """Credential-pool recovery for an auth failure (401/403) on the dispatched credential.
- Nous answers bursts of ``401 {"error": "authentication required"}`` for agent keys that are
- still unexpired and valid a few seconds later. Re-minting cannot fix a server-side blip, and
- each forced refresh redeems the single-use refresh token, so a Nous 401 episode gets exactly
- ONE forced refresh. Any later 401 in the same turn (or a refresh that handed back the bearer
- that just failed) waits out ``TRANSIENT_AUTH_BACKOFF_S`` and retries with the store's current
- key, adopting a peer's rotation but never POSTing again. A dead grant (refresh rejected)
- still fails fast through rotation, and so does an entitlement 403.
- """
- from __future__ import annotations
- import time
- from typing import Any, Optional
- # Waits before each retry of an unexpired Nous key that keeps answering 401 (~50 s in total).
- TRANSIENT_AUTH_BACKOFF_S = (5.0, 15.0, 30.0)
- # Cap same-entry OAuth refreshes on a persistent auth failure, else a single-entry pool re-mints forever.
- _MAX_AUTH_REFRESH_ATTEMPTS = 2
- _ACTIVITY_TOUCH_S = 30.0
- # A 401 this long after the episode's last one starts a new episode (one more refresh allowed):
- # hour-long turns must still recover from a key revoked later in the same turn.
- _EPISODE_IDLE_S = 120.0
- def _ra():
- import run_agent
- return run_agent
- def _episode(agent: Any, provider: str, entry_id: Optional[str]) -> dict[str, Any]:
- """Per-turn auth-failure state for one pool entry (the turn loop resets the map)."""
- if getattr(agent, "_auth_pool_refresh_counts", None) is None:
- agent._auth_pool_refresh_counts = {}
- now = time.monotonic()
- episode = agent._auth_pool_refresh_counts.get((provider, entry_id))
- if episode is None or now - episode["last"] > _EPISODE_IDLE_S:
- episode = agent._auth_pool_refresh_counts[(provider, entry_id)] = {"refreshes": 0, "waits": 0}
- episode["last"] = now
- return episode
- def nous_pool_refreshed_this_turn(agent: Any) -> bool:
- """True once the pool spent this turn's forced Nous refresh (the singleton path must not re-POST)."""
- counts = getattr(agent, "_auth_pool_refresh_counts", None) or {}
- return any(key[0] == "nous" and state.get("refreshes") for key, state in counts.items())
- def _interruptible_wait(agent: Any, seconds: float) -> bool:
- """Sleep in 200 ms slices; False when the user interrupted."""
- deadline = time.monotonic() + seconds
- next_touch = time.monotonic() + _ACTIVITY_TOUCH_S
- while time.monotonic() < deadline:
- if getattr(agent, "_interrupt_requested", False):
- return False
- time.sleep(min(0.2, max(0.0, deadline - time.monotonic())))
- if time.monotonic() >= next_touch:
- agent._touch_activity("waiting out a Nous 401 on a valid key")
- next_touch = time.monotonic() + _ACTIVITY_TOUCH_S
- return True
- def _entry_for(pool: Any, credential_id: Optional[str], api_key_hint: Optional[str]) -> Any:
- entries = pool.entries()
- if credential_id:
- found = next((e for e in entries if e.id == credential_id), None)
- if found is not None:
- return found
- return next((e for e in entries if api_key_hint and e.runtime_api_key == api_key_hint), None)
- def _retry_after_backoff(agent: Any, pool: Any, episode: dict[str, Any], *, failed_key: Optional[str],
- entry_id: Optional[str], credential_id: Optional[str], has_retried_429: bool):
- """Wait out a server-side 401 on an unexpired key, then retry with the store's current key."""
- log = _ra().logger
- step = episode["waits"]
- if step >= len(TRANSIENT_AUTH_BACKOFF_S):
- log.warning(
- "Credential auth failure persists for pool entry %s after one refresh and %s "
- "backoff retries — treating as unrecoverable and allowing fallback to activate.",
- entry_id, step)
- return False, has_retried_429
- wait = TRANSIENT_AUTH_BACKOFF_S[step]
- episode["waits"] = step + 1
- log.info(
- "Credential auth failure on an unexpired key for pool entry %s — transient 401, "
- "retrying in %.0fs (backoff %s/%s, no further refresh)",
- entry_id, wait, step + 1, len(TRANSIENT_AUTH_BACKOFF_S))
- agent._vprint(
- f"{agent.log_prefix}🔐 Nous rejected a still-valid key (transient); retrying in {wait:.0f}s...",
- force=True, diagnostic=True)
- if not _interruptible_wait(agent, wait):
- # Not recovered: the caller's interrupt check then aborts (or redirects) the turn. A
- # True here re-sends the request, and the retry loop never re-checks the interrupt.
- return False, has_retried_429
- # A peer process may have rotated meanwhile: adopt what the store holds now (no POST).
- entry = _entry_for(pool, credential_id or entry_id, failed_key)
- if entry is not None:
- if agent._swap_credential(pool._sync_nous_entry_from_auth_store(entry)) is False:
- return False, has_retried_429
- return True, has_retried_429
- def recover_auth_failure(agent, pool, *, status_code, has_retried_429, error_context, api_key_hint,
- credential_id, rotate_and_swap):
- from agent.agent_runtime_helpers import _is_entitlement_403
- log = _ra().logger
- if _is_entitlement_403(agent, status_code, error_context):
- log.info(
- "Credential %s — entitlement-shaped 403 from %s; "
- "skipping pool refresh (account lacks subscription, "
- "not a transient auth failure).", status_code if status_code is not None else "auth",
- agent.provider or "provider",
- )
- return False, has_retried_429
- provider = agent.provider
- # Only a 401 is the "valid key refused" shape worth backing off on; a non-entitlement 403
- # keeps the capped-refresh path and fails fast.
- nous = provider == "nous" and status_code == 401
- entry_id = credential_id
- if nous:
- found = _entry_for(pool, credential_id, api_key_hint)
- entry_id = getattr(found, "id", None) or credential_id
- episode = _episode(agent, provider, entry_id)
- if episode["refreshes"] >= 1:
- # The one forced refresh of this episode already happened: never re-redeem the grant.
- return _retry_after_backoff(
- agent, pool, episode, failed_key=api_key_hint, entry_id=entry_id,
- credential_id=credential_id, has_retried_429=has_retried_429)
- # Refresh the entry that supplied the failing key, not current(): refreshing a healthy entry
- # burns its single-use refresh token for a failure it never had.
- refresh_kwargs = {"api_key_hint": api_key_hint}
- if credential_id:
- refresh_kwargs["credential_id"] = credential_id
- refreshed = pool.try_refresh_matching(**refresh_kwargs)
- if refreshed is None:
- # Refresh failed; rotate (the failed entry is already marked exhausted).
- return (True, False) if rotate_and_swap(401, "auth refresh failed") else (False, has_retried_429)
- refreshed_id = getattr(refreshed, "id", None)
- if nous:
- episode = _episode(agent, provider, refreshed_id or entry_id)
- episode["refreshes"] += 1
- if api_key_hint and getattr(refreshed, "runtime_api_key", None) == api_key_hint:
- # The "refresh" handed back the bearer that just failed: replaying it at once is
- # the 1.5 s give-up; it is a valid key the server is refusing, so back off instead.
- return _retry_after_backoff(
- agent, pool, episode, failed_key=api_key_hint, entry_id=refreshed_id,
- credential_id=credential_id, has_retried_429=has_retried_429)
- elif refreshed_id is not None:
- # try_refresh_matching() reports success even when upstream keeps rejecting; cap same-entry
- # refreshes so a single-entry pool falls through to fallback.
- state = _episode(agent, provider, refreshed_id)
- state["refreshes"] += 1
- if state["refreshes"] > _MAX_AUTH_REFRESH_ATTEMPTS:
- log.warning(
- "Credential auth failure persists after %s refreshes for "
- "pool entry %s — treating as unrecoverable and allowing "
- "fallback to activate.", state["refreshes"] - 1, refreshed_id,
- )
- return False, has_retried_429
- log.info("Credential auth failure — refreshed pool entry %s", refreshed_id or "?")
- if agent._swap_credential(refreshed) is False:
- return False, has_retried_429
- return True, has_retried_429
|