credential_auth_recovery.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165
  1. """Credential-pool recovery for an auth failure (401/403) on the dispatched credential.
  2. Nous answers bursts of ``401 {"error": "authentication required"}`` for agent keys that are
  3. still unexpired and valid a few seconds later. Re-minting cannot fix a server-side blip, and
  4. each forced refresh redeems the single-use refresh token, so a Nous 401 episode gets exactly
  5. ONE forced refresh. Any later 401 in the same turn (or a refresh that handed back the bearer
  6. that just failed) waits out ``TRANSIENT_AUTH_BACKOFF_S`` and retries with the store's current
  7. key, adopting a peer's rotation but never POSTing again. A dead grant (refresh rejected)
  8. still fails fast through rotation, and so does an entitlement 403.
  9. """
  10. from __future__ import annotations
  11. import time
  12. from typing import Any, Optional
  13. # Waits before each retry of an unexpired Nous key that keeps answering 401 (~50 s in total).
  14. TRANSIENT_AUTH_BACKOFF_S = (5.0, 15.0, 30.0)
  15. # Cap same-entry OAuth refreshes on a persistent auth failure, else a single-entry pool re-mints forever.
  16. _MAX_AUTH_REFRESH_ATTEMPTS = 2
  17. _ACTIVITY_TOUCH_S = 30.0
  18. # A 401 this long after the episode's last one starts a new episode (one more refresh allowed):
  19. # hour-long turns must still recover from a key revoked later in the same turn.
  20. _EPISODE_IDLE_S = 120.0
  21. def _ra():
  22. import run_agent
  23. return run_agent
  24. def _episode(agent: Any, provider: str, entry_id: Optional[str]) -> dict[str, Any]:
  25. """Per-turn auth-failure state for one pool entry (the turn loop resets the map)."""
  26. if getattr(agent, "_auth_pool_refresh_counts", None) is None:
  27. agent._auth_pool_refresh_counts = {}
  28. now = time.monotonic()
  29. episode = agent._auth_pool_refresh_counts.get((provider, entry_id))
  30. if episode is None or now - episode["last"] > _EPISODE_IDLE_S:
  31. episode = agent._auth_pool_refresh_counts[(provider, entry_id)] = {"refreshes": 0, "waits": 0}
  32. episode["last"] = now
  33. return episode
  34. def nous_pool_refreshed_this_turn(agent: Any) -> bool:
  35. """True once the pool spent this turn's forced Nous refresh (the singleton path must not re-POST)."""
  36. counts = getattr(agent, "_auth_pool_refresh_counts", None) or {}
  37. return any(key[0] == "nous" and state.get("refreshes") for key, state in counts.items())
  38. def _interruptible_wait(agent: Any, seconds: float) -> bool:
  39. """Sleep in 200 ms slices; False when the user interrupted."""
  40. deadline = time.monotonic() + seconds
  41. next_touch = time.monotonic() + _ACTIVITY_TOUCH_S
  42. while time.monotonic() < deadline:
  43. if getattr(agent, "_interrupt_requested", False):
  44. return False
  45. time.sleep(min(0.2, max(0.0, deadline - time.monotonic())))
  46. if time.monotonic() >= next_touch:
  47. agent._touch_activity("waiting out a Nous 401 on a valid key")
  48. next_touch = time.monotonic() + _ACTIVITY_TOUCH_S
  49. return True
  50. def _entry_for(pool: Any, credential_id: Optional[str], api_key_hint: Optional[str]) -> Any:
  51. entries = pool.entries()
  52. if credential_id:
  53. found = next((e for e in entries if e.id == credential_id), None)
  54. if found is not None:
  55. return found
  56. return next((e for e in entries if api_key_hint and e.runtime_api_key == api_key_hint), None)
  57. def _retry_after_backoff(agent: Any, pool: Any, episode: dict[str, Any], *, failed_key: Optional[str],
  58. entry_id: Optional[str], credential_id: Optional[str], has_retried_429: bool):
  59. """Wait out a server-side 401 on an unexpired key, then retry with the store's current key."""
  60. log = _ra().logger
  61. step = episode["waits"]
  62. if step >= len(TRANSIENT_AUTH_BACKOFF_S):
  63. log.warning(
  64. "Credential auth failure persists for pool entry %s after one refresh and %s "
  65. "backoff retries — treating as unrecoverable and allowing fallback to activate.",
  66. entry_id, step)
  67. return False, has_retried_429
  68. wait = TRANSIENT_AUTH_BACKOFF_S[step]
  69. episode["waits"] = step + 1
  70. log.info(
  71. "Credential auth failure on an unexpired key for pool entry %s — transient 401, "
  72. "retrying in %.0fs (backoff %s/%s, no further refresh)",
  73. entry_id, wait, step + 1, len(TRANSIENT_AUTH_BACKOFF_S))
  74. agent._vprint(
  75. f"{agent.log_prefix}🔐 Nous rejected a still-valid key (transient); retrying in {wait:.0f}s...",
  76. force=True, diagnostic=True)
  77. if not _interruptible_wait(agent, wait):
  78. # Not recovered: the caller's interrupt check then aborts (or redirects) the turn. A
  79. # True here re-sends the request, and the retry loop never re-checks the interrupt.
  80. return False, has_retried_429
  81. # A peer process may have rotated meanwhile: adopt what the store holds now (no POST).
  82. entry = _entry_for(pool, credential_id or entry_id, failed_key)
  83. if entry is not None:
  84. if agent._swap_credential(pool._sync_nous_entry_from_auth_store(entry)) is False:
  85. return False, has_retried_429
  86. return True, has_retried_429
  87. def recover_auth_failure(agent, pool, *, status_code, has_retried_429, error_context, api_key_hint,
  88. credential_id, rotate_and_swap):
  89. from agent.agent_runtime_helpers import _is_entitlement_403
  90. log = _ra().logger
  91. if _is_entitlement_403(agent, status_code, error_context):
  92. log.info(
  93. "Credential %s — entitlement-shaped 403 from %s; "
  94. "skipping pool refresh (account lacks subscription, "
  95. "not a transient auth failure).", status_code if status_code is not None else "auth",
  96. agent.provider or "provider",
  97. )
  98. return False, has_retried_429
  99. provider = agent.provider
  100. # Only a 401 is the "valid key refused" shape worth backing off on; a non-entitlement 403
  101. # keeps the capped-refresh path and fails fast.
  102. nous = provider == "nous" and status_code == 401
  103. entry_id = credential_id
  104. if nous:
  105. found = _entry_for(pool, credential_id, api_key_hint)
  106. entry_id = getattr(found, "id", None) or credential_id
  107. episode = _episode(agent, provider, entry_id)
  108. if episode["refreshes"] >= 1:
  109. # The one forced refresh of this episode already happened: never re-redeem the grant.
  110. return _retry_after_backoff(
  111. agent, pool, episode, failed_key=api_key_hint, entry_id=entry_id,
  112. credential_id=credential_id, has_retried_429=has_retried_429)
  113. # Refresh the entry that supplied the failing key, not current(): refreshing a healthy entry
  114. # burns its single-use refresh token for a failure it never had.
  115. refresh_kwargs = {"api_key_hint": api_key_hint}
  116. if credential_id:
  117. refresh_kwargs["credential_id"] = credential_id
  118. refreshed = pool.try_refresh_matching(**refresh_kwargs)
  119. if refreshed is None:
  120. # Refresh failed; rotate (the failed entry is already marked exhausted).
  121. return (True, False) if rotate_and_swap(401, "auth refresh failed") else (False, has_retried_429)
  122. refreshed_id = getattr(refreshed, "id", None)
  123. if nous:
  124. episode = _episode(agent, provider, refreshed_id or entry_id)
  125. episode["refreshes"] += 1
  126. if api_key_hint and getattr(refreshed, "runtime_api_key", None) == api_key_hint:
  127. # The "refresh" handed back the bearer that just failed: replaying it at once is
  128. # the 1.5 s give-up; it is a valid key the server is refusing, so back off instead.
  129. return _retry_after_backoff(
  130. agent, pool, episode, failed_key=api_key_hint, entry_id=refreshed_id,
  131. credential_id=credential_id, has_retried_429=has_retried_429)
  132. elif refreshed_id is not None:
  133. # try_refresh_matching() reports success even when upstream keeps rejecting; cap same-entry
  134. # refreshes so a single-entry pool falls through to fallback.
  135. state = _episode(agent, provider, refreshed_id)
  136. state["refreshes"] += 1
  137. if state["refreshes"] > _MAX_AUTH_REFRESH_ATTEMPTS:
  138. log.warning(
  139. "Credential auth failure persists after %s refreshes for "
  140. "pool entry %s — treating as unrecoverable and allowing "
  141. "fallback to activate.", state["refreshes"] - 1, refreshed_id,
  142. )
  143. return False, has_retried_429
  144. log.info("Credential auth failure — refreshed pool entry %s", refreshed_id or "?")
  145. if agent._swap_credential(refreshed) is False:
  146. return False, has_retried_429
  147. return True, has_retried_429