credential_pool.py 147 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051
  1. """Persistent multi-credential pool for same-provider failover."""
  2. from __future__ import annotations
  3. from agent.credential_pool_admin import CredentialPoolAdminMixin
  4. from agent.credential_pool_model_cooldowns import CredentialPoolModelCooldownMixin, model_cooldown_until
  5. from agent.credential_pool_timestamps import _parse_absolute_timestamp
  6. import logging
  7. import os
  8. import random
  9. import threading
  10. import time
  11. import uuid
  12. import re
  13. from dataclasses import dataclass, fields, replace
  14. from datetime import datetime, timezone, UTC
  15. from pathlib import Path
  16. from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple
  17. from hermes_constants import OPENROUTER_BASE_URL
  18. from hermes_cli.config import load_env
  19. from agent.secret_scope import get_secret as _get_secret, get_secret_str
  20. from agent.retry_utils import reset_delay_from_message
  21. from hermes_cli.auth_plugin_providers import plugin_refresh_hook
  22. from agent.credential_pool_plugin import apply_plugin_refresh_result, plugin_row_is_expiring, recover_failed_plugin_refresh
  23. from agent.credential_persistence import (
  24. fingerprint_secret_value,
  25. is_borrowed_credential_source,
  26. sanitize_borrowed_credential_payload,
  27. )
  28. import hermes_cli.auth as auth_mod
  29. from hermes_cli.auth_oauth_grants import owned_profile_reads_root_state
  30. from hermes_cli.auth import (
  31. CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
  32. PROVIDER_REGISTRY,
  33. SINGLE_USE_REFRESH_POOL_PROVIDERS,
  34. _auth_store_lock,
  35. _codex_access_token_is_expiring,
  36. _decode_jwt_claims,
  37. _global_auth_file_path,
  38. _load_auth_store,
  39. _load_provider_state,
  40. _load_provider_state_with_source,
  41. _resolve_kimi_base_url,
  42. _resolve_zai_base_url,
  43. _same_path,
  44. _save_auth_store,
  45. _save_provider_state,
  46. _store_provider_state,
  47. read_credential_pool,
  48. write_credential_pool,
  49. )
  50. logger = logging.getLogger(__name__)
  51. def _load_config_safe() -> Optional[dict]:
  52. """Read-only config (``load_config_readonly`` skips the per-row deepcopy), None on any error.
  53. Inside ``frozen_runtime_config`` it is the session's frozen config: a key rotated in config.yaml
  54. must never seed a frozen session's pool."""
  55. try:
  56. from hermes_cli.config import load_config_readonly
  57. from hermes_cli.runtime_provider import _FROZEN_RUNTIME_CONFIG
  58. return _FROZEN_RUNTIME_CONFIG.get() or load_config_readonly()
  59. except Exception:
  60. return None
  61. def _is_source_suppressed_fn() -> Callable[[str, str], bool]:
  62. """``hermes_cli.auth.is_source_suppressed`` (late-bound), or an always-False stub."""
  63. try:
  64. from hermes_cli.auth import is_source_suppressed
  65. return is_source_suppressed
  66. except ImportError:
  67. return lambda _p, _s: False
  68. # --- Status and type constants ---
  69. STATUS_OK = "ok"
  70. STATUS_EXHAUSTED = "exhausted"
  71. # Terminal failure — the credential will never recover on its own (upstream
  72. # ``token_invalidated`` / ``token_revoked``). DEAD entries are excluded from
  73. # rotation unconditionally and only clear when an explicit write-side sync
  74. # (e.g. ``_save_codex_tokens`` after a fresh device-code login) rewrites tokens.
  75. STATUS_DEAD = "dead"
  76. # OAuth error reasons that mean the credential is permanently invalid
  77. # server-side (OpenAI Codex, Anthropic, xAI, Google OAuth, RFC 6749/6750/7009).
  78. _TERMINAL_AUTH_REASONS = frozenset({
  79. "token_invalidated",
  80. "token_revoked",
  81. "invalid_token",
  82. "invalid_grant",
  83. "unauthorized_client",
  84. "refresh_token_reused", # single-use refresh token consumed by another process
  85. })
  86. # Locally generated terminal reason (no HTTP status): a refresh POST rotated a
  87. # single-use pair but the replacement never reached its authoritative store, so
  88. # the pre-rotation token still on disk is already spent. Kept out of
  89. # _TERMINAL_AUTH_REASONS (upstream 401 reasons) and handled explicitly.
  90. CREDENTIAL_PERSIST_FAILED_REASON = "credential_persist_failed"
  91. # DEAD ``manual:*`` entries are pruned after this quiet window — they have no
  92. # singleton to re-seed from and the user can re-add via ``hermes auth add``.
  93. # Singleton-seeded entries (device_code, claude_code) are NOT pruned because
  94. # ``_seed_from_singletons`` would re-create them from the same stale tokens.
  95. DEAD_MANUAL_PRUNE_TTL_SECONDS = 24 * 60 * 60
  96. AUTH_TYPE_OAUTH = "oauth"
  97. AUTH_TYPE_API_KEY = "api_key"
  98. SOURCE_MANUAL = "manual"
  99. SOURCE_MANUAL_DEVICE_CODE = f"{SOURCE_MANUAL}:device_code"
  100. STRATEGY_FILL_FIRST = "fill_first"
  101. STRATEGY_ROUND_ROBIN = "round_robin"
  102. STRATEGY_RANDOM = "random"
  103. STRATEGY_LEAST_USED = "least_used"
  104. SUPPORTED_POOL_STRATEGIES = {
  105. STRATEGY_FILL_FIRST,
  106. STRATEGY_ROUND_ROBIN,
  107. STRATEGY_RANDOM,
  108. STRATEGY_LEAST_USED,
  109. }
  110. # Cooldowns before retrying an exhausted credential. Transient 401s cool down
  111. # briefly so single-key setups recover; 429/402/other take an hour.
  112. # Provider-supplied reset_at timestamps override these defaults.
  113. EXHAUSTED_TTL_401_SECONDS = 5 * 60
  114. EXHAUSTED_TTL_429_SECONDS = 60 * 60
  115. EXHAUSTED_TTL_DEFAULT_SECONDS = 60 * 60
  116. # When the offending key is the sole non-DEAD entry, an hour-long bench means
  117. # an hour of hard failures. Throttles (429/403/5xx) reset in seconds, so a sole
  118. # credential cools down briefly instead.
  119. EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS = 60
  120. # ``FailoverReason.billing`` as a bare string: the pool persists classified
  121. # failure semantics to JSON and must not import the classifier.
  122. FAILURE_REASON_BILLING = "billing"
  123. # Billing verdict resting on an ambiguous body (#82154): Anthropic's "out of
  124. # extra usage" 400 is returned both for genuine overage and for a server-side
  125. # content-filter rejection, which leaves the credential healthy. Unverified
  126. # billing gets the short transient cooldown; genuine depletion re-latches.
  127. FAILURE_REASON_BILLING_UNVERIFIED = "billing_unverified"
  128. # Throttle window for the "no available entries" INFO line. Selection runs on
  129. # every model call; on Windows several processes share one rotating log behind
  130. # a cross-process lock, and per-selection logging stormed that lock, pegged a
  131. # core, and stalled the event loop (Desktop backend readiness timeouts).
  132. # Credential selection runs on a hot path (every model call, plus auxiliary tasks like
  133. # compression/moa/titles), so when a pool is empty or fully exhausted the un-throttled log fires on *every*
  134. # selection. On Windows several Hermes processes share one rotating log guarded by concurrent-log-handler's
  135. # cross-process lock; that per-selection volume storms the lock (``RuntimeError: Cannot acquire lock after
  136. # 20 attempts``), pegs a core, and stalls the asyncio event loop long enough to fail the Desktop backend
  137. # readiness handshake ("Timed out connecting to Hermes backend after 15000ms"). Logging the condition at
  138. # most once per window preserves the signal while removing the storm — same class of fix as the warn-once
  139. # dedup in #58265.
  140. NO_AVAILABLE_ENTRIES_LOG_THROTTLE_SECONDS = 60.0
  141. # Pool key prefix for custom OpenAI-compatible endpoints: all share
  142. # provider='custom' but are keyed 'custom:<normalized_name>'.
  143. CUSTOM_POOL_PREFIX = "custom:"
  144. # Fields only round-tripped through JSON — never used for logic as attributes.
  145. _EXTRA_KEYS = frozenset({
  146. "token_type", "scope", "client_id", "portal_base_url", "obtained_at",
  147. "expires_in", "agent_key_id", "agent_key_expires_in", "agent_key_reused",
  148. "agent_key_obtained_at", "tls", "secret_source", "secret_fingerprint",
  149. # Nous guest identity (``auth_method: anonymous``): the anon_ credential is the refresh material.
  150. "auth_method", "account_tier", "anon_token", "user_id", "org_id",
  151. # Classified failure semantics for the last exhaustion (agent/error_classifier.py).
  152. # Providers return 403 for both an edge throttle and a spending limit, so the
  153. # raw status cannot size a cooldown; persisted so a restart doesn't downgrade
  154. # a billing bench to a 60s transient cooldown.
  155. "failure_reason",
  156. })
  157. # Nous singleton metadata mirrored between auth.json state and ``entry.extra``.
  158. _NOUS_EXTRA_STATE_KEYS = (
  159. "obtained_at", "expires_in", "agent_key_id",
  160. "agent_key_expires_in", "agent_key_reused", "agent_key_obtained_at",
  161. "auth_method", "account_tier", "anon_token", "user_id", "org_id",
  162. )
  163. # ``replace(entry, **_CLEAR_STATUS)`` returns an entry with no error state.
  164. _CLEAR_STATUS: dict[str, Any] = {
  165. "last_status": None,
  166. "last_status_at": None,
  167. "last_error_code": None,
  168. "last_error_reason": None,
  169. "last_error_message": None,
  170. "last_error_reset_at": None,
  171. }
  172. _MARK_OK: dict[str, Any] = {**_CLEAR_STATUS, "last_status": STATUS_OK}
  173. def _normalize_pool_auth_type(provider: str, token: Any, auth_type: Any) -> str:
  174. """Infer pool auth metadata for token formats with one unambiguous meaning."""
  175. if provider == "anthropic" and isinstance(token, str) and token.startswith("sk-ant-oat"):
  176. return AUTH_TYPE_OAUTH
  177. return str(auth_type or AUTH_TYPE_API_KEY)
  178. @dataclass
  179. class PooledCredential:
  180. provider: str
  181. id: str
  182. label: str
  183. auth_type: str
  184. priority: int
  185. source: str
  186. access_token: str
  187. refresh_token: Optional[str] = None
  188. last_status: Optional[str] = None
  189. last_status_at: Optional[float] = None
  190. last_error_code: Optional[int] = None
  191. last_error_reason: Optional[str] = None
  192. last_error_message: Optional[str] = None
  193. last_error_reset_at: Optional[float] = None
  194. # Epoch of the last deliberate ``hermes auth reset`` of this entry. Sticky: a later exhaustion
  195. # stamps a newer ``last_status_at``, so "reset postdates status" stays decidable across processes.
  196. status_cleared_at: Optional[float] = None
  197. base_url: Optional[str] = None
  198. expires_at: Optional[str] = None
  199. expires_at_ms: Optional[int] = None
  200. last_refresh: Optional[str] = None
  201. inference_base_url: Optional[str] = None
  202. agent_key: Optional[str] = None
  203. agent_key_expires_at: Optional[str] = None
  204. request_count: int = 0
  205. # A provider may rate-limit one model while the same credential remains
  206. # usable for its sibling models. Keep that observation separate from the
  207. # credential-wide status used for auth and billing failures.
  208. model_cooldowns: Optional[dict[str, float]] = None
  209. extra: dict[str, Any] = None # type: ignore[assignment]
  210. def __post_init__(self):
  211. if self.extra is None:
  212. self.extra = {}
  213. self.auth_type = _normalize_pool_auth_type(self.provider, self.access_token, self.auth_type)
  214. def __getattr__(self, name: str):
  215. if name in _EXTRA_KEYS:
  216. return self.extra.get(name)
  217. raise AttributeError(f"'{type(self).__name__}' object has no attribute {name!r}")
  218. @classmethod
  219. def from_dict(cls, provider: str, payload: dict[str, Any]) -> PooledCredential:
  220. field_names = {f.name for f in fields(cls) if f.name != "provider"}
  221. data = {k: payload.get(k) for k in field_names if k in payload}
  222. # Rehydrated last_status_at may be an ISO string from to_dict() — normalize to float epoch
  223. if isinstance(data.get("last_status_at"), str):
  224. data["last_status_at"] = _parse_absolute_timestamp(data["last_status_at"])
  225. # Every non-field key rides in ``extra`` (to_dict writes them all back), so metadata a plugin
  226. # stores on its own rows survives load -> save -> load. ``_EXTRA_KEYS`` stays the attribute
  227. # surface for core logic; unknown keys are opaque payload. ``provider`` is the row's owner
  228. # (excluded from ``field_names`` above), never metadata — sweeping it in would write a
  229. # stray provider name back over the row on to_dict().
  230. data["extra"] = {
  231. k: v for k, v in payload.items() if k not in field_names and k != "provider" and v is not None
  232. }
  233. data.setdefault("id", uuid.uuid4().hex[:6])
  234. data.setdefault("label", payload.get("source", provider))
  235. data.setdefault("auth_type", AUTH_TYPE_API_KEY)
  236. data.setdefault("priority", 0)
  237. data.setdefault("source", SOURCE_MANUAL)
  238. data.setdefault("access_token", "")
  239. return cls(provider=provider, **data)
  240. def to_dict(self) -> dict[str, Any]:
  241. result: dict[str, Any] = {}
  242. for field_def in fields(self):
  243. if field_def.name in {"provider", "extra"}:
  244. continue
  245. value = getattr(self, field_def.name)
  246. if value is not None or field_def.name in _CLEAR_STATUS:
  247. result[field_def.name] = value
  248. for k, v in self.extra.items():
  249. if v is not None:
  250. result[k] = v
  251. return sanitize_borrowed_credential_payload(result, self.provider)
  252. @property
  253. def runtime_api_key(self) -> str:
  254. if self.provider == "nous":
  255. # Nous stores the runtime inference credential in agent_key for
  256. # compatibility. It must be a NAS invoke JWT.
  257. for token, expires_at in (
  258. (self.agent_key, self.agent_key_expires_at),
  259. (self.access_token, self.expires_at),
  260. ):
  261. if (
  262. isinstance(token, str)
  263. and token.strip()
  264. and auth_mod._nous_invoke_jwt_is_usable(
  265. token, scope=getattr(self, "scope", None), expires_at=expires_at,
  266. )
  267. ):
  268. return token.strip()
  269. return ""
  270. return str(self.access_token or "")
  271. @property
  272. def runtime_base_url(self) -> Optional[str]:
  273. if self.provider == "nous":
  274. return self.inference_base_url or self.base_url
  275. if self.provider == "openai-codex":
  276. # Pool rows keep the canonical ChatGPT URL; the profile-scoped proxy override must win
  277. # for every reader of the row — initial resolution AND a 401/429 rotation
  278. # (client_lifecycle._swap_credential), or a rotation silently leaves the proxy.
  279. return get_secret_str("HERMES_CODEX_BASE_URL", "").strip().rstrip("/") or self.base_url
  280. return self.base_url
  281. def label_from_token(token: str, fallback: str) -> str:
  282. claims = _decode_jwt_claims(token)
  283. for key in ("email", "preferred_username", "upn"):
  284. value = claims.get(key)
  285. if isinstance(value, str) and value.strip():
  286. return value.strip()
  287. return fallback
  288. def _codex_principal_identity(access_token: Any) -> Optional[tuple[str, str]]:
  289. """``(chatgpt_account_id, sub)`` of a Codex access token, or None when either claim is missing.
  290. Decoded without signature verification: this only decides whether two credentials Hermes
  291. already holds belong to the same principal, never whether a token is valid. Both claims are
  292. required because members of one ChatGPT workspace share ``chatgpt_account_id`` yet have their
  293. own subjects and quotas.
  294. """
  295. claims = _decode_jwt_claims(access_token)
  296. auth_claims = claims.get("https://api.openai.com/auth") if isinstance(claims, dict) else None
  297. account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None
  298. subject = claims.get("sub") if isinstance(claims, dict) else None
  299. if not (isinstance(account_id, str) and account_id.strip() and isinstance(subject, str) and subject.strip()):
  300. return None
  301. return account_id.strip(), subject.strip()
  302. def _codex_entry_tracks_singleton(entry: PooledCredential, singleton_tokens: dict[str, Any]) -> bool:
  303. """Whether a Codex pool entry may adopt the auth.json singleton's token pair.
  304. ``device_code`` IS the singleton. ``manual:device_code`` is ambiguous: a legacy alias of the
  305. singleton (same account, must follow its rotations) or an independent account added with
  306. ``hermes auth add openai-codex`` (must never be overwritten — adopting turned two logins into
  307. one account, both hitting the same usage limit). Same principal proves the alias; unknown
  308. identity fails closed.
  309. """
  310. if entry.source == "device_code":
  311. return True
  312. if entry.source != SOURCE_MANUAL_DEVICE_CODE:
  313. return False
  314. entry_identity = _codex_principal_identity(entry.access_token)
  315. return entry_identity is not None and entry_identity == _codex_principal_identity(singleton_tokens.get("access_token"))
  316. def _next_priority(entries: list[PooledCredential]) -> int:
  317. return max((entry.priority for entry in entries), default=-1) + 1
  318. def _is_manual_source(source: str) -> bool:
  319. normalized = (source or "").strip().lower()
  320. return normalized == SOURCE_MANUAL or normalized.startswith(f"{SOURCE_MANUAL}:")
  321. def _exhausted_ttl(
  322. error_code: Optional[int],
  323. *,
  324. sole_credential: bool = False,
  325. failure_reason: Optional[str] = None,
  326. ) -> int:
  327. """Return cooldown seconds based on the HTTP status that caused exhaustion.
  328. *sole_credential*: the pool has nothing to rotate to, so transient
  329. throttles (429 and the catch-all default covering 403/5xx/unknown) are
  330. capped to a brief cooldown; 401 keeps its own already-short TTL.
  331. *failure_reason* is the classifier verdict: an OpenRouter ``key limit
  332. exceeded`` and an xAI spending block both arrive as 403 but are billing,
  333. and a 60s retry on a spent account just re-fails. Billing keeps the full
  334. bench regardless of status; 402 is billing by definition.
  335. Unverified billing (#82154) gets the short cooldown regardless of pool
  336. size (the credential may be healthy), unless the status is a true 402.
  337. """
  338. if error_code == 401:
  339. return EXHAUSTED_TTL_401_SECONDS
  340. base = EXHAUSTED_TTL_429_SECONDS if error_code == 429 else EXHAUSTED_TTL_DEFAULT_SECONDS
  341. if failure_reason == FAILURE_REASON_BILLING_UNVERIFIED and error_code != 402:
  342. return min(base, EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS)
  343. is_billing = error_code == 402 or failure_reason == FAILURE_REASON_BILLING
  344. if sole_credential and not is_billing:
  345. return min(base, EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS)
  346. return base
  347. def _singleton_predates_entry(state: Any, entry: PooledCredential) -> bool:
  348. """True only when the auth.json singleton is PROVABLY older than *entry*.
  349. Both sides stamp ``last_refresh`` on every successful rotation. When
  350. either side lacks a parseable stamp this returns False (cannot prove),
  351. which keeps the historical adopt-on-difference behavior (#70111) intact
  352. for legacy writers.
  353. """
  354. entry_ts = _parse_absolute_timestamp(entry.last_refresh)
  355. if entry_ts is None:
  356. return False
  357. state_ts = _parse_absolute_timestamp(state.get("last_refresh") if isinstance(state, dict) else None)
  358. if state_ts is None:
  359. return False
  360. return state_ts < entry_ts
  361. def _normalize_error_context(error_context: Optional[dict[str, Any]]) -> dict[str, Any]:
  362. if not isinstance(error_context, dict):
  363. return {}
  364. normalized: dict[str, Any] = {}
  365. for key in ("reason", "message"):
  366. value = error_context.get(key)
  367. if isinstance(value, str) and value.strip():
  368. normalized[key] = value.strip()
  369. reset_at = (
  370. error_context.get("reset_at")
  371. or error_context.get("resets_at")
  372. or error_context.get("retry_until")
  373. )
  374. parsed_reset_at = _parse_absolute_timestamp(reset_at)
  375. message = error_context.get("message")
  376. if parsed_reset_at is None and isinstance(message, str):
  377. retry_delay_seconds = reset_delay_from_message(message)
  378. if retry_delay_seconds is not None:
  379. parsed_reset_at = time.time() + retry_delay_seconds
  380. if parsed_reset_at is not None:
  381. normalized["reset_at"] = parsed_reset_at
  382. return normalized
  383. def _exhausted_until(entry: PooledCredential, *, sole_credential: bool = False) -> Optional[float]:
  384. if entry.last_status != STATUS_EXHAUSTED:
  385. return None
  386. reset_at = _parse_absolute_timestamp(entry.last_error_reset_at)
  387. if reset_at is not None:
  388. return reset_at
  389. if entry.last_status_at:
  390. return entry.last_status_at + _exhausted_ttl(
  391. entry.last_error_code,
  392. sole_credential=sole_credential,
  393. failure_reason=entry.failure_reason,
  394. )
  395. return None
  396. # --- Custom (OpenAI-compatible) endpoint pool keys ---
  397. def _normalize_custom_pool_name(name: str) -> str:
  398. """Normalize a custom provider name for use as a pool key suffix."""
  399. return name.strip().lower().replace(" ", "-")
  400. def _norm_url(url: Any) -> str:
  401. return str(url or "").strip().rstrip("/")
  402. def _iter_custom_providers(config: Optional[dict] = None):
  403. """Yield ``(normalized_name, entry)`` from the merged custom-provider config view."""
  404. if config is None:
  405. config = _load_config_safe()
  406. if config is None:
  407. return
  408. try:
  409. from hermes_cli.config import get_compatible_custom_providers
  410. custom_providers = get_compatible_custom_providers(config)
  411. except Exception:
  412. return
  413. for entry in custom_providers or ():
  414. if isinstance(entry, dict) and isinstance(entry.get("name"), str):
  415. yield _normalize_custom_pool_name(entry["name"]), entry
  416. def _custom_entry_name_aliases(norm_name: str, entry: dict[str, Any]) -> set:
  417. aliases = {norm_name}
  418. provider_key = _normalize_custom_pool_name(str(entry.get("provider_key") or ""))
  419. if provider_key:
  420. aliases.add(provider_key)
  421. return aliases
  422. def _requested_custom_name_aliases(provider_name: str) -> set:
  423. normalized = _normalize_custom_pool_name(provider_name)
  424. aliases = {normalized} if normalized else set()
  425. if normalized.startswith(CUSTOM_POOL_PREFIX):
  426. suffix = _normalize_custom_pool_name(normalized[len(CUSTOM_POOL_PREFIX):])
  427. if suffix:
  428. aliases.add(suffix)
  429. return aliases
  430. def _pool_keys_for_custom_entry(norm_name: str, entry: dict[str, Any]) -> list[str]:
  431. """Durable ``providers.<key>`` slug first, then legacy ``custom:<name>``."""
  432. keys: list[str] = []
  433. provider_key = _normalize_custom_pool_name(str(entry.get("provider_key") or ""))
  434. for key in (provider_key, f"{CUSTOM_POOL_PREFIX}{norm_name}" if norm_name else ""):
  435. normalized = key.strip().lower()
  436. if normalized and normalized not in keys:
  437. keys.append(normalized)
  438. return keys
  439. def custom_provider_pool_key_candidates(
  440. base_url: Optional[str], provider_name: Optional[str] = None, config: Optional[dict] = None,
  441. ) -> list[str]:
  442. """Return pool keys to try for a custom endpoint: the durable ``providers.<key>`` slug
  443. (``hermes auth add <key>``), then legacy ``custom:<display-name>``, so a populated pool is not
  444. skipped for the ``no-key-required`` placeholder. ``config`` is a frozen route's config: ITS
  445. entry owns the endpoint, so a live edit of that entry's URL neither drops nor redirects the pool.
  446. """
  447. # A named pool holds credentials for its configured endpoint only: a launch-overridden base_url
  448. # must not inherit them (R2-M1), nor borrow a sibling's by URL.
  449. from agent.custom_entry_pool import custom_entry_for_endpoint
  450. match = custom_entry_for_endpoint(base_url, provider_name, config)
  451. return _pool_keys_for_custom_entry(*match) if match else []
  452. def get_custom_provider_pool_key(base_url: Optional[str], provider_name: Optional[str] = None) -> Optional[str]:
  453. """Preferred pool key for a custom provider: durable slug, else ``custom:<name>``.
  454. When provider_name is given, match by name first so two custom providers
  455. sharing a base_url keep separate keys.
  456. """
  457. candidates = custom_provider_pool_key_candidates(base_url, provider_name)
  458. return candidates[0] if candidates else None
  459. def list_custom_pool_providers() -> list[str]:
  460. """Return all 'custom:*' pool keys that have entries in auth.json."""
  461. pool_data = read_credential_pool(None)
  462. return sorted(
  463. key for key in pool_data
  464. if key.startswith(CUSTOM_POOL_PREFIX)
  465. and isinstance(pool_data.get(key), list)
  466. and pool_data[key]
  467. )
  468. def _get_custom_provider_config(pool_key: str) -> Optional[dict[str, Any]]:
  469. """Return the custom_providers config entry matching a pool key like 'custom:together.ai'."""
  470. if not pool_key.startswith(CUSTOM_POOL_PREFIX):
  471. return None
  472. suffix = pool_key[len(CUSTOM_POOL_PREFIX):]
  473. return next((entry for norm_name, entry in _iter_custom_providers() if norm_name == suffix), None)
  474. def get_pool_strategy(provider: str) -> str:
  475. """Return the configured selection strategy for a provider."""
  476. config = _load_config_safe()
  477. strategies = config.get("credential_pool_strategies") if config else None
  478. if not isinstance(strategies, dict):
  479. return STRATEGY_FILL_FIRST
  480. strategy = str(strategies.get(provider, "") or "").strip().lower()
  481. return strategy if strategy in SUPPORTED_POOL_STRATEGIES else STRATEGY_FILL_FIRST
  482. def _keyed_custom_pool_matches(
  483. pool_provider: str,
  484. provider_norm: str,
  485. base_url: Optional[str],
  486. ) -> bool:
  487. """Match a durable ``providers.<key>`` pool against runtime identities."""
  488. runtime_url = _norm_url(base_url)
  489. if not runtime_url:
  490. return False
  491. try:
  492. for normalized_name, entry in _iter_custom_providers():
  493. provider_key = _normalize_custom_pool_name(str(entry.get("provider_key") or ""))
  494. if provider_key != pool_provider:
  495. continue
  496. aliases = _custom_entry_name_aliases(normalized_name, entry)
  497. aliases.add(f"{CUSTOM_POOL_PREFIX}{normalized_name}")
  498. if provider_key:
  499. aliases.add(f"{CUSTOM_POOL_PREFIX}{provider_key}")
  500. configured_url = _norm_url(entry.get("base_url"))
  501. if provider_norm == "custom":
  502. return runtime_url == configured_url
  503. runtime_aliases = _requested_custom_name_aliases(provider_norm)
  504. return bool(runtime_aliases & aliases) and runtime_url == configured_url
  505. except Exception:
  506. return False
  507. return False
  508. def _legacy_custom_pool_matches(
  509. pool_provider: str,
  510. provider_norm: str,
  511. runtime_url: str,
  512. ) -> bool:
  513. """Match a legacy ``custom:<name>`` pool against a named runtime identity."""
  514. try:
  515. for normalized_name, entry in _iter_custom_providers():
  516. if f"{CUSTOM_POOL_PREFIX}{normalized_name}" != pool_provider:
  517. continue
  518. aliases = {normalized_name}
  519. for value in (entry.get("name"), entry.get("provider_key")):
  520. alias = _normalize_custom_pool_name(str(value or ""))
  521. if alias:
  522. aliases.add(alias)
  523. if alias.startswith(CUSTOM_POOL_PREFIX):
  524. aliases.add(alias[len(CUSTOM_POOL_PREFIX):])
  525. configured_url = _norm_url(entry.get("base_url"))
  526. runtime_aliases = {_normalize_custom_pool_name(provider_norm)}
  527. if provider_norm.startswith(CUSTOM_POOL_PREFIX):
  528. runtime_aliases.add(_normalize_custom_pool_name(provider_norm[len(CUSTOM_POOL_PREFIX):]))
  529. return bool(runtime_aliases & aliases) and runtime_url == configured_url
  530. except Exception:
  531. return False
  532. return False
  533. def credential_pool_entry_serves_endpoint(entry: Any, base_url: Any) -> bool:
  534. """Whether a pooled credential may be bound to a session running at ``base_url``. ``_swap_credential``
  535. adopts the entry's base_url too, so a same-provider entry for another endpoint (public OpenAI vs. an
  536. Azure resource) would send the session's requests — and the entry's key — to the wrong host (#68237).
  537. Entries or sessions without endpoint metadata (legacy adapters, test doubles) cannot rebind and are accepted."""
  538. if not isinstance(base_url, str) or not base_url:
  539. return True
  540. entry_url = getattr(entry, "runtime_base_url", None) or getattr(entry, "base_url", None)
  541. if not isinstance(entry_url, str) or not entry_url:
  542. return True
  543. from hermes_cli.route_identity import normalize_route_base_url
  544. return normalize_route_base_url(entry_url) == normalize_route_base_url(base_url)
  545. def credential_pool_matches_provider(
  546. pool_or_provider: Any,
  547. provider: Optional[str],
  548. *,
  549. base_url: Optional[str] = None,
  550. ) -> bool:
  551. """Return whether a pool belongs to the requested runtime provider.
  552. Named custom endpoints may use three identities: the live agent can retain
  553. the configured name/provider key, newer runtime paths normalize it to
  554. ``custom``, and the pool may be keyed as the durable ``providers.<key>``
  555. slug or as legacy ``custom:<name>``. Accept those aliases only when the
  556. runtime endpoint belongs to the same configured custom provider. Empty
  557. identities fail closed. Legacy pool adapters without a ``provider``
  558. attribute remain compatible; production pools are scoped.
  559. """
  560. raw_pool_provider = getattr(pool_or_provider, "provider", None)
  561. if raw_pool_provider is None:
  562. if not isinstance(pool_or_provider, str):
  563. # Lightweight/unscoped pool adapters (old plugins, tests) may
  564. # expose only select()/has_credentials().
  565. return True
  566. raw_pool_provider = pool_or_provider
  567. pool_provider = str(raw_pool_provider or "").strip().lower()
  568. provider_norm = str(provider or "").strip().lower()
  569. if not pool_provider or not provider_norm:
  570. return False
  571. if not pool_provider.startswith(CUSTOM_POOL_PREFIX):
  572. if pool_provider == provider_norm:
  573. return True
  574. return _keyed_custom_pool_matches(pool_provider, provider_norm, base_url)
  575. if provider_norm == "custom":
  576. try:
  577. matched_pool = get_custom_provider_pool_key(base_url or "")
  578. if str(matched_pool or "").strip().lower() == pool_provider:
  579. return True
  580. candidates = custom_provider_pool_key_candidates(base_url or "")
  581. except Exception:
  582. return False
  583. return pool_provider in {str(key).strip().lower() for key in candidates}
  584. runtime_url = _norm_url(base_url)
  585. if not runtime_url:
  586. return False
  587. return _legacy_custom_pool_matches(pool_provider, provider_norm, runtime_url)
  588. def resolve_runtime_pool_key(provider: Optional[str], base_url: Optional[str]) -> str:
  589. """Resolve the credential-pool key for a runtime provider identity.
  590. Named custom runtimes retain their configured alias while their pool may
  591. be stored under the durable ``providers.<key>`` slug or legacy
  592. ``custom:<name>``. Return that scoped key only when the canonical
  593. provider/endpoint boundary accepts it; otherwise preserve the normalized
  594. runtime identity so callers fail closed.
  595. """
  596. provider_norm = str(provider or "").strip().lower()
  597. if not provider_norm:
  598. return ""
  599. def _accepts(candidate: str) -> bool:
  600. return credential_pool_matches_provider(candidate, provider_norm, base_url=base_url)
  601. try:
  602. if provider_norm == "custom":
  603. candidate = get_custom_provider_pool_key(base_url)
  604. if candidate and _accepts(candidate):
  605. return str(candidate).strip().lower()
  606. else:
  607. # Named/exact custom runtimes are keyed by identity: search the
  608. # configured candidates by identity before endpoint so a sibling
  609. # sharing the URL cannot lend its pool.
  610. for normalized_name, entry in _iter_custom_providers():
  611. for candidate in _pool_keys_for_custom_entry(normalized_name, entry):
  612. if _accepts(candidate):
  613. return candidate
  614. except Exception:
  615. pass
  616. return provider_norm
  617. DEFAULT_MAX_CONCURRENT_PER_CREDENTIAL = 1
  618. # --- Multi-profile root write-through ---
  619. def _guarded_global_root(global_path: Optional[Path]) -> Optional[Path]:
  620. """Apply the pytest seat belt to a resolved global-root auth.json path.
  621. ``None`` means classic mode (profile == root) or "refuse": under pytest,
  622. never write the real user's ``~/.hermes/auth.json`` even when HERMES_HOME
  623. points at a profile path (mirrors the read-side guard in
  624. ``_load_global_auth_store``). Uses the unmodified HOME env, not
  625. ``Path.home()`` which fixtures may monkeypatch.
  626. """
  627. if global_path is None:
  628. return None
  629. if os.environ.get("PYTEST_CURRENT_TEST"):
  630. real_home_env = os.environ.get("HOME", "")
  631. if real_home_env:
  632. real_root = Path(real_home_env) / ".hermes" / "auth.json"
  633. try:
  634. # Comparing the guard path must not probe the real auth store.
  635. if os.path.normcase(os.path.abspath(global_path)) == os.path.normcase(os.path.abspath(real_root)):
  636. return None
  637. except Exception:
  638. return None
  639. return global_path
  640. def _write_through_provider_state_to_global_root(
  641. provider_id: str, state: dict[str, Any]
  642. ) -> None:
  643. """Persist a rotated OAuth ``state`` into the global-root auth.json.
  644. Best-effort write-through for the multi-profile rotation hazard: nous,
  645. openai-codex, and xai-oauth rotate the refresh_token on refresh, so when
  646. a profile pool refresh rotates a grant it resolved from the root fallback,
  647. the rotated chain must land back in root. Otherwise root keeps a revoked
  648. refresh token and every other profile dies with ``refresh_token_reused``
  649. / ``invalid_grant`` once its access token expires.
  650. Only updates ``providers.<provider_id>`` in the root store; never touches
  651. the profile store (the caller already saved that). Swallows all errors —
  652. a failed write-through degrades to root-stale and must never break the
  653. profile's own successful save. Mirrors
  654. ``hermes_cli.auth._write_through_xai_oauth_to_global_root``.
  655. See #48415.
  656. """
  657. try:
  658. global_path = _guarded_global_root(auth_mod._global_auth_file_path())
  659. except Exception:
  660. return
  661. if global_path is None:
  662. return
  663. try:
  664. auth_mod._persist_provider_state_to_store(provider_id, state, global_path, set_active=False)
  665. except Exception as exc: # pragma: no cover - best effort
  666. logger.debug("%s pool refresh: write-through to global root failed: %s", provider_id, exc)
  667. def _singleton_target_for_entry(pool: CredentialPool, entry: PooledCredential) -> Optional[Path]:
  668. """Root ``.anthropic_oauth.json`` when *entry* is a borrowed hermes_pkce row, else None."""
  669. if entry.source != "hermes_pkce" or entry.id not in getattr(pool, "_borrowed_root_ids", ()):
  670. return None
  671. try:
  672. from agent.anthropic_credentials import _root_hermes_oauth_file
  673. return _root_hermes_oauth_file()
  674. except Exception:
  675. return None
  676. def _store_owns_pool_provider(auth_store: dict[str, Any], provider: str) -> bool:
  677. """True when an already-loaded *auth_store* has its own rows for *provider*."""
  678. pool = auth_store.get("credential_pool")
  679. entries = pool.get(provider) if isinstance(pool, dict) else None
  680. return isinstance(entries, list) and bool(entries)
  681. def _profile_owns_pool_provider(provider: str) -> bool:
  682. """True when the ACTIVE auth.json has its own rows for *provider*.
  683. Named profiles with no local rows read the provider through the
  684. ``read_credential_pool`` global-root fallback ("borrowing").
  685. """
  686. # Classic mode (profile == root) has no root fallback, so the answer is always "owns";
  687. # skip the per-call auth.json re-read on this hot load_pool path.
  688. if auth_mod._global_auth_file_path() is None:
  689. return True
  690. try:
  691. auth_store = _load_auth_store()
  692. except Exception:
  693. return True # unreadable store: assume ownership, keep legacy path
  694. return _store_owns_pool_provider(auth_store, provider)
  695. def _borrowed_single_use_pool_root() -> Optional[Path]:
  696. """Global-root auth.json when persisting a BORROWED single-use pool, else None.
  697. ``None`` means "persist to the active store as usual": classic mode
  698. (profile == root), or the profile owns its own rows for this provider.
  699. """
  700. try:
  701. return _guarded_global_root(_global_auth_file_path())
  702. except Exception:
  703. return None
  704. def _update_root_pool_rows(
  705. provider: str, payloads: list[dict[str, Any]], global_path: Path,
  706. *, status_cleared_ids: Optional[Iterable[str]] = None,
  707. token_bases: Optional[dict[str, tuple[Any, Any]]] = None,
  708. ) -> list[dict[str, Any]]:
  709. """UPDATE-ONLY merge of *payloads* into the root store's rows for *provider*.
  710. A borrower may refresh the root's rows (rotation, cooldown state) but
  711. never add or delete them — the root owns their lifecycle. In particular a
  712. profile's singleton-prune (it has no ``.anthropic_oauth.json`` of its own)
  713. must not delete the root grant, so ``removed_ids`` is ignored by callers.
  714. """
  715. with _auth_store_lock(target_path=global_path):
  716. store = _load_auth_store(global_path)
  717. pool = store.get("credential_pool")
  718. if not isinstance(pool, dict):
  719. pool = {}
  720. store["credential_pool"] = pool
  721. existing = pool.get(provider)
  722. existing_list = existing if isinstance(existing, list) else []
  723. incoming_by_id = auth_mod._entry_ids(payloads)
  724. cleared = {cid for cid in (status_cleared_ids or ()) if cid}
  725. bases = token_bases or {}
  726. merged: list[dict[str, Any]] = []
  727. changed = False
  728. for disk_entry in existing_list:
  729. did = disk_entry.get("id") if isinstance(disk_entry, dict) else None
  730. incoming = incoming_by_id.get(did) if did else None
  731. if incoming is None:
  732. merged.append(disk_entry)
  733. continue
  734. updated = auth_mod._merge_pool_row_generation(
  735. incoming, disk_entry, provider,
  736. base_pair=bases.get(did), status_cleared=did in cleared,
  737. )
  738. if updated != disk_entry:
  739. changed = True
  740. merged.append(updated)
  741. if changed:
  742. pool[provider] = merged
  743. _save_auth_store(store, target_path=global_path)
  744. return merged
  745. def persist_pool_entries(
  746. provider: str,
  747. payloads: list[dict[str, Any]],
  748. *,
  749. removed_ids: Optional[Iterable[str]] = None,
  750. status_cleared_ids: Optional[Iterable[str]] = None,
  751. token_bases: Optional[dict[str, tuple[Any, Any]]] = None,
  752. ) -> Optional[list[dict[str, Any]]]:
  753. """Persist a provider's pool rows to the store that OWNS them.
  754. A named profile that sees a single-use-refresh provider (see
  755. ``SINGLE_USE_REFRESH_POOL_PROVIDERS``) only through the global-root fallback must not
  756. materialize a local ``credential_pool.<provider>`` copy: that copy forks
  757. the single-use refresh token, the first profile to rotate commits the new
  758. pair only to its own file, and root plus every sibling die with
  759. ``invalid_grant`` (#100339). Such rows are written back to the root store
  760. (under the root lock); everything else goes to the active store.
  761. """
  762. if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS and not _profile_owns_pool_provider(provider):
  763. global_path = _borrowed_single_use_pool_root()
  764. if global_path is not None:
  765. try:
  766. return _update_root_pool_rows(
  767. provider, payloads, global_path,
  768. status_cleared_ids=status_cleared_ids, token_bases=token_bases,
  769. )
  770. except Exception as exc:
  771. # Fail closed on the FORK, not on the save: never fall back to
  772. # writing a local copy (that IS the bug). The in-memory pool
  773. # still holds the rotated pair for this process.
  774. logger.warning(
  775. "%s pool: write-through of borrowed root grant failed (%s); "
  776. "not materializing a profile-local copy",
  777. provider, exc,
  778. )
  779. return None
  780. return write_credential_pool(
  781. provider, payloads, removed_ids=removed_ids, status_cleared_ids=status_cleared_ids,
  782. token_bases=token_bases,
  783. )
  784. # --- Per-provider singleton refresh plumbing -------------------------------
  785. #
  786. # Providers whose OAuth singleton lives in auth.json ``providers.<id>.tokens``
  787. # (Codex, xAI): log names (sync-message form, "<name> OAuth" form),
  788. # ``hermes_cli.auth`` refresh function and terminal-error predicate (looked
  789. # up at call time so tests can patch them).
  790. _TOKENS_SINGLETON_PROVIDERS: dict[str, tuple[str, str, str, str]] = {
  791. "openai-codex": ("Codex", "Codex", "refresh_codex_oauth_pure", "_is_terminal_codex_oauth_refresh_error"),
  792. "xai-oauth": ("xAI OAuth", "xAI", "refresh_xai_oauth_pure", "_is_terminal_xai_oauth_refresh_error"),
  793. }
  794. # Built-in providers whose pooled OAuth entries ``_refresh_entry_impl`` can actually refresh. Plugin
  795. # providers are refreshable when their profile ships ``refresh_credential`` (see
  796. # ``hermes_cli.auth_plugin_providers.is_refreshable_oauth_provider``); any other provider is returned
  797. # unchanged by that path, so callers must not report a refresh for them.
  798. REFRESHABLE_OAUTH_PROVIDERS = frozenset({"anthropic", "nous", *_TOKENS_SINGLETON_PROVIDERS})
  799. # Providers whose refresh tokens are single-use: the sync -> POST -> write-back
  800. # sequence must be serialized across processes under the auth-store flock.
  801. # ``nous`` is deliberately absent even though it is in SINGLE_USE_REFRESH_POOL_PROVIDERS:
  802. # its refresh path serializes on its own auth-store lock (``_refresh_entry_impl`` nous branch).
  803. _SINGLE_USE_REFRESH_PROVIDERS = ("openai-codex", "xai-oauth", "anthropic")
  804. # Lock-free window between consecutive auth-store holds in a deferred refresh sweep
  805. # (_refresh_pending_entries): a waiter with a shorter timeout (Desktop assistant start,
  806. # AUTH_LOCK_TIMEOUT_SECONDS = 15s) can acquire between two entries' holds instead of
  807. # starving behind the whole chain of single-use refreshes (#124533).
  808. _REFRESH_SWEEP_SPACING_SECONDS = 0.5
  809. _REFRESH_TIMEOUT_ENV_VARS = {
  810. "openai-codex": "HERMES_CODEX_REFRESH_TIMEOUT_SECONDS",
  811. "xai-oauth": "HERMES_XAI_REFRESH_TIMEOUT_SECONDS",
  812. }
  813. # Singleton-seeded source whose exhausted/DEAD pool row may be revived by a
  814. # re-auth another process wrote to the provider's store.
  815. _RESYNC_SOURCE = {
  816. "anthropic": "claude_code",
  817. "nous": "device_code",
  818. "openai-codex": "device_code",
  819. "xai-oauth": "device_code",
  820. }
  821. class _RefreshDone(Exception):
  822. """Raised inside a provider refresher to short-circuit ``_refresh_entry_impl`` with ``result``."""
  823. def __init__(self, result: Optional[PooledCredential]):
  824. super().__init__()
  825. self.result = result
  826. class CredentialPool(CredentialPoolAdminMixin, CredentialPoolModelCooldownMixin):
  827. def __init__(self, provider: str, entries: list[PooledCredential]):
  828. self.provider = provider
  829. self._entries = sorted(entries, key=lambda entry: entry.priority)
  830. self._current_id: Optional[str] = None
  831. # Ids of rows read via the global-root fallback (single-use OAuth
  832. # providers only); set by load_pool(), consumed by add_entry().
  833. self._borrowed_root_ids: set[str] = set()
  834. self._persisted_token_pairs: dict[str, tuple[Any, Any]] = {}
  835. self._strategy = get_pool_strategy(provider)
  836. # RLock: _replace_entry/_persist self-acquire it so the DEFERRED
  837. # single-use-token refresh path (network I/O outside the lock by
  838. # design) still serializes its pool mutations; in-lock callers
  839. # re-acquire reentrantly.
  840. self._lock = threading.RLock()
  841. self._active_leases: dict[str, int] = {}
  842. self._max_concurrent = DEFAULT_MAX_CONCURRENT_PER_CREDENTIAL
  843. # Monotonic timestamp of the last "no available entries" log (see
  844. # NO_AVAILABLE_ENTRIES_LOG_THROTTLE_SECONDS). Re-armed to None on every
  845. # successful selection so a recover->re-exhaust transition logs promptly.
  846. self._last_no_entries_log_at: Optional[float] = None
  847. # #70401: consecutive mark_exhausted_and_rotate() calls whose supplied
  848. # credential identity matched no pool entry. These mark nothing
  849. # exhausted, so without a cap the pool never converges to "no available
  850. # entries" and the caller's 401 retry loop runs unbounded. Reset when a
  851. # real entry is identified or an escape path returns None.
  852. self._unmatched_rotation_streak: int = 0
  853. # ---- read accessors ---------------------------------------------------
  854. def has_credentials(self) -> bool:
  855. with self._lock:
  856. return bool(self._entries)
  857. def has_available(self, *, model: Optional[str] = None) -> bool:
  858. """True if at least one entry is not currently in exhaustion cooldown.
  859. ``_available_entries`` is not read-only (it prunes aged-out DEAD
  860. manual entries and persists), so it must run under ``self._lock``
  861. like every other caller or a probe can race a concurrent rotation.
  862. """
  863. with self._lock:
  864. available, _pending = self._available_entries(model=model)
  865. return bool(available)
  866. def lift_reopened_cooldowns(self, *, model: Optional[str] = None) -> bool:
  867. """Clear cooldowns that have elapsed or (Codex) reopened early, then report availability.
  868. The lift ``select()`` performs, without leasing an entry. ``has_available`` and
  869. ``next_available_at`` never run the early-reopen probe.
  870. """
  871. with self._lock:
  872. available, _pending = self._available_entries(clear_expired=True, model=model)
  873. return bool(available)
  874. def next_available_at(self, *, model: Optional[str] = None) -> Optional[float]:
  875. """Earliest epoch time (seconds) any entry re-enters rotation.
  876. ``None`` when an entry is available now, or when no exhausted entry
  877. carries a usable recovery time (empty pool, or only ``STATUS_DEAD``
  878. entries). Callers must treat ``None`` as "no wait information".
  879. Runs under ``self._lock`` for the same reason as ``has_available``.
  880. """
  881. with self._lock:
  882. available, _pending = self._available_entries(model=model)
  883. if available:
  884. return None
  885. # Mirror _available_entries: a sole credential's transient throttle
  886. # cools down in seconds, and the fallback restore gate must not
  887. # wait an hour for a 60s cooldown.
  888. sole_credential = self._is_sole_credential()
  889. candidates = [
  890. until
  891. for until in (
  892. _exhausted_until(entry, sole_credential=sole_credential)
  893. for entry in self._entries
  894. if entry.last_status == STATUS_EXHAUSTED
  895. )
  896. if until is not None
  897. ]
  898. candidates.extend(
  899. until
  900. for entry in self._entries
  901. if entry.last_status != STATUS_DEAD
  902. for until in (model_cooldown_until(entry, model),)
  903. if until is not None
  904. )
  905. return min(candidates) if candidates else None
  906. def entries(self) -> list[PooledCredential]:
  907. with self._lock:
  908. return list(self._entries)
  909. def _is_sole_credential(self) -> bool:
  910. """DEAD entries never re-enter rotation, so <=1 non-DEAD entry means nothing to rotate to."""
  911. return sum(1 for e in self._entries if e.last_status != STATUS_DEAD) <= 1
  912. def _find(self, predicate: Callable[[PooledCredential], bool]) -> Optional[PooledCredential]:
  913. return next((e for e in self._entries if predicate(e)), None)
  914. def _current_unlocked(self) -> Optional[PooledCredential]:
  915. if not self._current_id:
  916. return None
  917. return self._find(lambda e: e.id == self._current_id)
  918. def current(self) -> Optional[PooledCredential]:
  919. with self._lock:
  920. return self._current_unlocked()
  921. def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]:
  922. """Stable id for the runtime credential in use.
  923. Prefer the current selection when it still supplies ``api_key_hint``;
  924. if the cursor was cleared, fall back to an unambiguous key match.
  925. """
  926. with self._lock:
  927. current = self._current_unlocked()
  928. if current is not None and (api_key_hint is None or current.runtime_api_key == api_key_hint):
  929. return current.id
  930. if api_key_hint is None:
  931. return None
  932. matches = [e for e in self._entries if e.runtime_api_key == api_key_hint]
  933. return matches[0].id if len(matches) == 1 else None
  934. # ---- mutation primitives (self-locking) --------------------------------
  935. def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None:
  936. """Swap an entry in-place by id, preserving sort order.
  937. Self-locking (RLock) so the deferred refresh path — which runs outside
  938. the pool lock — cannot tear ``self._entries`` against a concurrent
  939. select()/rotation.
  940. """
  941. with self._lock:
  942. for idx, entry in enumerate(self._entries):
  943. if entry.id == old.id:
  944. self._entries[idx] = new
  945. return
  946. def _persist(
  947. self,
  948. *,
  949. removed_ids: Optional[list[str]] = None,
  950. status_cleared_ids: Optional[list[str]] = None,
  951. ) -> None:
  952. # Self-locking: snapshotting self._entries must not race a rotation.
  953. with self._lock:
  954. payloads = [entry.to_dict() for entry in self._entries]
  955. written = persist_pool_entries(
  956. self.provider, payloads,
  957. removed_ids=removed_ids,
  958. status_cleared_ids=status_cleared_ids,
  959. token_bases=self._persisted_token_pairs,
  960. )
  961. if written is None:
  962. return
  963. rows = auth_mod._entry_ids(written)
  964. self._persisted_token_pairs = auth_mod._token_pairs_by_id(written)
  965. for entry in self._entries:
  966. row = rows.get(entry.id)
  967. pair = self._persisted_token_pairs.get(entry.id, (None, None))
  968. # Reference-only rows are intentionally secret-free on disk; never dehydrate
  969. # their live in-memory credential while adopting a concurrent generation.
  970. if row is None or not any(pair):
  971. continue
  972. # Adopt only rows the store overrode with a peer's newer pair; re-hydrating an
  973. # unchanged row would replace the live object (and pull peer cooldown state
  974. # merged into the written row) on every ordinary flush.
  975. if pair != (entry.access_token, entry.refresh_token):
  976. self._replace_entry(entry, PooledCredential.from_dict(self.provider, row))
  977. def _adopt(self, entry: PooledCredential, *, persist: bool = True, **updates: Any) -> PooledCredential:
  978. """``replace(entry, **updates)``, swap it into the pool, optionally persist."""
  979. updated = replace(entry, **updates)
  980. self._replace_entry(entry, updated)
  981. if persist:
  982. self._persist()
  983. # _persist may have swapped in a peer's newer token generation; hand callers
  984. # the live entry so they don't rebind the client to the stale pair.
  985. return self._find(lambda e: e.id == updated.id) or updated
  986. return updated
  987. def _quarantine_sources(self, entry: PooledCredential, sources: set[str]) -> None:
  988. """Drop every entry seeded from *sources* and persist the removal.
  989. Atomic read-modify-write of ``self._entries``: this runs on the
  990. DEFERRED refresh path (outside the pool lock), so take the RLock here;
  991. still-locked callers re-enter safely.
  992. """
  993. with self._lock:
  994. removed_ids = [item.id for item in self._entries if item.source in sources]
  995. self._entries = [item for item in self._entries if item.source not in sources]
  996. if self._current_id == entry.id:
  997. self._current_id = None
  998. self._persist(removed_ids=removed_ids)
  999. # ---- exhaustion --------------------------------------------------------
  1000. def _is_terminal_auth_failure(
  1001. self,
  1002. status_code: Optional[int],
  1003. normalized_error: dict[str, Any],
  1004. ) -> bool:
  1005. """Detect upstream-permanent OAuth failures that won't recover on TTL.
  1006. Only 401s whose reason is a known terminal OAuth state count;
  1007. token_expired (refreshable) and reason-less 401s (possible glitch)
  1008. stay transient, as do 429/402. The one status-independent case is
  1009. ``CREDENTIAL_PERSIST_FAILED_REASON``: no upstream response is involved,
  1010. the rotated pair never became durable and only a re-auth recovers it.
  1011. """
  1012. raw_reason = normalized_error.get("reason")
  1013. reason = raw_reason.strip().lower() if isinstance(raw_reason, str) else ""
  1014. if reason == CREDENTIAL_PERSIST_FAILED_REASON:
  1015. return True
  1016. return status_code == 401 and reason in _TERMINAL_AUTH_REASONS
  1017. def _mark_exhausted(
  1018. self,
  1019. entry: PooledCredential,
  1020. status_code: Optional[int],
  1021. error_context: Optional[dict[str, Any]] = None,
  1022. *,
  1023. persist: bool = True,
  1024. failure_reason: Optional[str] = None,
  1025. ) -> PooledCredential:
  1026. normalized_error = _normalize_error_context(error_context)
  1027. # Permanent OAuth failures become STATUS_DEAD, not STATUS_EXHAUSTED:
  1028. # otherwise a revoked credential re-enters rotation every hour and
  1029. # fails immediately until the user removes it (#32849); a row whose
  1030. # refresh grant just went DEAD stays DEAD on the 401 that follows.
  1031. terminal = entry.last_status == STATUS_DEAD or self._is_terminal_auth_failure(status_code, normalized_error)
  1032. # Carry the classifier's verdict so the cooldown is sized by what
  1033. # actually failed (a billing 403 must not get the sole-credential
  1034. # transient cooldown); absent a classification, clear a stale one.
  1035. updated_extra = dict(entry.extra)
  1036. if failure_reason:
  1037. updated_extra["failure_reason"] = failure_reason
  1038. else:
  1039. updated_extra.pop("failure_reason", None)
  1040. return self._adopt(
  1041. entry,
  1042. persist=persist,
  1043. last_status=STATUS_DEAD if terminal else STATUS_EXHAUSTED,
  1044. last_status_at=time.time(),
  1045. last_error_code=status_code,
  1046. last_error_reason=normalized_error.get("reason"),
  1047. last_error_message=normalized_error.get("message"),
  1048. last_error_reset_at=normalized_error.get("reset_at"),
  1049. extra=updated_extra,
  1050. )
  1051. # ---- cross-process token resync ---------------------------------------
  1052. #
  1053. # OAuth refresh tokens are single-use. When another process (CLI, another
  1054. # profile, a concurrent cron) rotates a pair, our in-memory entry holds a
  1055. # consumed refresh token; replaying it yields ``refresh_token_reused`` /
  1056. # ``invalid_grant``. These helpers adopt the fresher pair from wherever the
  1057. # provider's token authority lives, clearing stale exhaustion state.
  1058. def _sync_anthropic_entry_from_credentials_file(self, entry: PooledCredential) -> PooledCredential:
  1059. """Sync a claude_code entry from ~/.claude/.credentials.json if tokens differ."""
  1060. if self.provider != "anthropic" or entry.source != "claude_code":
  1061. return entry
  1062. try:
  1063. from agent.anthropic_credentials import read_claude_code_credentials
  1064. creds = read_claude_code_credentials()
  1065. if not creds:
  1066. return entry
  1067. file_refresh = creds.get("refreshToken", "")
  1068. file_access = creds.get("accessToken", "")
  1069. # Access tokens can be re-issued without a new refresh token, so
  1070. # checking only refresh_token leaves a stale access_token in the
  1071. # pool -> 401 on every request until the exhausted TTL expires.
  1072. if (file_access or file_refresh) and (
  1073. (file_access and file_access != (entry.access_token or ""))
  1074. or (file_refresh and file_refresh != (entry.refresh_token or ""))
  1075. ):
  1076. logger.debug("Pool entry %s: syncing tokens from credentials file (tokens changed)", entry.id)
  1077. return self._adopt(
  1078. entry,
  1079. access_token=file_access or entry.access_token,
  1080. refresh_token=file_refresh or entry.refresh_token,
  1081. expires_at_ms=creds.get("expiresAt", 0) or entry.expires_at_ms,
  1082. **_CLEAR_STATUS,
  1083. )
  1084. except Exception as exc:
  1085. logger.debug("Failed to sync from credentials file: %s", exc)
  1086. return entry
  1087. def _sync_entry_from_pool_store(self, entry: PooledCredential) -> PooledCredential:
  1088. """Adopt a token pair rotated by another pool instance (anthropic, xai-oauth).
  1089. Re-reads the exact persisted row from the credential-pool store while
  1090. the shared cross-process auth-store lock is held. Direct integrations
  1091. load a fresh ``CredentialPool`` per request, so in-memory locks cannot
  1092. protect a single-use refresh token across requests or processes.
  1093. Anthropic borrowed sources (``claude_code``) are excluded: they are
  1094. reference-only rows whose secrets are stripped before reaching
  1095. auth.json, so re-reading yields empty tokens that would be adopted as
  1096. a "rotation" — blanking a usable credential. The singleton file, not
  1097. the pool store, is token authority for those sources; a row with no
  1098. token material at all is refused for the same reason.
  1099. """
  1100. if self.provider not in ("anthropic", "xai-oauth") and plugin_refresh_hook(self.provider) is None:
  1101. return entry
  1102. is_anthropic = self.provider == "anthropic"
  1103. is_xai = self.provider == "xai-oauth"
  1104. display = {"anthropic": "Anthropic", "xai-oauth": "xAI"}.get(self.provider, self.provider)
  1105. if is_anthropic and is_borrowed_credential_source(entry.source, self.provider):
  1106. return entry
  1107. try:
  1108. persisted = next(
  1109. (p for p in read_credential_pool(self.provider) if isinstance(p, dict) and p.get("id") == entry.id),
  1110. None,
  1111. )
  1112. if not isinstance(persisted, dict):
  1113. return entry
  1114. # Same base policy as _persist/load_pool: a token-less disk row is a known
  1115. # (blank) generation, recorded before the no-token-material bail-out below.
  1116. self._persisted_token_pairs[entry.id] = auth_mod._credential_token_pair(persisted)
  1117. stored = PooledCredential.from_dict(self.provider, persisted)
  1118. # No token material at all is never a "rotation" (anthropic borrowed rows, a plugin row a
  1119. # peer blanked mid-write): adopting it would replace a usable credential with nothing.
  1120. if not is_xai and not (stored.access_token or "").strip() and not (stored.refresh_token or "").strip():
  1121. return entry
  1122. if stored.access_token != entry.access_token or stored.refresh_token != entry.refresh_token:
  1123. logger.debug(
  1124. "Pool entry %s: adopting %s OAuth tokens rotated by another pool instance",
  1125. entry.id, display,
  1126. )
  1127. self._replace_entry(entry, stored)
  1128. return stored
  1129. except Exception as exc:
  1130. logger.debug("Failed to sync %s OAuth entry from credential pool: %s", display, exc)
  1131. return entry
  1132. _sync_anthropic_entry_from_pool_store = _sync_entry_from_pool_store
  1133. def _sync_entry_from_auth_store(self, entry: PooledCredential) -> PooledCredential:
  1134. """Sync a Codex / xAI device_code entry from auth.json ``providers.<id>.tokens``.
  1135. A fresh ``hermes model`` / ``hermes auth`` login writes new tokens
  1136. under ``_auth_store_lock`` while the pool entry may sit frozen behind
  1137. a ``last_error_reset_at`` hours in the future; without this sync every
  1138. request fails with "no available entries" despite fresh credentials on
  1139. disk. Only singleton-seeded entries apply — env/API-key rows have no
  1140. auth.json shadow.
  1141. """
  1142. spec = _TOKENS_SINGLETON_PROVIDERS.get(self.provider)
  1143. if spec is None:
  1144. return entry
  1145. display = spec[0]
  1146. is_codex = self.provider == "openai-codex"
  1147. sources = ("device_code", "manual:device_code") if is_codex else ("device_code",)
  1148. if entry.source not in sources:
  1149. return entry
  1150. try:
  1151. with _auth_store_lock():
  1152. state = _load_provider_state(_load_auth_store(), self.provider)
  1153. tokens = state.get("tokens") if isinstance(state, dict) else None
  1154. if not isinstance(tokens, dict):
  1155. return entry
  1156. if is_codex and not _codex_entry_tracks_singleton(entry, tokens):
  1157. return entry
  1158. store_access = tokens.get("access_token", "")
  1159. store_refresh = tokens.get("refresh_token", "")
  1160. entry_refresh = entry.refresh_token or ""
  1161. # Adopt when either side differs: a fresh refresh_token from
  1162. # another process means our pair is consumed/stale.
  1163. should_adopt = bool(store_access) and (
  1164. store_access != (entry.access_token or "")
  1165. or (store_refresh and store_refresh != entry_refresh)
  1166. )
  1167. if not should_adopt and is_codex and store_refresh and store_refresh != entry_refresh and not store_access:
  1168. # Store has only a refresh_token — another process rotated the
  1169. # pair and the access_token was consumed. Adopt the
  1170. # refresh_token so we don't replay the consumed one.
  1171. logger.info(
  1172. "Pool entry %s: auth.json has newer refresh_token "
  1173. "but no access_token; adopting refresh_token to "
  1174. "avoid replaying consumed token",
  1175. entry.id,
  1176. )
  1177. should_adopt = True
  1178. if should_adopt and _singleton_predates_entry(state, entry):
  1179. # #106705: manual:* entries never write back to the singleton
  1180. # (#39236), so after a pool-side rotation the singleton sits
  1181. # one chain behind. Adopting it would replay the consumed
  1182. # refresh token. ``last_refresh`` is stamped on every
  1183. # successful rotation on both sides; when either side lacks a
  1184. # parseable stamp this falls through to the historical
  1185. # adopt-on-difference above (#70111).
  1186. logger.info(
  1187. "Pool entry %s: auth.json singleton predates this entry's "
  1188. "rotation (last_refresh %s < %s); keeping pool chain to "
  1189. "avoid replaying the consumed refresh token",
  1190. entry.id,
  1191. state.get("last_refresh") if isinstance(state, dict) else None,
  1192. entry.last_refresh,
  1193. )
  1194. should_adopt = False
  1195. if should_adopt:
  1196. logger.debug(
  1197. "Pool entry %s: syncing %s tokens from auth.json (refreshed by another process)",
  1198. entry.id, display,
  1199. )
  1200. field_updates: dict[str, Any] = {
  1201. "access_token": store_access or entry.access_token,
  1202. "refresh_token": store_refresh or entry.refresh_token,
  1203. **_CLEAR_STATUS,
  1204. }
  1205. if state.get("last_refresh"):
  1206. field_updates["last_refresh"] = state["last_refresh"]
  1207. return self._adopt(entry, **field_updates)
  1208. except Exception as exc:
  1209. logger.debug("Failed to sync %s entry from auth.json: %s", display, exc)
  1210. return entry
  1211. def _sync_nous_entry_from_auth_store(self, entry: PooledCredential) -> PooledCredential:
  1212. """Sync a Nous device_code entry from auth.json ``providers.nous`` if state differs.
  1213. Another process refreshing via ``resolve_nous_runtime_credentials``
  1214. writes fresh tokens under ``_auth_store_lock``; adopting them avoids a
  1215. "refresh token reuse" revocation on the Nous Portal.
  1216. """
  1217. if self.provider != "nous" or entry.source != "device_code":
  1218. return entry
  1219. try:
  1220. with _auth_store_lock():
  1221. state = _load_provider_state(_load_auth_store(), "nous")
  1222. if not state:
  1223. return entry
  1224. comparable = {
  1225. key: state.get(key)
  1226. for key in (
  1227. "access_token", "refresh_token", "expires_at",
  1228. "agent_key", "agent_key_expires_at", "inference_base_url",
  1229. )
  1230. }
  1231. if not any(v not in (None, "") and getattr(entry, k, None) != v for k, v in comparable.items()):
  1232. return entry
  1233. logger.debug("Pool entry %s: syncing Nous state from auth.json", entry.id)
  1234. field_updates: dict[str, Any] = dict(_CLEAR_STATUS)
  1235. field_updates.update({k: v for k, v in comparable.items() if v})
  1236. extra_updates = dict(entry.extra)
  1237. extra_updates.update(
  1238. {k: state[k] for k in _NOUS_EXTRA_STATE_KEYS if state.get(k) is not None}
  1239. )
  1240. return self._adopt(entry, extra=extra_updates, **field_updates)
  1241. except Exception as exc:
  1242. logger.debug("Failed to sync Nous entry from auth.json: %s", exc)
  1243. return entry
  1244. def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None:
  1245. """Write refreshed pool entry tokens back to auth.json ``providers.<id>``.
  1246. Otherwise the next ``load_pool()`` re-seeds the stale singleton state
  1247. over the fresh entry — potentially a consumed single-use refresh
  1248. token. Applies to Nous, OpenAI Codex and xAI OAuth singletons.
  1249. ``set_active=False`` everywhere: a sync-back is a token-rotation side
  1250. effect, not the user choosing a provider; ``_save_provider_state``
  1251. would flip ``active_provider`` to whichever provider refreshed last.
  1252. #74339: decide the root write-through on WHERE the state resolved
  1253. from (``_load_provider_state_with_source``), not on whether the
  1254. profile has a ``providers.<id>`` key — ``_store_provider_state``
  1255. creates that key unconditionally, which self-sealed the check after
  1256. the first refresh. When the grant came from the global root, write
  1257. back to root ONLY and skip the profile store so it never accrues a
  1258. shadowing key that blocks both the fallback and the write-through.
  1259. """
  1260. # Only singleton-seeded entries sync back; ``manual:*`` entries are
  1261. # independent credentials and must not write to the singleton.
  1262. if entry.source != "device_code" or self.provider not in ("nous", *_TOKENS_SINGLETON_PROVIDERS):
  1263. return
  1264. try:
  1265. with _auth_store_lock():
  1266. auth_store = _load_auth_store()
  1267. state, source_path = _load_provider_state_with_source(auth_store, self.provider)
  1268. if not isinstance(state, dict):
  1269. return
  1270. global_root = _global_auth_file_path()
  1271. is_from_root = bool(
  1272. source_path is not None and global_root is not None and _same_path(source_path, global_root)
  1273. )
  1274. if not self._apply_entry_to_singleton_state(entry, state):
  1275. return
  1276. if is_from_root:
  1277. _write_through_provider_state_to_global_root(self.provider, state)
  1278. else:
  1279. _store_provider_state(auth_store, self.provider, state, set_active=False)
  1280. _save_auth_store(auth_store)
  1281. except Exception as exc:
  1282. logger.debug("Failed to sync %s pool entry back to auth store: %s", self.provider, exc)
  1283. def _apply_entry_to_singleton_state(self, entry: PooledCredential, state: dict[str, Any]) -> bool:
  1284. """Copy *entry*'s tokens into the provider's auth.json ``state`` in place."""
  1285. if self.provider == "nous":
  1286. state["access_token"] = entry.access_token
  1287. for key in ("refresh_token", "expires_at", "agent_key", "agent_key_expires_at"):
  1288. if getattr(entry, key):
  1289. state[key] = getattr(entry, key)
  1290. for extra_key in _NOUS_EXTRA_STATE_KEYS:
  1291. val = entry.extra.get(extra_key)
  1292. if val is not None:
  1293. state[extra_key] = val
  1294. if entry.inference_base_url:
  1295. state["inference_base_url"] = entry.inference_base_url
  1296. return True
  1297. tokens = state.get("tokens")
  1298. if not isinstance(tokens, dict):
  1299. return False
  1300. tokens["access_token"] = entry.access_token
  1301. if entry.refresh_token:
  1302. tokens["refresh_token"] = entry.refresh_token
  1303. if entry.last_refresh:
  1304. state["last_refresh"] = entry.last_refresh
  1305. return True
  1306. # ---- refresh -----------------------------------------------------------
  1307. def _refresh_entry(self, entry: PooledCredential, *, force: bool, failed_key: Optional[str] = None):
  1308. if entry.auth_type != AUTH_TYPE_OAUTH or not entry.refresh_token:
  1309. if force:
  1310. self._mark_exhausted(entry, None)
  1311. return None
  1312. # Plugin providers with a ``refresh_credential`` hook are treated as single-use by default:
  1313. # the pool cannot know their grant semantics, and a needless in-lock re-read is cheaper than
  1314. # a ``refresh_token_reused`` login loss. Eligibility comes from the hook, never a name set.
  1315. if self.provider not in _SINGLE_USE_REFRESH_PROVIDERS and plugin_refresh_hook(self.provider) is None:
  1316. return self._refresh_entry_impl(entry, force=force, failed_key=failed_key)
  1317. # Single-use refresh tokens: sync -> POST -> write-back must be atomic
  1318. # across Hermes processes, or two processes adopt the same on-disk
  1319. # token, both POST it, and the loser gets ``refresh_token_reused`` /
  1320. # ``invalid_grant`` (for Anthropic sources other than claude_code
  1321. # there was no recovery path at all). Serialize through the shared
  1322. # cross-process auth-store flock; a waiter's in-lock re-sync picks up
  1323. # the winner's rotated token and skips the POST.
  1324. with _auth_store_lock(timeout_seconds=self._single_use_refresh_lock_timeout()):
  1325. if self.provider == "openai-codex":
  1326. synced = self._sync_entry_from_auth_store(entry)
  1327. if synced is not entry and not force and not self._entry_needs_refresh(synced):
  1328. return synced
  1329. return self._refresh_entry_impl(synced, force=force)
  1330. synced = self._sync_entry_from_pool_store(entry)
  1331. if self.provider == "anthropic" and synced.source == "claude_code":
  1332. # claude_code entries are NOT profile-owned: the refresh token
  1333. # lives in one shared ~/.claude/.credentials.json (or Keychain)
  1334. # every profile reads. The profile-scoped lock above only covers
  1335. # THIS profile's auth.json, so take the dedicated shared-file
  1336. # lock (inner, per the ordering invariant on ``_auth_store_lock``)
  1337. # and re-read that authoritative file before any
  1338. # adopt-and-return shortcut fires. The official ``claude`` CLI
  1339. # rotating out-of-band is handled by the sync-and-retry-once
  1340. # fallback in ``_recover_failed_refresh``.
  1341. with self._claude_code_credentials_lock():
  1342. synced = self._sync_anthropic_entry_from_credentials_file(synced)
  1343. if synced.refresh_token != entry.refresh_token:
  1344. return synced
  1345. return self._refresh_entry_impl(synced, force=force)
  1346. if synced.access_token != entry.access_token or synced.refresh_token != entry.refresh_token:
  1347. return synced
  1348. return self._refresh_entry_impl(synced, force=force)
  1349. def _claude_code_credentials_lock(self):
  1350. """Cross-process lock keyed to the shared claude_code credentials file.
  1351. Unlike the per-profile ``_auth_store_lock()`` this serializes every
  1352. profile and process that might refresh a ``claude_code`` entry.
  1353. """
  1354. from agent.anthropic_credentials import claude_code_credentials_path
  1355. return _auth_store_lock(
  1356. timeout_seconds=self._single_use_refresh_lock_timeout(),
  1357. target_path=claude_code_credentials_path(),
  1358. )
  1359. def _fail_closed_unpersisted_rotation(
  1360. self,
  1361. entry: PooledCredential,
  1362. exc: BaseException,
  1363. *,
  1364. store: str,
  1365. ) -> None:
  1366. """Quarantine an entry whose rotated pair never reached its store.
  1367. For ``claude_code`` / ``hermes_pkce`` the singleton file — not
  1368. auth.json — is authoritative: ``_seed_from_singletons()`` re-reads it
  1369. on every ``load_pool()``. When the refresh POST succeeded but the
  1370. singleton write failed, the replacement pair exists only in memory
  1371. while the consumed pair survives on disk and would be re-seeded over
  1372. any row we persisted; the next refresh would replay the spent token.
  1373. So never expose or persist the rotated pair; mark the entry terminally
  1374. so it surfaces as an explicit re-auth requirement.
  1375. """
  1376. logger.error(
  1377. "Anthropic %s refresh rotated the single-use token but could not commit it "
  1378. "to %s (%s) — failing closed and quarantining the credential; "
  1379. "re-authenticate to recover",
  1380. entry.source, store, exc,
  1381. )
  1382. try:
  1383. from agent.anthropic_credentials import (
  1384. mark_rotation_consumed_uncommitted,
  1385. spent_rotation_source_path,
  1386. )
  1387. # The singleton still holds the spent pair and load_pool() re-seeds
  1388. # it, so record the fingerprints — persisted to the shared source's
  1389. # sidecar registry (we hold its path-keyed lock here) so OTHER
  1390. # processes/profiles adopt the terminal verdict instead of leasing
  1391. # the stale pair or re-POSTing the spent refresh token.
  1392. mark_rotation_consumed_uncommitted(
  1393. entry.access_token,
  1394. entry.refresh_token,
  1395. source_path=spent_rotation_source_path(entry.source),
  1396. )
  1397. except Exception: # pragma: no cover - never block the quarantine
  1398. logger.debug("Failed to record consumed rotation fingerprints", exc_info=True)
  1399. self._mark_exhausted(
  1400. entry,
  1401. None,
  1402. {
  1403. "reason": CREDENTIAL_PERSIST_FAILED_REASON,
  1404. "message": f"rotated credential was not durably written to {store}: {exc}",
  1405. },
  1406. )
  1407. def _single_use_refresh_lock_timeout(self) -> float:
  1408. """Configured refresh POST timeout plus margin, so a slow token endpoint cannot starve the flock."""
  1409. env_var = _REFRESH_TIMEOUT_ENV_VARS.get(self.provider, "HERMES_ANTHROPIC_REFRESH_TIMEOUT_SECONDS")
  1410. refresh_timeout_seconds = auth_mod.env_float(env_var, 20)
  1411. return max(float(auth_mod.AUTH_LOCK_TIMEOUT_SECONDS), float(refresh_timeout_seconds) + 5.0)
  1412. def _commit_anthropic_rotation(
  1413. self, entry: PooledCredential, refreshed: dict[str, Any]
  1414. ) -> None:
  1415. """Write a rotated Anthropic pair to its authoritative singleton, or fail closed.
  1416. claude_code -> ~/.claude/.credentials.json (so the fallback resolver
  1417. and other profiles see it). hermes_pkce -> ~/.hermes/.anthropic_oauth.json
  1418. (``_seed_from_singletons`` re-seeds it every load; a borrowed row commits
  1419. to the ROOT's file, never a new profile-local copy, #100339). Not
  1420. ``endswith``: manual:hermes_pkce is pool-owned and a singleton for it
  1421. would be a second authority for the same refresh-token family.
  1422. """
  1423. if entry.source == "claude_code":
  1424. store = "~/.claude/.credentials.json"
  1425. elif entry.source == "hermes_pkce":
  1426. store = "~/.hermes/.anthropic_oauth.json"
  1427. else:
  1428. return
  1429. try:
  1430. from agent import anthropic_credentials as ac
  1431. args = (refreshed["access_token"], refreshed["refresh_token"], refreshed["expires_at_ms"])
  1432. if entry.source == "claude_code":
  1433. ac._write_claude_code_credentials(*args, spent_refresh_token=entry.refresh_token or "")
  1434. else:
  1435. ac._write_hermes_oauth_credentials(*args, target=_singleton_target_for_entry(self, entry))
  1436. except Exception as wexc:
  1437. # Authoritative commit failed: do not mark, persist or return the
  1438. # rotation as successful, and bypass the re-POST recovery path —
  1439. # there is nothing left to retry with.
  1440. raise _RefreshDone(self._fail_closed_unpersisted_rotation(entry, wexc, store=store))
  1441. def _refresh_anthropic(self, entry: PooledCredential) -> PooledCredential:
  1442. """POST the Anthropic refresh, commit to the singleton, return the rotated (unpersisted) entry."""
  1443. from agent.anthropic_credentials import (
  1444. is_rotation_consumed_uncommitted,
  1445. refresh_anthropic_oauth_pure,
  1446. spent_rotation_source_path,
  1447. )
  1448. # Never POST a refresh token another process already spent: the
  1449. # durable sidecar verdict is what a fresh interpreter sees here.
  1450. source_path = spent_rotation_source_path(entry.source)
  1451. if is_rotation_consumed_uncommitted(entry.refresh_token, source_path=source_path) or (
  1452. is_rotation_consumed_uncommitted(entry.access_token, source_path=source_path)
  1453. ):
  1454. raise _RefreshDone(self._fail_closed_unpersisted_rotation(
  1455. entry,
  1456. RuntimeError(
  1457. "credential pair was rotated by another process but the "
  1458. "rotation never committed (spent-rotation sidecar verdict)"
  1459. ),
  1460. store=str(source_path or "credential store"),
  1461. ))
  1462. refreshed = refresh_anthropic_oauth_pure(entry.refresh_token, use_json=entry.source.endswith("hermes_pkce"))
  1463. updated = replace(
  1464. entry,
  1465. access_token=refreshed["access_token"],
  1466. refresh_token=refreshed["refresh_token"],
  1467. expires_at_ms=refreshed["expires_at_ms"],
  1468. )
  1469. self._commit_anthropic_rotation(entry, refreshed)
  1470. return updated
  1471. def _post_tokens_refresh(self, entry: PooledCredential) -> PooledCredential:
  1472. """Codex / xAI: POST the refresh and return the rotated (unpersisted) entry."""
  1473. refresh_fn_name = _TOKENS_SINGLETON_PROVIDERS[self.provider][2]
  1474. refreshed = getattr(auth_mod, refresh_fn_name)(entry.access_token, entry.refresh_token)
  1475. return replace(
  1476. entry,
  1477. access_token=refreshed["access_token"],
  1478. refresh_token=refreshed["refresh_token"],
  1479. last_refresh=refreshed.get("last_refresh"),
  1480. )
  1481. def _refresh_entry_impl(self, entry: PooledCredential, *, force: bool, failed_key: Optional[str] = None):
  1482. # Single-use-token providers adopt fresher tokens from their store
  1483. # BEFORE spending the refresh_token; ``entry`` is rebound to the synced
  1484. # row so the failure path below recovers against the pair we POSTed.
  1485. try:
  1486. if self.provider == "anthropic":
  1487. updated = self._refresh_anthropic(entry)
  1488. elif self.provider in _TOKENS_SINGLETON_PROVIDERS:
  1489. entry = self._sync_entry_from_auth_store(entry)
  1490. updated = self._post_tokens_refresh(entry)
  1491. elif (plugin_refresh := plugin_refresh_hook(self.provider)) is not None:
  1492. rotated = plugin_refresh(entry)
  1493. if not rotated:
  1494. # ``None``/empty = the plugin could not rotate: bench like a failed refresh POST, never
  1495. # report the stale row as refreshed (the loop would replay the dead bearer).
  1496. raise RuntimeError("provider refresh_credential returned no rotated fields")
  1497. updated = apply_plugin_refresh_result(entry, rotated)
  1498. elif self.provider == "nous":
  1499. # The bearer the request SENT, not this row's (maybe older) copy: adopting it replays the 401.
  1500. stale_key = failed_key or entry.runtime_api_key or entry.agent_key or entry.access_token
  1501. synced = self._sync_nous_entry_from_auth_store(entry)
  1502. if synced is not entry:
  1503. entry = synced
  1504. # A peer persisted a DIFFERENT usable key: adopt it without re-spending the refresh token.
  1505. if force and entry.runtime_api_key and entry.runtime_api_key != stale_key:
  1506. logger.debug("Nous entry %s: adopting peer-rotated token, skipping refresh", entry.id)
  1507. return entry
  1508. auth_mod.resolve_nous_runtime_credentials(force_refresh=force, stale_access_token=stale_key or None)
  1509. updated = self._sync_nous_entry_from_auth_store(entry)
  1510. else:
  1511. return entry
  1512. except _RefreshDone as done:
  1513. return done.result
  1514. except Exception as exc:
  1515. logger.debug("Credential refresh failed for %s/%s: %s", self.provider, entry.id, exc)
  1516. return self._recover_failed_refresh(entry, exc)
  1517. updated = replace(updated, **_MARK_OK)
  1518. self._replace_entry(entry, updated)
  1519. # Declare the cleared id: a borrowed row carries no access_token on disk, so
  1520. # the merge's token-change bypass cannot apply and a plain persist would copy
  1521. # the still-binding cooldown back over this success.
  1522. self._persist(status_cleared_ids=[updated.id])
  1523. # Sync back so _seed_from_singletons() on the next load_pool() sees
  1524. # fresh state instead of re-seeding consumed tokens.
  1525. self._sync_device_code_entry_to_auth_store(updated)
  1526. return updated
  1527. def _recover_failed_refresh(self, entry: PooledCredential, exc: Exception) -> Optional[PooledCredential]:
  1528. """After a failed refresh POST: adopt a peer's rotation, quarantine a dead grant, or bench.
  1529. Another process may have consumed the refresh token between our
  1530. pre-POST sync and the HTTP call; re-read the provider's token
  1531. authority once more and adopt fresher tokens before giving up.
  1532. """
  1533. if self.provider == "anthropic":
  1534. if entry.source == "claude_code":
  1535. synced = self._sync_anthropic_entry_from_credentials_file(entry)
  1536. if synced.refresh_token != entry.refresh_token:
  1537. logger.debug("Retrying refresh with synced token from credentials file")
  1538. try:
  1539. from agent.anthropic_credentials import refresh_anthropic_oauth_pure
  1540. refreshed = refresh_anthropic_oauth_pure(
  1541. synced.refresh_token, use_json=synced.source.endswith("hermes_pkce"),
  1542. )
  1543. # Commit to the authoritative singleton BEFORE marking or
  1544. # persisting the pool row, or a failed write leaves an
  1545. # "ok" row that the next load_pool() re-seeds over.
  1546. self._commit_anthropic_rotation(synced, refreshed)
  1547. return self._adopt(
  1548. synced,
  1549. access_token=refreshed["access_token"],
  1550. refresh_token=refreshed["refresh_token"],
  1551. expires_at_ms=refreshed["expires_at_ms"],
  1552. last_status=STATUS_OK,
  1553. last_status_at=None,
  1554. last_error_code=None,
  1555. )
  1556. except _RefreshDone as done:
  1557. return done.result
  1558. except Exception as retry_exc:
  1559. logger.debug("Retry refresh also failed: %s", retry_exc)
  1560. elif not self._entry_needs_refresh(synced):
  1561. logger.debug("Credentials file has valid token, using without refresh")
  1562. return synced
  1563. else:
  1564. # Backstop for pool-owned sources (hermes_pkce, manual:dashboard_pkce):
  1565. # the winner may have persisted between our pre-check and our POST.
  1566. synced = self._sync_entry_from_pool_store(entry)
  1567. if synced.refresh_token != entry.refresh_token:
  1568. logger.debug("Anthropic OAuth refresh failed but pool store has newer tokens — adopting")
  1569. return self._adopt(synced, **_MARK_OK)
  1570. from agent.anthropic_credentials import is_terminal_anthropic_refresh_error
  1571. if is_terminal_anthropic_refresh_error(exc):
  1572. # A dead grant is not "exhausted": benching it for a TTL replays the dead token every
  1573. # hour at DEBUG, so the lost login left no trace (#113023). Never touch the external
  1574. # CLI's credentials file here — only Hermes' own row goes DEAD.
  1575. logger.warning(
  1576. "Anthropic OAuth refresh token for %s is terminally invalid (%s); the credential "
  1577. "leaves rotation. Re-run 'hermes auth add anthropic' to sign in again.",
  1578. entry.label or entry.id[:8], exc)
  1579. self._mark_dead_refresh_grant(entry, exc)
  1580. return None
  1581. elif self.provider in _TOKENS_SINGLETON_PROVIDERS:
  1582. _, display, _, terminal_fn_name = _TOKENS_SINGLETON_PROVIDERS[self.provider]
  1583. synced = self._sync_entry_from_auth_store(entry)
  1584. if synced.refresh_token != entry.refresh_token:
  1585. logger.debug("%s OAuth refresh failed but auth.json has newer tokens — adopting", display)
  1586. return self._adopt(synced, **_MARK_OK)
  1587. # Terminal error with no newer tokens: the stored refresh_token is
  1588. # dead. Clear it from auth.json so the next session does not
  1589. # re-seed the revoked credentials, and drop singleton-seeded
  1590. # entries from the pool (mirrors the Nous quarantine path).
  1591. if getattr(auth_mod, terminal_fn_name)(exc):
  1592. # WARNING, not debug: this is the moment a login is lost. At the default log level a
  1593. # silent quarantine looked like "I logged in once and Hermes keeps failing" (#113023).
  1594. logger.warning(
  1595. "%s OAuth refresh token is terminally invalid (%s); clearing local token state. "
  1596. "Re-run 'hermes auth add %s' to sign in again.", display, exc, self.provider)
  1597. self._clear_terminal_tokens_state(entry, exc)
  1598. self._quarantine_sources(entry, {"device_code"})
  1599. self._mark_dead_refresh_grant(entry, exc)
  1600. return None
  1601. elif self.provider == "nous":
  1602. synced = self._sync_nous_entry_from_auth_store(entry)
  1603. if synced.refresh_token != entry.refresh_token:
  1604. logger.debug("Nous refresh failed but auth.json has newer tokens — adopting")
  1605. updated = self._adopt(synced, **_MARK_OK)
  1606. self._sync_device_code_entry_to_auth_store(updated)
  1607. return updated
  1608. if isinstance(exc, TimeoutError):
  1609. # Lost the auth-store lock race under heavy fan-out. That says
  1610. # nothing about the credential — benching it here emptied the
  1611. # pool for ~120 sessions ("matched no nous entry ... pool size
  1612. # 0"). The caller's retry re-syncs once the winner persisted.
  1613. logger.debug("Nous refresh skipped: auth store lock busy; not benching entry")
  1614. return entry
  1615. if auth_mod._is_terminal_nous_refresh_error(exc):
  1616. logger.warning(
  1617. "Nous refresh token is terminally invalid (%s); clearing local token state. "
  1618. "Re-run 'hermes auth add nous' to sign in again.", exc)
  1619. self._clear_terminal_nous_state(entry, exc)
  1620. self._quarantine_sources(
  1621. entry,
  1622. {auth_mod.NOUS_DEVICE_CODE_SOURCE, f"manual:{auth_mod.NOUS_DEVICE_CODE_SOURCE}"},
  1623. )
  1624. self._mark_dead_refresh_grant(entry, exc)
  1625. return None
  1626. elif plugin_refresh_hook(self.provider) is not None:
  1627. handled, result = recover_failed_plugin_refresh(self, entry, exc)
  1628. if handled:
  1629. return result
  1630. self._mark_exhausted(entry, None)
  1631. return None
  1632. def _mark_dead_refresh_grant(self, entry: PooledCredential, exc: Exception) -> None:
  1633. """Mark a row whose refresh token was terminally rejected DEAD, if the quarantine kept it.
  1634. ``_quarantine_sources`` drops only singleton-seeded rows; an independent ``manual:*`` login
  1635. (``hermes auth add``) survives, and an unmarked survivor re-enters rotation and re-fires the
  1636. terminal WARNING on every later refresh attempt. DEAD leaves rotation until a write-side
  1637. re-auth sync clears it (never via TTL).
  1638. """
  1639. with self._lock:
  1640. current = next((item for item in self._entries if item.id == entry.id), None)
  1641. if current is None or current.last_status == STATUS_DEAD:
  1642. return
  1643. self._adopt(
  1644. current,
  1645. last_status=STATUS_DEAD,
  1646. last_status_at=time.time(),
  1647. last_error_code=None,
  1648. last_error_reason=str(getattr(exc, "code", None) or "invalid_grant"),
  1649. last_error_message=str(exc),
  1650. )
  1651. def _clear_terminal_tokens_state(self, entry: PooledCredential, exc: Exception) -> None:
  1652. """Drop the dead Codex/xAI token pair from auth.json unless a peer already rotated it."""
  1653. display = _TOKENS_SINGLETON_PROVIDERS[self.provider][1]
  1654. try:
  1655. with _auth_store_lock():
  1656. auth_store = _load_auth_store()
  1657. state = _load_provider_state(auth_store, self.provider) or {}
  1658. tokens = (state.get("tokens") or {}) if isinstance(state, dict) else None
  1659. if isinstance(tokens, dict):
  1660. store_refresh = str(tokens.get("refresh_token") or "").strip()
  1661. if not store_refresh or store_refresh == str(entry.refresh_token or "").strip():
  1662. tokens.pop("access_token", None)
  1663. tokens.pop("refresh_token", None)
  1664. state["tokens"] = tokens
  1665. state["last_auth_error"] = {
  1666. "provider": self.provider,
  1667. "code": getattr(exc, "code", "unknown"),
  1668. "message": str(exc),
  1669. "reason": "credential_pool_refresh_failure",
  1670. "relogin_required": True,
  1671. "at": datetime.now(UTC).isoformat(),
  1672. }
  1673. _save_provider_state(auth_store, self.provider, state)
  1674. _save_auth_store(auth_store)
  1675. except Exception as clear_exc:
  1676. logger.debug("Failed to clear terminal %s OAuth state: %s", display, clear_exc)
  1677. def _clear_terminal_nous_state(self, entry: PooledCredential, exc: Exception) -> None:
  1678. try:
  1679. with _auth_store_lock():
  1680. auth_store = _load_auth_store()
  1681. state = _load_provider_state(auth_store, "nous") or {
  1682. "client_id": entry.client_id,
  1683. "portal_base_url": entry.portal_base_url,
  1684. "inference_base_url": entry.inference_base_url,
  1685. "token_type": entry.token_type,
  1686. "scope": entry.scope,
  1687. "tls": entry.tls,
  1688. }
  1689. store_refresh = str(state.get("refresh_token") or "").strip()
  1690. if not store_refresh or store_refresh == str(entry.refresh_token or "").strip():
  1691. auth_mod._quarantine_nous_oauth_state(state, exc, reason="credential_pool_refresh_failure")
  1692. auth_mod._quarantine_nous_pool_entries(auth_store, exc, reason="credential_pool_refresh_failure")
  1693. _save_provider_state(auth_store, "nous", state)
  1694. _save_auth_store(auth_store)
  1695. except Exception as clear_exc:
  1696. logger.debug("Failed to clear terminal Nous OAuth state: %s", clear_exc)
  1697. def _codex_quota_restored_upstream(self, entry: PooledCredential) -> bool:
  1698. """Live-check whether an exhausted Codex entry's quota reset early.
  1699. A Codex 429 persists a ``last_error_reset_at`` that can be days out
  1700. (weekly windows), but the window can reopen before then (redeemed
  1701. reset, plan upgrade, OpenAI reset) — issue #43747. Only fires for
  1702. 429/quota-shaped errors; the probe is throttled per token (5 min) so
  1703. it is safe on the hot selection path.
  1704. """
  1705. if self.provider != "openai-codex" or entry.last_status != STATUS_EXHAUSTED:
  1706. return False
  1707. if not auth_mod._is_codex_rate_limit_shaped(
  1708. entry.last_error_code, entry.last_error_reason, entry.last_error_message,
  1709. ):
  1710. return False
  1711. token = entry.access_token or ""
  1712. if not token:
  1713. return False
  1714. try:
  1715. # An exhausted entry is skipped by the refresh chain, so its stored token is usually
  1716. # expired by probe time (401 -> None -> cooldown kept, #89415): refresh it first.
  1717. fresh = auth_mod._refresh_expired_codex_probe_token(token, entry.refresh_token)
  1718. if fresh:
  1719. # Persist the rotated pair on both sides the way ``_refresh_entry`` does:
  1720. # ``last_refresh`` plus the singleton write-back, or the next selection's
  1721. # auth-store sync re-adopts the consumed pair from ``providers.openai-codex``
  1722. # and clears the cooldown with it.
  1723. entry = self._adopt(
  1724. entry, access_token=fresh["access_token"], refresh_token=fresh["refresh_token"],
  1725. last_refresh=fresh.get("last_refresh") or entry.last_refresh,
  1726. )
  1727. self._sync_device_code_entry_to_auth_store(entry)
  1728. token = entry.access_token or token
  1729. # The row keeps the canonical URL; a gateway key belongs to its route host (#121486).
  1730. from hermes_cli.auth_codex import _codex_pool_route_base_url
  1731. return bool(auth_mod._probe_codex_quota_restored(
  1732. token, base_url=_codex_pool_route_base_url(entry.base_url)))
  1733. except Exception:
  1734. logger.debug("Codex quota-restored probe failed", exc_info=True)
  1735. return False
  1736. def _entry_needs_refresh(self, entry: PooledCredential) -> bool:
  1737. if entry.auth_type != AUTH_TYPE_OAUTH:
  1738. return False
  1739. if self.provider == "anthropic" or plugin_refresh_hook(self.provider) is not None:
  1740. return plugin_row_is_expiring(entry) # expiry-stamped rows: rotate 2 min ahead of expires_at_ms
  1741. if self.provider == "openai-codex":
  1742. return _codex_access_token_is_expiring(entry.access_token, CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS)
  1743. if self.provider == "xai-oauth":
  1744. return auth_mod._xai_access_token_is_expiring(
  1745. entry.access_token, auth_mod._xai_proactive_refresh_skew_seconds(entry.access_token),
  1746. )
  1747. # Nous refresh can require network access and happens when runtime
  1748. # credentials are actually resolved, not on enumeration/selection.
  1749. return False
  1750. # ---- selection ---------------------------------------------------------
  1751. def select(self, *, model: Optional[str] = None) -> Optional[PooledCredential]:
  1752. entry, pending_refresh = self._select_under_lock(model=model)
  1753. if pending_refresh:
  1754. self._refresh_pending_entries(pending_refresh)
  1755. # Re-select now that the refreshed entries are back in the pool.
  1756. if entry is None:
  1757. entry, _ = self._select_under_lock(model=model)
  1758. if entry is not None:
  1759. self._unmatched_rotation_streak = 0
  1760. return entry
  1761. def _select_under_lock(self, *, model: Optional[str] = None) -> tuple[Optional[PooledCredential], list[PooledCredential]]:
  1762. with self._lock:
  1763. return self._select_unlocked(model=model)
  1764. def _refresh_pending_entries(self, pending: list[PooledCredential]) -> None:
  1765. """Refresh deferred single-use-token entries OUTSIDE the pool lock.
  1766. Each refresh takes the cross-process ``_auth_store_lock`` (20+ s
  1767. possible) and merges into the pool through the self-locking mutation
  1768. primitives; failures are silently skipped.
  1769. Entries are refreshed one at a time with a lock-free spacing window
  1770. between consecutive holds: ``_refresh_entry`` legitimately keeps the
  1771. store lock across its POST (single-use-token safety — sync -> POST ->
  1772. write-back must be atomic across processes), so a back-to-back chain
  1773. of N holds can starve a waiter on the same profile's auth.json whose
  1774. timeout is shorter (a Desktop assistant start, AUTH_LOCK_TIMEOUT_SECONDS
  1775. = 15s, vs the holder's max(15s, refresh_timeout + 5s)) for the whole
  1776. sweep (#124533). A short window between holds lets the waiter's
  1777. 50ms poll cadence interleave; the chain still finishes promptly.
  1778. """
  1779. for index, entry in enumerate(pending):
  1780. if index:
  1781. time.sleep(_REFRESH_SWEEP_SPACING_SECONDS)
  1782. self._refresh_entry(entry, force=False)
  1783. def _reset_cleared_after(self, entry: PooledCredential) -> Optional[float]:
  1784. """Epoch of a ``hermes auth reset`` persisted by another process AFTER *entry*'s status, else None."""
  1785. try:
  1786. row = next((p for p in read_credential_pool(self.provider)
  1787. if isinstance(p, dict) and p.get("id") == entry.id), None)
  1788. cleared = _parse_absolute_timestamp((row or {}).get("status_cleared_at"))
  1789. except Exception as exc:
  1790. logger.debug("Pool entry %s: could not read reset marker: %s", entry.id, exc)
  1791. return None
  1792. return cleared if cleared and cleared > (entry.last_status_at or 0.0) else None
  1793. def _resync_stale_entry(self, entry: PooledCredential) -> PooledCredential:
  1794. """Re-read an exhausted/DEAD singleton-seeded entry from its token authority.
  1795. The user may have re-authed (``hermes model`` / ``hermes auth``, the
  1796. Claude Code CLI, another profile) leaving fresh tokens on disk while
  1797. the pool entry is frozen behind ``last_error_reset_at``. A ``hermes auth
  1798. reset`` run from another process while this pool is live is honoured the
  1799. same way (#89415): the in-memory cooldown would otherwise outlive it.
  1800. """
  1801. if entry.last_status not in {STATUS_EXHAUSTED, STATUS_DEAD}:
  1802. return entry
  1803. cleared_at = self._reset_cleared_after(entry)
  1804. if cleared_at is not None:
  1805. return self._adopt(entry, persist=False, **_MARK_OK, status_cleared_at=cleared_at)
  1806. if entry.source != _RESYNC_SOURCE.get(self.provider):
  1807. return entry
  1808. if self.provider == "anthropic":
  1809. return self._sync_anthropic_entry_from_credentials_file(entry)
  1810. if self.provider == "nous":
  1811. return self._sync_nous_entry_from_auth_store(entry)
  1812. return self._sync_entry_from_auth_store(entry)
  1813. def _available_entries(
  1814. self, *, clear_expired: bool = False, refresh: bool = False, model: Optional[str] = None,
  1815. ) -> tuple[list[PooledCredential], list[PooledCredential]]:
  1816. """Return (available, pending_refresh) for entries not in cooldown.
  1817. *clear_expired* resets elapsed cooldowns to STATUS_OK and persists.
  1818. *refresh* refreshes entries needing a token refresh (skipped on
  1819. failure) — except single-use-token providers (openai-codex,
  1820. xai-oauth), which are returned as *pending_refresh* so the caller
  1821. refreshes them outside the lock instead of stalling every pool
  1822. consumer during cross-process flock acquisition + OAuth network I/O.
  1823. """
  1824. now = time.time()
  1825. cleared_any = False
  1826. entries_to_prune: list[str] = []
  1827. available: list[PooledCredential] = []
  1828. pending_refresh: list[PooledCredential] = []
  1829. sole_credential = self._is_sole_credential()
  1830. for entry in self._entries:
  1831. # Borrowed credentials persist as metadata-only references and are
  1832. # hydrated from their live source on load; never lease an
  1833. # unhydrated duplicate as an empty key.
  1834. if entry.auth_type == AUTH_TYPE_API_KEY and not entry.runtime_api_key:
  1835. continue
  1836. synced = self._resync_stale_entry(entry)
  1837. if synced is not entry:
  1838. entry = synced
  1839. cleared_any = True
  1840. if entry.last_status == STATUS_DEAD:
  1841. # Manual DEAD credentials are pruned after a 24h quiet window;
  1842. # singleton-seeded ones stay (audit trail, and the seeder would
  1843. # re-create them anyway). DEAD never re-enters via TTL — only a
  1844. # write-side re-auth sync clears it.
  1845. if _is_manual_source(entry.source):
  1846. dead_at = entry.last_status_at or 0
  1847. if dead_at and now - dead_at > DEAD_MANUAL_PRUNE_TTL_SECONDS:
  1848. logger.warning(
  1849. "credential pool: pruning DEAD manual entry %s "
  1850. "(reason=%s, age=%.1fh) — re-add via `hermes auth add %s`",
  1851. entry.label or entry.id[:8],
  1852. entry.last_error_reason or "unknown",
  1853. (now - dead_at) / 3600.0,
  1854. self.provider,
  1855. )
  1856. entries_to_prune.append(entry.id) # can't mutate while iterating
  1857. cleared_any = True
  1858. continue
  1859. if model_cooldown_until(entry, model) is not None:
  1860. continue
  1861. if entry.last_status == STATUS_EXHAUSTED:
  1862. exhausted_until = _exhausted_until(entry, sole_credential=sole_credential)
  1863. # Codex quota windows can reopen EARLY; a throttled live probe
  1864. # lifts a stale cooldown (issue #43747).
  1865. if (
  1866. exhausted_until is not None
  1867. and now < exhausted_until
  1868. and not (clear_expired and self._codex_quota_restored_upstream(entry))
  1869. ):
  1870. continue
  1871. if clear_expired:
  1872. # The probe may have rotated this row's single-use token pair: clear the
  1873. # cooldown on the live row, not on this pre-probe copy.
  1874. entry = self._find(lambda e, i=entry.id: e.id == i) or entry
  1875. entry = self._adopt(entry, persist=False, **_MARK_OK)
  1876. cleared_any = True
  1877. if refresh and self._entry_needs_refresh(entry):
  1878. if self.provider in _TOKENS_SINGLETON_PROVIDERS:
  1879. pending_refresh.append(entry)
  1880. continue
  1881. refreshed = self._refresh_entry(entry, force=False)
  1882. if refreshed is None:
  1883. continue
  1884. entry = refreshed
  1885. if entry.auth_type == AUTH_TYPE_OAUTH and not (entry.access_token or "").strip():
  1886. # A borrowed OAuth row that failed to hydrate (or a sanitized
  1887. # row read straight off disk); leasing it would send an empty
  1888. # bearer. The API-key guard above does not cover it.
  1889. continue
  1890. available.append(entry)
  1891. if entries_to_prune:
  1892. pruned_ids = set(entries_to_prune)
  1893. self._entries = [e for e in self._entries if e.id not in pruned_ids]
  1894. if cleared_any:
  1895. self._persist(removed_ids=entries_to_prune)
  1896. return available, pending_refresh
  1897. def _log_no_available_entries(self) -> None:
  1898. """Emit the empty-pool INFO line at most once per throttle window."""
  1899. now = time.monotonic()
  1900. last = self._last_no_entries_log_at
  1901. if last is not None and (now - last) < NO_AVAILABLE_ENTRIES_LOG_THROTTLE_SECONDS:
  1902. return
  1903. self._last_no_entries_log_at = now
  1904. logger.info("credential pool: no available entries (all exhausted or empty)")
  1905. def _select_unlocked(
  1906. self, *, refresh: bool = True, count: bool = True, model: Optional[str] = None,
  1907. ) -> tuple[Optional[PooledCredential], list[PooledCredential]]:
  1908. """Select the best available entry; returns ``(entry, pending_refresh)``.
  1909. ``count=False`` skips the ``request_count`` bump for selections that are
  1910. not going to serve a request (a forced-refresh target lookup).
  1911. """
  1912. available, pending_refresh = self._available_entries(clear_expired=True, refresh=refresh, model=model)
  1913. if not available:
  1914. self._current_id = None
  1915. self._log_no_available_entries()
  1916. return None, pending_refresh
  1917. # The pool recovered; re-arm the throttle so a later re-exhaustion
  1918. # logs immediately.
  1919. self._last_no_entries_log_at = None
  1920. if self._strategy == STRATEGY_RANDOM:
  1921. entry = random.choice(available)
  1922. elif self._strategy == STRATEGY_LEAST_USED and len(available) > 1:
  1923. entry = min(available, key=lambda e: e.request_count)
  1924. else:
  1925. entry = available[0]
  1926. # Count the selection under every strategy. The counter is ``least_used``'s
  1927. # baseline and reaches auth.json on the next persist (exhaustion, rotation,
  1928. # refresh); it used to move only while ``least_used`` was active.
  1929. if count:
  1930. entry = self._adopt(entry, persist=False, request_count=entry.request_count + 1)
  1931. if self._strategy == STRATEGY_ROUND_ROBIN and len(available) > 1:
  1932. rotated = [candidate for candidate in self._entries if candidate.id != entry.id]
  1933. rotated.append(replace(entry, priority=len(self._entries) - 1))
  1934. self._entries = [replace(candidate, priority=idx) for idx, candidate in enumerate(rotated)]
  1935. self._persist()
  1936. entry = self._find(lambda candidate: candidate.id == entry.id) or entry
  1937. self._current_id = entry.id
  1938. return entry, pending_refresh
  1939. def peek(self) -> Optional[PooledCredential]:
  1940. with self._lock:
  1941. current = self._current_unlocked()
  1942. if current is not None:
  1943. return current
  1944. available, _pending = self._available_entries()
  1945. return available[0] if available else None
  1946. def reclaim(self, credential_id: str, *, model: Optional[str] = None) -> Optional[PooledCredential]:
  1947. """Entry *credential_id* once its cooldown has lifted (cleared and token-refreshed the way
  1948. ``select`` would), else ``None``. Never bumps ``request_count`` or round-robin order: a
  1949. live session asking "may I go back?" every turn is not a request."""
  1950. with self._lock:
  1951. available, pending = self._available_entries(clear_expired=True, refresh=True, model=model)
  1952. if any(e.id == credential_id for e in pending):
  1953. self._refresh_pending_entries([e for e in pending if e.id == credential_id])
  1954. with self._lock:
  1955. available, _pending = self._available_entries(clear_expired=True, refresh=True, model=model)
  1956. return next((e for e in available if e.id == credential_id), None)
  1957. # ---- rotation ----------------------------------------------------------
  1958. def _identify_failed_entry(
  1959. self, credential_id: Optional[str], api_key_hint: Optional[str],
  1960. ) -> Optional[PooledCredential]:
  1961. """Resolve the entry that issued a failed request from its supplied identity."""
  1962. entry = None
  1963. if credential_id:
  1964. entry = self._find(lambda e: e.id == credential_id)
  1965. # #79156: when both identities disagree, trust the key that made
  1966. # the request. A stale ``_credential_pool_entry_id`` (per-turn env
  1967. # refresh rewrote ``api_key`` without rebinding the id) would
  1968. # otherwise quarantine a healthy fallback for days.
  1969. if entry is not None and api_key_hint and entry.runtime_api_key != api_key_hint:
  1970. hint_entry = self._find(lambda e: e.runtime_api_key == api_key_hint)
  1971. if hint_entry is not None:
  1972. logger.info(
  1973. "credential pool: credential_id %s runtime key "
  1974. "does not match api_key_hint; attributing failure "
  1975. "to key-matched entry %s instead (#79156)",
  1976. (entry.label or entry.id[:8]),
  1977. (hint_entry.label or hint_entry.id[:8]),
  1978. )
  1979. # Otherwise the id is stale and the request key is not in the
  1980. # pool — drop the id so we do not mark the wrong entry.
  1981. entry = hint_entry
  1982. if entry is None and api_key_hint:
  1983. # Prefer the entry whose key actually failed: on a pool freshly
  1984. # loaded from disk current() is None and _select_unlocked() would
  1985. # return the NEXT key — the wrong one.
  1986. entry = self._find(lambda e: e.runtime_api_key == api_key_hint)
  1987. return entry
  1988. def _rotate_unmatched(self) -> Optional[PooledCredential]:
  1989. """Rotate without marking anything when the failed identity matches no entry.
  1990. Falling through to current()/_select_unlocked() would bench an
  1991. innocent healthy key for the full TTL. But this must be BOUNDED
  1992. (#70401): with OAuth-token auth the 401's key hint never matches any
  1993. ``runtime_api_key``, so every retry lands here, nothing is marked, and
  1994. the caller retries the same dead token forever (~6/sec, starving the
  1995. event loop). Cap consecutive no-mark rotations at one lap of the
  1996. available entries, then surface the error; no cooldown is written.
  1997. """
  1998. self._unmatched_rotation_streak += 1
  1999. available_count = len(self._available_entries()[0])
  2000. if self._unmatched_rotation_streak > max(available_count, 1):
  2001. logger.warning(
  2002. "credential pool: failed credential identity matched no "
  2003. "%s entry for %d consecutive rotations (pool size %d) — "
  2004. "surfacing the error instead of rotating again",
  2005. self.provider, self._unmatched_rotation_streak, available_count,
  2006. )
  2007. self._unmatched_rotation_streak = 0
  2008. self._current_id = None
  2009. return None
  2010. logger.info(
  2011. "credential pool: failed credential identity matched no %s "
  2012. "entry; rotating without marking any credential exhausted",
  2013. self.provider,
  2014. )
  2015. self._current_id = None
  2016. next_entry, _pending = self._select_unlocked(refresh=False)
  2017. if next_entry is not None and len(self._available_entries()[0]) == 1:
  2018. # A single-entry pool cannot rotate: returning its only entry would
  2019. # report a recovery without changing the credential, and the
  2020. # caller retries the same 401 indefinitely.
  2021. self._unmatched_rotation_streak = 0
  2022. self._current_id = None
  2023. return None
  2024. return next_entry
  2025. def mark_exhausted_and_rotate(
  2026. self,
  2027. *,
  2028. status_code: Optional[int],
  2029. error_context: Optional[dict[str, Any]] = None,
  2030. api_key_hint: Optional[str] = None,
  2031. credential_id: Optional[str] = None,
  2032. failure_reason: Optional[str] = None,
  2033. model: Optional[str] = None,
  2034. ) -> Optional[PooledCredential]:
  2035. with self._lock:
  2036. identity_supplied = bool(credential_id or api_key_hint)
  2037. entry = self._identify_failed_entry(credential_id, api_key_hint)
  2038. if entry is None and identity_supplied:
  2039. return self._rotate_unmatched()
  2040. # A real entry was identified — any prior unmatched streak is stale.
  2041. self._unmatched_rotation_streak = 0
  2042. if entry is None:
  2043. entry = self._current_unlocked() or self._select_unlocked(refresh=False)[0]
  2044. if entry is None:
  2045. return None
  2046. _label = entry.label or entry.id[:8]
  2047. if self._is_model_scoped_failure(status_code, model, failure_reason):
  2048. # A generic Anthropic 429 (per-model rate limit) or a Codex account model
  2049. # entitlement rejection: bench this model only, the credential stays
  2050. # available for its siblings.
  2051. self._cool_down_model(entry, model, error_context, failure_reason=failure_reason)
  2052. logger.info("credential pool: %s unavailable for model %s; other models stay available", _label, model)
  2053. self._current_id = None
  2054. next_entry, _pending = self._select_unlocked(refresh=False, model=model)
  2055. return next_entry
  2056. self._mark_exhausted(entry, status_code, error_context, failure_reason=failure_reason)
  2057. # A 402/429/401 is a key-level failure, and the same key can back
  2058. # several entries (an explicit entry plus a ``model_config`` row
  2059. # auto-seeded from ``model.api_key``). Marking only the first
  2060. # leaves siblings OK, ``_select_unlocked()`` keeps handing back
  2061. # the depleted key, and rotation never converges (~2.5 min hang).
  2062. # Mark every entry sharing the failed key.
  2063. failed_runtime_key = entry.runtime_api_key
  2064. if identity_supplied and failed_runtime_key:
  2065. siblings = [
  2066. s for s in self._entries if s.id != entry.id and s.runtime_api_key == failed_runtime_key
  2067. ]
  2068. for sibling in siblings:
  2069. self._mark_exhausted(
  2070. sibling, status_code, error_context, persist=False, failure_reason=failure_reason,
  2071. )
  2072. if siblings:
  2073. self._persist()
  2074. # Re-read the updated entry to log the correct terminal state.
  2075. updated_entry = self._find(lambda e: e.id == entry.id) or entry
  2076. if updated_entry.last_status == STATUS_DEAD:
  2077. logger.warning(
  2078. "credential pool: marking %s DEAD (status=%s, reason=%s) — "
  2079. "permanently failed, will NOT re-enter rotation until re-auth",
  2080. _label, status_code, updated_entry.last_error_reason or "unknown",
  2081. )
  2082. else:
  2083. logger.info("credential pool: marking %s exhausted (status=%s), rotating", _label, status_code)
  2084. self._current_id = None
  2085. next_entry, _pending = self._select_unlocked(refresh=False)
  2086. if next_entry is not None and next_entry.id == entry.id:
  2087. # No-recovery guard (#97315): selection handed back the very entry that was
  2088. # just marked (the auth-store sync adopted fresher tokens, or a quota probe
  2089. # false-positive lifted the bench mid-selection). Returning it reports a
  2090. # successful rotation without changing the credential, so the caller retries
  2091. # the same 429 forever (~2 req/s for hours). Mirror the single-entry guard on
  2092. # the unmatched-identity branch: surface the failure instead.
  2093. logger.warning(
  2094. "credential pool: rotation returned the just-marked entry %s — "
  2095. "treating as no-recovery so the failure surfaces", _label,
  2096. )
  2097. self._current_id = None
  2098. return None
  2099. if next_entry:
  2100. logger.info("credential pool: rotated to %s", next_entry.label or next_entry.id[:8])
  2101. return next_entry
  2102. # ---- leases ------------------------------------------------------------
  2103. def acquire_lease(self, credential_id: Optional[str] = None) -> Optional[str]:
  2104. """Acquire a soft lease on a credential.
  2105. With *credential_id*, lease that entry directly. Otherwise prefer the
  2106. least-leased available credential (priority as tie-breaker); when
  2107. every credential is at the soft cap, still return the least-leased
  2108. one instead of blocking.
  2109. """
  2110. chosen_id, pending_refresh = self._acquire_lease_under_lock(credential_id)
  2111. if pending_refresh:
  2112. self._refresh_pending_entries(pending_refresh)
  2113. # Mirror select(): a pool whose entries all needed a deferred
  2114. # refresh must retry once they are back in rotation, or the caller
  2115. # sees "no credentials available" after a successful refresh.
  2116. if chosen_id is None:
  2117. chosen_id, _ = self._acquire_lease_under_lock(credential_id)
  2118. return chosen_id
  2119. def _acquire_lease_under_lock(
  2120. self, credential_id: Optional[str],
  2121. ) -> tuple[Optional[str], list[PooledCredential]]:
  2122. with self._lock:
  2123. if credential_id:
  2124. self._active_leases[credential_id] = self._active_leases.get(credential_id, 0) + 1
  2125. self._current_id = credential_id
  2126. return credential_id, []
  2127. available, pending_refresh = self._available_entries(clear_expired=True, refresh=True)
  2128. if not available:
  2129. return None, pending_refresh
  2130. below_cap = [e for e in available if self._active_leases.get(e.id, 0) < self._max_concurrent]
  2131. chosen = min(
  2132. below_cap or available,
  2133. key=lambda entry: (self._active_leases.get(entry.id, 0), entry.priority),
  2134. )
  2135. self._active_leases[chosen.id] = self._active_leases.get(chosen.id, 0) + 1
  2136. self._current_id = chosen.id
  2137. return chosen.id, pending_refresh
  2138. def release_lease(self, credential_id: str) -> None:
  2139. with self._lock:
  2140. count = self._active_leases.get(credential_id, 0)
  2141. if count <= 1:
  2142. self._active_leases.pop(credential_id, None)
  2143. else:
  2144. self._active_leases[credential_id] = count - 1
  2145. # ---- explicit refresh / admin ------------------------------------------
  2146. def try_refresh_current(self) -> Optional[PooledCredential]:
  2147. with self._lock:
  2148. return self._try_refresh_current_unlocked()
  2149. def try_refresh_matching(
  2150. self,
  2151. api_key_hint: Optional[str] = None,
  2152. credential_id: Optional[str] = None,
  2153. ) -> Optional[PooledCredential]:
  2154. """Force-refresh the entry that supplied the failed request.
  2155. Direct integrations may reload the pool after a request failed, so
  2156. ``current_id`` cannot identify the issuing credential. With no hint,
  2157. select WITHOUT the normal proactive refresh: the forced refresh below
  2158. must consume a rotating refresh token exactly once.
  2159. """
  2160. with self._lock:
  2161. entry = self._find(lambda e: e.id == credential_id) if credential_id else None
  2162. if entry is None:
  2163. if api_key_hint:
  2164. entry = self._find(lambda e: e.runtime_api_key == api_key_hint)
  2165. else:
  2166. entry = self._current_unlocked() or self._select_unlocked(refresh=False, count=False)[0]
  2167. if entry is None:
  2168. return None
  2169. self._current_id = entry.id
  2170. return self._try_refresh_current_unlocked(failed_key=api_key_hint)
  2171. def _try_refresh_current_unlocked(self, failed_key: Optional[str] = None) -> Optional[PooledCredential]:
  2172. entry = self._current_unlocked()
  2173. if entry is None:
  2174. return None
  2175. refreshed = self._refresh_entry(entry, force=True, failed_key=failed_key)
  2176. if refreshed is not None:
  2177. self._current_id = refreshed.id
  2178. return refreshed
  2179. # --- Seeding --------------------------------------------------------------
  2180. def _upsert_entry(entries: list[PooledCredential], provider: str, source: str, payload: dict[str, Any]) -> bool:
  2181. matching_indices = [idx for idx, entry in enumerate(entries) if entry.source == source]
  2182. existing_idx = matching_indices[0] if matching_indices else None
  2183. duplicate_indices = set(matching_indices[1:])
  2184. if duplicate_indices:
  2185. entries[:] = [entry for idx, entry in enumerate(entries) if idx not in duplicate_indices]
  2186. if existing_idx is None:
  2187. payload.setdefault("id", uuid.uuid4().hex[:6])
  2188. payload.setdefault("priority", _next_priority(entries))
  2189. payload.setdefault("label", payload.get("label") or source)
  2190. entries.append(PooledCredential.from_dict(provider, payload))
  2191. return True
  2192. existing = entries[existing_idx]
  2193. field_updates: dict[str, Any] = {}
  2194. extra_updates: dict[str, Any] = {}
  2195. _field_names = {f.name for f in fields(existing)}
  2196. incoming_token = payload.get("access_token")
  2197. token_changed = incoming_token is not None and incoming_token != existing.access_token
  2198. if token_changed and not existing.access_token:
  2199. # Borrowed sources (claude_code, env-backed rows) are written to
  2200. # auth.json without their secret, so a reloaded entry carries only a
  2201. # ``secret_fingerprint``. Comparing against the empty string reported
  2202. # a rotation on EVERY load and cleared the DEAD/exhausted state the
  2203. # previous process had just persisted. Compare fingerprints instead.
  2204. known_fingerprint = existing.extra.get("secret_fingerprint")
  2205. if isinstance(known_fingerprint, str) and known_fingerprint:
  2206. token_changed = fingerprint_secret_value(incoming_token) != known_fingerprint
  2207. for key, value in payload.items():
  2208. if key in {"id", "priority"} or value is None or (key == "label" and existing.label):
  2209. continue
  2210. if key in _field_names:
  2211. if getattr(existing, key) != value:
  2212. field_updates[key] = value
  2213. elif key in _EXTRA_KEYS and existing.extra.get(key) != value:
  2214. extra_updates[key] = value
  2215. # A rotated token makes the old exhaustion/error state stale.
  2216. if token_changed and existing.last_status is not None:
  2217. field_updates.update(_CLEAR_STATUS)
  2218. if field_updates or extra_updates:
  2219. if extra_updates:
  2220. field_updates["extra"] = {**existing.extra, **extra_updates}
  2221. updated = replace(existing, **field_updates)
  2222. entries[existing_idx] = updated
  2223. # Runtime-only borrowed secret updates refresh the in-memory entry
  2224. # without forcing auth.json churn when the disk-safe payload is
  2225. # unchanged (e.g. env keys with the same fingerprint).
  2226. return bool(duplicate_indices) or existing.to_dict() != updated.to_dict()
  2227. return bool(duplicate_indices)
  2228. _ANTHROPIC_SOURCE_RANK = {
  2229. "env:ANTHROPIC_TOKEN": 0,
  2230. "env:CLAUDE_CODE_OAUTH_TOKEN": 1,
  2231. "hermes_pkce": 2,
  2232. "claude_code": 3,
  2233. "env:ANTHROPIC_API_KEY": 4,
  2234. }
  2235. def _normalize_pool_priorities(provider: str, entries: list[PooledCredential]) -> bool:
  2236. if provider != "anthropic":
  2237. return False
  2238. manual_entries = sorted(
  2239. (entry for entry in entries if _is_manual_source(entry.source)),
  2240. key=lambda entry: entry.priority,
  2241. )
  2242. seeded_entries = sorted(
  2243. (entry for entry in entries if not _is_manual_source(entry.source)),
  2244. key=lambda entry: (
  2245. _ANTHROPIC_SOURCE_RANK.get(entry.source, len(_ANTHROPIC_SOURCE_RANK)),
  2246. entry.priority,
  2247. entry.label,
  2248. ),
  2249. )
  2250. id_to_idx = {entry.id: idx for idx, entry in enumerate(entries)}
  2251. changed = False
  2252. for new_priority, entry in enumerate([*manual_entries, *seeded_entries]):
  2253. if entry.priority != new_priority:
  2254. entries[id_to_idx[entry.id]] = replace(entry, priority=new_priority)
  2255. changed = True
  2256. return changed
  2257. def _retain_sources_not_in(entries: list[PooledCredential], drop: set[str]) -> bool:
  2258. """Remove entries whose source is in *drop*; True if anything was removed."""
  2259. retained = [entry for entry in entries if entry.source not in drop]
  2260. if len(retained) == len(entries):
  2261. return False
  2262. entries[:] = retained
  2263. return True
  2264. class _Seeder:
  2265. """Accumulates ``_upsert_entry`` results for one ``load_pool`` seeding pass."""
  2266. def __init__(self, provider: str, entries: list[PooledCredential]):
  2267. self.provider = provider
  2268. self.entries = entries
  2269. self.changed = False
  2270. self.active_sources: set[str] = set()
  2271. self.is_suppressed = _is_source_suppressed_fn()
  2272. def upsert(self, source: str, payload: dict[str, Any]) -> bool:
  2273. """Upsert unless suppressed (``hermes auth remove`` must stay stable across loads)."""
  2274. if self.is_suppressed(self.provider, source):
  2275. return False
  2276. self.active_sources.add(source)
  2277. ingested = _upsert_entry(self.entries, self.provider, source, {"source": source, **payload})
  2278. self.changed |= ingested
  2279. return ingested
  2280. @property
  2281. def result(self) -> tuple[bool, set[str]]:
  2282. return self.changed, self.active_sources
  2283. def _seed_anthropic_singletons(seed: _Seeder) -> None:
  2284. # Only auto-discover external credentials (Claude Code, Hermes PKCE) when
  2285. # the user explicitly configured anthropic; otherwise auxiliary fallback
  2286. # chains would read ~/.claude/.credentials.json without consent (PR #4210).
  2287. try:
  2288. from hermes_cli.auth import is_provider_explicitly_configured
  2289. if not is_provider_explicitly_configured("anthropic"):
  2290. return
  2291. except ImportError:
  2292. pass
  2293. # API-key vs OAuth is a user-visible choice at `hermes setup`. The API-key
  2294. # signal is ANTHROPIC_API_KEY set AND no OAuth env vars (the save_* helpers
  2295. # zero the other side). Then we MUST NOT seed autodiscovered OAuth tokens:
  2296. # rotation on a 401/429 would silently flip the session onto OAuth, which
  2297. # forces the Claude Code identity injection, `mcp_` tool-name rewrite and
  2298. # claude-cli User-Agent the user explicitly opted out of. Prefer
  2299. # ~/.hermes/.env over os.environ, as `_seed_from_env` does.
  2300. _env_file = load_env()
  2301. def _env_val(key: str) -> str:
  2302. return (_env_file.get(key) or _get_secret(key, "") or "").strip()
  2303. anthropic_oauth_env = _env_val("ANTHROPIC_TOKEN") or _env_val("CLAUDE_CODE_OAUTH_TOKEN")
  2304. if _env_val("ANTHROPIC_API_KEY") and not anthropic_oauth_env:
  2305. # Prune stale autodiscovered OAuth entries from a previous OAuth
  2306. # session so a transient 401 cannot revive them.
  2307. seed.changed |= _retain_sources_not_in(seed.entries, {"hermes_pkce", "claude_code"})
  2308. return
  2309. from agent.anthropic_credentials import (
  2310. read_claude_code_credentials,
  2311. read_hermes_oauth_credentials,
  2312. )
  2313. from agent.credential_sources import adopt_external_logins_enabled
  2314. sources = [("hermes_pkce", read_hermes_oauth_credentials())]
  2315. if adopt_external_logins_enabled():
  2316. sources.append(("claude_code", read_claude_code_credentials()))
  2317. else:
  2318. # Singleton-seeded rows are otherwise never pruned; the opt-out must also drop the row an
  2319. # earlier (adopting) process persisted, or it keeps rotating a login Hermes no longer reads.
  2320. seed.changed |= _retain_sources_not_in(seed.entries, {"claude_code"})
  2321. for source_name, creds in sources:
  2322. if creds and creds.get("accessToken"):
  2323. seed.upsert(source_name, {
  2324. "auth_type": AUTH_TYPE_OAUTH,
  2325. "access_token": creds.get("accessToken", ""),
  2326. "refresh_token": creds.get("refreshToken"),
  2327. "expires_at_ms": creds.get("expiresAt"),
  2328. "label": label_from_token(creds.get("accessToken", ""), source_name),
  2329. })
  2330. def _seed_nous_singleton(seed: _Seeder, auth_store: dict[str, Any]) -> None:
  2331. state, source_path = _load_provider_state_with_source(auth_store, "nous")
  2332. if owned_profile_reads_root_state(auth_store, "nous", source_path):
  2333. # e.g. an agent_key-only row surviving a fork strip/heal
  2334. return
  2335. from hermes_cli.anon_auth import guest_without_free_tier # kept for connectors only: no inference rows
  2336. has_runtime_material = not guest_without_free_tier(state) and bool(isinstance(state, dict) and (
  2337. str(state.get("access_token") or "").strip() or str(state.get("agent_key") or "").strip()))
  2338. if state and not has_runtime_material:
  2339. seed.changed |= _retain_sources_not_in(seed.entries, {"device_code", "manual:device_code"})
  2340. if not (state and has_runtime_material):
  2341. return
  2342. # Prefer a user-supplied label embedded in the singleton state (``hermes
  2343. # auth add nous --label <name>``) over the token-derived fingerprint.
  2344. custom_label = str(state.get("label") or "").strip()
  2345. seed.upsert("device_code", {
  2346. "auth_type": AUTH_TYPE_OAUTH,
  2347. "access_token": state.get("access_token", ""),
  2348. "refresh_token": state.get("refresh_token"),
  2349. "expires_at": state.get("expires_at"),
  2350. "token_type": state.get("token_type"),
  2351. "scope": state.get("scope"),
  2352. "client_id": state.get("client_id"),
  2353. "portal_base_url": state.get("portal_base_url"),
  2354. "inference_base_url": state.get("inference_base_url"),
  2355. "agent_key": state.get("agent_key"),
  2356. "agent_key_expires_at": state.get("agent_key_expires_at"),
  2357. # Refresh timestamps let freshness-sensitive consumers (self-heal
  2358. # hooks, pruning by age) tell just-refreshed credentials from stale
  2359. # ones (#15099).
  2360. **{key: state.get(key) for key in _NOUS_EXTRA_STATE_KEYS},
  2361. "tls": state.get("tls") if isinstance(state.get("tls"), dict) else None,
  2362. "label": custom_label or label_from_token(state.get("access_token", ""), "device_code"),
  2363. })
  2364. def _seed_profile_singleton(seed: _Seeder) -> None:
  2365. """Rows a provider profile seeds itself (``seed_pool_credentials``): ambient/env tokens it
  2366. validates or exchanges. The profile gates its own work on suppression and explicit config."""
  2367. from providers import get_provider_profile
  2368. profile = get_provider_profile(seed.provider)
  2369. if profile is None or not _profile_seeds_pool(seed.provider):
  2370. return
  2371. from hermes_cli.auth import is_provider_explicitly_configured, suppressed_credential_sources
  2372. rows = profile.seed_pool_credentials( # values, not callbacks: a hosted profile gets plain JSON
  2373. suppressed_sources=suppressed_credential_sources(seed.provider),
  2374. explicitly_configured=is_provider_explicitly_configured(seed.provider))
  2375. pconfig = PROVIDER_REGISTRY.get(seed.provider)
  2376. for row in rows or ():
  2377. seed.upsert(row["source"], {
  2378. "auth_type": AUTH_TYPE_API_KEY,
  2379. "access_token": row["access_token"],
  2380. "base_url": row.get("base_url") or (pconfig.inference_base_url if pconfig else ""),
  2381. "label": row.get("label") or row["source"],
  2382. })
  2383. def _profile_seeds_pool(provider: str) -> bool:
  2384. """Whether the provider's profile owns pool seeding (the generic env loop must not overwrite it)."""
  2385. from providers import get_provider_profile
  2386. from providers.base import ProviderProfile
  2387. profile = get_provider_profile(provider)
  2388. return profile is not None and type(profile).seed_pool_credentials is not ProviderProfile.seed_pool_credentials
  2389. def _seed_qwen_singleton(seed: _Seeder) -> None:
  2390. # Qwen OAuth tokens live in ~/.qwen/oauth_creds.json (written by the Qwen
  2391. # CLI). refresh_if_expiring=False avoids network calls during pool loading.
  2392. try:
  2393. from hermes_cli.auth import resolve_qwen_runtime_credentials
  2394. creds = resolve_qwen_runtime_credentials(refresh_if_expiring=False)
  2395. token = creds.get("api_key", "")
  2396. if token:
  2397. source_name = creds.get("source", "qwen-cli")
  2398. seed.upsert(source_name, {
  2399. "auth_type": AUTH_TYPE_OAUTH,
  2400. "access_token": token,
  2401. "expires_at_ms": creds.get("expires_at_ms"),
  2402. "base_url": creds.get("base_url", ""),
  2403. "label": creds.get("auth_file", source_name),
  2404. })
  2405. except Exception as exc:
  2406. logger.debug("Qwen OAuth token seed failed: %s", exc)
  2407. def _seed_minimax_singleton(seed: _Seeder) -> None:
  2408. # Read the raw auth.json state rather than resolve_minimax_oauth_runtime_credentials,
  2409. # which always refreshes on expiry (surprise network calls during discovery).
  2410. try:
  2411. from hermes_cli.auth import get_provider_auth_state
  2412. state = get_provider_auth_state("minimax-oauth")
  2413. if not (state and state.get("access_token")):
  2414. return
  2415. expires_at_ms = None
  2416. try:
  2417. raw = state.get("expires_at", "")
  2418. if raw:
  2419. expires_at_ms = int(datetime.fromisoformat(raw).timestamp() * 1000)
  2420. except Exception:
  2421. expires_at_ms = None
  2422. seed.upsert("oauth", {
  2423. "auth_type": AUTH_TYPE_OAUTH,
  2424. "access_token": state["access_token"],
  2425. "refresh_token": state.get("refresh_token"),
  2426. "expires_at_ms": expires_at_ms,
  2427. "base_url": str(state.get("inference_base_url", "") or "").rstrip("/"),
  2428. "label": state.get("label", "") or label_from_token(state.get("access_token", ""), "oauth"),
  2429. })
  2430. except Exception as exc:
  2431. logger.debug("MiniMax OAuth token seed failed: %s", exc)
  2432. def _seed_tokens_singleton(seed: _Seeder, auth_store: dict[str, Any]) -> None:
  2433. """Codex / xAI: surface the auth.json ``providers.<id>.tokens`` singleton as ``device_code``.
  2434. Hermes owns its own Codex auth state and does NOT auto-import
  2435. ~/.codex/auth.json: refresh tokens are single-use, so sharing them with
  2436. Codex CLI / VS Code causes refresh_token_reused races. Adoption is an
  2437. explicit one-time prompt via `hermes auth openai-codex`.
  2438. """
  2439. state, source_path = _load_provider_state_with_source(auth_store, seed.provider)
  2440. if owned_profile_reads_root_state(auth_store, seed.provider, source_path):
  2441. return
  2442. tokens = state.get("tokens") if isinstance(state, dict) else None
  2443. if not (isinstance(tokens, dict) and tokens.get("access_token")):
  2444. return
  2445. if seed.provider == "openai-codex":
  2446. base_url = auth_mod.DEFAULT_CODEX_BASE_URL
  2447. custom_label = str(state.get("label") or "").strip()
  2448. else:
  2449. base_url = auth_mod.DEFAULT_XAI_OAUTH_BASE_URL
  2450. custom_label = ""
  2451. seed.upsert("device_code", {
  2452. "auth_type": AUTH_TYPE_OAUTH,
  2453. "access_token": tokens.get("access_token", ""),
  2454. "refresh_token": tokens.get("refresh_token"),
  2455. "base_url": base_url,
  2456. "last_refresh": state.get("last_refresh"),
  2457. "label": custom_label or label_from_token(tokens.get("access_token", ""), "device_code"),
  2458. })
  2459. def _seed_from_singletons(provider: str, entries: list[PooledCredential]) -> tuple[bool, set[str]]:
  2460. seed = _Seeder(provider, entries)
  2461. auth_store = _load_auth_store()
  2462. if provider == "anthropic":
  2463. _seed_anthropic_singletons(seed)
  2464. elif provider == "nous":
  2465. _seed_nous_singleton(seed, auth_store)
  2466. elif provider == "qwen-oauth":
  2467. _seed_qwen_singleton(seed)
  2468. elif provider == "minimax-oauth":
  2469. _seed_minimax_singleton(seed)
  2470. elif provider in _TOKENS_SINGLETON_PROVIDERS:
  2471. # `hermes auth remove openai-codex` suppresses device_code; without
  2472. # this gate the removal is undone on the next load_pool().
  2473. if provider == "openai-codex" and seed.is_suppressed(provider, "device_code"):
  2474. return seed.result
  2475. _seed_tokens_singleton(seed, auth_store)
  2476. else:
  2477. _seed_profile_singleton(seed)
  2478. return seed.result
  2479. def get_env_prefer_dotenv(key: str) -> str:
  2480. """Resolve a credential env var, preferring ~/.hermes/.env over os.environ.
  2481. The user's config file is authoritative; stale env vars from parent
  2482. processes (Codex CLI, test scripts) must not override deliberate .env
  2483. changes. load_env() memoizes on mtime, so per-call reads cost a stat().
  2484. An unresolved ``op://`` reference in .env yields to the already-resolved
  2485. value from the active secret scope (set by apply_onepassword_secrets());
  2486. otherwise every provider auth attempt would receive a URL instead of a key.
  2487. """
  2488. env_file = load_env()
  2489. raw = env_file.get(key, "").strip()
  2490. scoped_value = (_get_secret(key, "") or "").strip()
  2491. if raw.startswith("op://") and scoped_value:
  2492. return scoped_value
  2493. return raw or scoped_value
  2494. # Providers already warned about env-key -> pool ingestion, once per process
  2495. # (#81952 expected-behavior #3).
  2496. _ENV_INGESTION_WARNED: set[str] = set()
  2497. def _warn_env_ingestion_once(provider: str, env_var: str) -> None:
  2498. """WARN once per process per provider when an env credential is ingested into a paid pool.
  2499. Auto-ingesting OPENROUTER_API_KEY is what ARMS silent OpenRouter spend —
  2500. every downstream auto-detect keys off the pool having credentials.
  2501. Ingestion stays allowed (an exported key is arguable intent) but must
  2502. never be silent.
  2503. """
  2504. if provider in _ENV_INGESTION_WARNED:
  2505. return
  2506. _ENV_INGESTION_WARNED.add(provider)
  2507. logger.warning(
  2508. "Ingested %s from environment into the %s credential pool — this "
  2509. "enables %s spend. Remove the key or run "
  2510. "hermes auth remove %s <n> to suppress.",
  2511. env_var,
  2512. provider,
  2513. "OpenRouter" if provider == "openrouter" else provider,
  2514. provider,
  2515. )
  2516. def _env_payload(*, env_var: str, token: str, base_url: str) -> dict[str, Any]:
  2517. payload: dict[str, Any] = {
  2518. "auth_type": AUTH_TYPE_API_KEY,
  2519. "access_token": token,
  2520. "base_url": base_url,
  2521. "label": env_var,
  2522. }
  2523. try:
  2524. from hermes_cli.env_loader import get_secret_source
  2525. source_label = get_secret_source(env_var)
  2526. except Exception:
  2527. source_label = None
  2528. secret_source = str(source_label).strip() if source_label else None
  2529. if secret_source:
  2530. payload["secret_source"] = secret_source
  2531. return payload
  2532. # Region-specific endpoints inferred from the key itself.
  2533. _ENV_BASE_URL_RESOLVERS = {
  2534. "kimi-coding": _resolve_kimi_base_url,
  2535. "zai": _resolve_zai_base_url,
  2536. }
  2537. def _env_key_var_candidates(env_vars: list[str], entries: list[PooledCredential]) -> list[str]:
  2538. """*env_vars*, their numbered siblings, and the ``env:VAR`` names already persisted.
  2539. ``VAR_2``, ``VAR_3``, ... are tried for every declared VAR until the first
  2540. one that does not resolve, so a `.env` or secret-manager project can back a
  2541. whole rotation pool with no config: setting ``NVIDIA_API_KEY_2`` is the
  2542. whole opt-in (#76593).
  2543. Env-backed rows are written to auth.json without their secret and
  2544. re-hydrated on every load; a row whose VAR the registry does not
  2545. declare would otherwise stay empty forever and be silently dropped
  2546. from rotation by ``_available_entries``.
  2547. """
  2548. names = list(env_vars)
  2549. for base in env_vars:
  2550. n = 2
  2551. while get_env_prefer_dotenv(f"{base}_{n}"):
  2552. names.append(f"{base}_{n}")
  2553. n += 1
  2554. for entry in entries:
  2555. if entry.source.startswith("env:"):
  2556. env_name = entry.source.split(":", 1)[1].strip()
  2557. if env_name and env_name not in names:
  2558. names.append(env_name)
  2559. return names
  2560. def _seed_from_env(provider: str, entries: list[PooledCredential]) -> tuple[bool, set[str]]:
  2561. seed = _Seeder(provider, entries)
  2562. # A profile that seeds its own rows may exchange the raw env token for a runtime one; the
  2563. # generic loop would re-read the same env var and overwrite it with the RAW token.
  2564. if _profile_seeds_pool(provider):
  2565. return seed.result
  2566. if provider == "openrouter":
  2567. for env_var in _env_key_var_candidates(["OPENROUTER_API_KEY"], entries):
  2568. token = get_env_prefer_dotenv(env_var)
  2569. if token and seed.upsert(
  2570. f"env:{env_var}",
  2571. _env_payload(env_var=env_var, token=token, base_url=OPENROUTER_BASE_URL),
  2572. ):
  2573. _warn_env_ingestion_once(provider, env_var)
  2574. return seed.result
  2575. pconfig = PROVIDER_REGISTRY.get(provider)
  2576. if not pconfig or pconfig.auth_type != AUTH_TYPE_API_KEY:
  2577. return seed.result
  2578. env_url = ""
  2579. if pconfig.base_url_env_var:
  2580. env_url = get_env_prefer_dotenv(pconfig.base_url_env_var).rstrip("/")
  2581. env_vars = list(pconfig.api_key_env_vars)
  2582. if provider == "anthropic":
  2583. env_vars = ["ANTHROPIC_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]
  2584. env_vars = _env_key_var_candidates(env_vars, entries)
  2585. resolve_base_url = _ENV_BASE_URL_RESOLVERS.get(provider)
  2586. for env_var in env_vars:
  2587. token = get_env_prefer_dotenv(env_var)
  2588. if not token:
  2589. continue
  2590. base_url = env_url or pconfig.inference_base_url
  2591. if resolve_base_url is not None:
  2592. base_url = resolve_base_url(token, pconfig.inference_base_url, env_url)
  2593. seed.upsert(f"env:{env_var}", _env_payload(env_var=env_var, token=token, base_url=base_url))
  2594. return seed.result
  2595. def _prune_stale_seeded_entries(
  2596. entries: list[PooledCredential],
  2597. active_sources: set[str],
  2598. *,
  2599. prune_env_sources: bool = True,
  2600. ) -> bool:
  2601. def _is_prunable(entry: PooledCredential) -> bool:
  2602. # ``env:*`` entries are persisted references re-hydrated on every load.
  2603. # A process that merely lacks the env var must NOT delete the on-disk
  2604. # entry for every other process (#9331); prune only when explicitly
  2605. # requested (an `hermes auth` command that confirmed the source is gone).
  2606. if entry.source.startswith("env:"):
  2607. return prune_env_sources
  2608. # File-backed singletons and Hermes PKCE disappear when their backing file is gone.
  2609. return is_borrowed_credential_source(entry.source, entry.provider) or entry.source == "hermes_pkce"
  2610. retained = [
  2611. entry
  2612. for entry in entries
  2613. if _is_manual_source(entry.source) or entry.source in active_sources or not _is_prunable(entry)
  2614. ]
  2615. if len(retained) == len(entries):
  2616. return False
  2617. entries[:] = retained
  2618. return True
  2619. def _seed_custom_pool(pool_key: str, entries: list[PooledCredential]) -> tuple[bool, set[str]]:
  2620. """Seed a custom endpoint pool from custom_providers config and model config."""
  2621. seed = _Seeder(pool_key, entries)
  2622. cp_config = _get_custom_provider_config(pool_key)
  2623. if cp_config:
  2624. api_key = str(cp_config.get("api_key") or "").strip()
  2625. name = str(cp_config.get("name") or "").strip()
  2626. if api_key:
  2627. seed.upsert(f"config:{name}", {
  2628. "auth_type": AUTH_TYPE_API_KEY,
  2629. "access_token": api_key,
  2630. "base_url": _norm_url(cp_config.get("base_url")),
  2631. "label": name or f"config:{name}",
  2632. })
  2633. # Seed from model.api_key when model.provider=='custom' and model.base_url matches
  2634. try:
  2635. config = _load_config_safe()
  2636. model_cfg = config.get("model") if config else None
  2637. if isinstance(model_cfg, dict):
  2638. model_provider = str(model_cfg.get("provider") or "").strip().lower()
  2639. model_base_url = _norm_url(model_cfg.get("base_url"))
  2640. model_api_key = next(
  2641. (v.strip() for k in ("api_key", "api") for v in (model_cfg.get(k),) if isinstance(v, str) and v.strip()),
  2642. "",
  2643. )
  2644. if model_provider == "custom" and model_base_url and model_api_key:
  2645. # The pool may be keyed under the durable ``providers.<key>``
  2646. # slug or legacy ``custom:<name>``; accept any candidate, or
  2647. # seeding is skipped when the pool holds the other identity.
  2648. # Check if this model's base_url matches our custom provider. See #100413.
  2649. matched_keys = {
  2650. str(key).strip().lower() for key in custom_provider_pool_key_candidates(model_base_url)
  2651. }
  2652. if pool_key in matched_keys:
  2653. seed.upsert("model_config", {
  2654. "auth_type": AUTH_TYPE_API_KEY,
  2655. "access_token": model_api_key,
  2656. "base_url": model_base_url,
  2657. "label": "model_config",
  2658. })
  2659. except Exception:
  2660. pass
  2661. return seed.result
  2662. def load_pool(provider: str) -> CredentialPool:
  2663. provider = (provider or "").strip().lower()
  2664. if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS:
  2665. # One-time heal for installs that forked this grant across profiles
  2666. # before the clone-strip / root write-through existed (#100339).
  2667. auth_mod.heal_forked_single_use_oauth_grants(provider)
  2668. raw_entries = read_credential_pool(provider)
  2669. disk_ids = {e.get("id") for e in raw_entries if isinstance(e, dict) and e.get("id")}
  2670. # Ownership (auth.json read) after the heal above; re-read at the tail only if _persist() ran.
  2671. owns_provider: Optional[bool] = None
  2672. changed = any(
  2673. isinstance(payload, dict) and sanitize_borrowed_credential_payload(payload, provider) != payload
  2674. for payload in raw_entries
  2675. )
  2676. entries = [PooledCredential.from_dict(provider, payload) for payload in raw_entries]
  2677. raw_needs_auth_normalization = any(
  2678. isinstance(payload, dict)
  2679. and _normalize_pool_auth_type(
  2680. provider, payload.get("access_token"), payload.get("auth_type", AUTH_TYPE_API_KEY),
  2681. ) != payload.get("auth_type", AUTH_TYPE_API_KEY)
  2682. for payload in raw_entries
  2683. )
  2684. if raw_needs_auth_normalization:
  2685. # A profile may be reading this provider from the global-root fallback.
  2686. # Keep that fallback read-only: only the owning store may rewrite these
  2687. # rows; loading the default/root profile heals global rows.
  2688. active_pool = _load_auth_store().get("credential_pool")
  2689. active_entries = active_pool.get(provider) if isinstance(active_pool, dict) else None
  2690. changed |= bool(active_entries)
  2691. if provider.startswith(CUSTOM_POOL_PREFIX):
  2692. custom_changed, custom_sources = _seed_custom_pool(provider, entries)
  2693. changed |= custom_changed
  2694. changed |= _prune_stale_seeded_entries(entries, custom_sources)
  2695. else:
  2696. singleton_changed, singleton_sources = _seed_from_singletons(provider, entries)
  2697. env_changed, env_sources = _seed_from_env(provider, entries)
  2698. changed |= singleton_changed or env_changed
  2699. # ``load_pool()`` is a non-destructive read for env-seeded entries
  2700. # (#9331); file-backed singletons still prune when their file is gone.
  2701. if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS and disk_ids:
  2702. owns_provider = _profile_owns_pool_provider(provider)
  2703. if owns_provider is False:
  2704. # Rows read through the global-root fallback are seeded from the
  2705. # ROOT's singleton files, which this profile cannot see; pruning
  2706. # them would hide (and, via write-through, delete) the shared
  2707. # grant. The root's own load_pool() prunes.
  2708. borrowed = [e for e in entries if e.id in disk_ids]
  2709. others = [e for e in entries if e.id not in disk_ids]
  2710. changed |= _prune_stale_seeded_entries(
  2711. others, singleton_sources | env_sources, prune_env_sources=False,
  2712. )
  2713. entries[:] = borrowed + others
  2714. else:
  2715. changed |= _prune_stale_seeded_entries(
  2716. entries, singleton_sources | env_sources, prune_env_sources=False,
  2717. )
  2718. changed |= _normalize_pool_priorities(provider, entries)
  2719. pool = CredentialPool(provider, entries)
  2720. pool._persisted_token_pairs = auth_mod._token_pairs_by_id(raw_entries)
  2721. if changed:
  2722. pool._persist(removed_ids=sorted(disk_ids - {entry.id for entry in entries}))
  2723. # Remember the root's borrowed rows so a later ``add_entry`` in this
  2724. # profile leaves them out of the profile's own store (#100339).
  2725. # No disk rows -> nothing borrowed; the ``set()`` default already applies.
  2726. if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS and disk_ids:
  2727. # Reuse the pre-persist ownership answer unless _persist() just rewrote
  2728. # the store (it can give the profile its own rows); nothing else between
  2729. # the two checks touches auth.json.
  2730. if changed:
  2731. owns_provider = _profile_owns_pool_provider(provider)
  2732. if not owns_provider:
  2733. pool._borrowed_root_ids = set(disk_ids)
  2734. return pool