auth_codex.py 59 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132
  1. """OpenAI Codex OAuth: token store, refresh, quota probe, device-code login.
  2. Tokens live in ~/.hermes/auth.json, NOT ~/.codex/: Hermes keeps its own Codex OAuth session
  3. separate from the Codex CLI / VS Code extension so one app's refresh-token rotation cannot
  4. invalidate the other's session.
  5. Split out of ``hermes_cli/auth.py``; origin helpers are imported lazily inside each function
  6. so ``hermes_cli.auth.<name>`` patches still intercept (and no import cycle).
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import hashlib
  11. import json
  12. import os
  13. import threading
  14. import time
  15. from contextlib import suppress
  16. from pathlib import Path
  17. from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
  18. from hermes_cli.auth_constants import (
  19. _decode_jwt_claims, AUTH_LOCK_TIMEOUT_SECONDS, AuthError,
  20. CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS, CODEX_OAUTH_CLIENT_ID, CODEX_OAUTH_TOKEN_URL,
  21. CODEX_RATE_LIMITED_CODE, DEFAULT_CODEX_BASE_URL, _codex_err, httpx)
  22. from hermes_cli.version_info import hermes_user_agent
  23. from utils import env_float, normalize_proxy_env_vars
  24. if TYPE_CHECKING: # annotation-only; the runtime import would be a cycle
  25. from hermes_cli.auth import ProviderConfig
  26. # Log-record parity with the origin module (caplog tests pin "hermes_cli.auth").
  27. logger = logging.getLogger("hermes_cli.auth")
  28. # ``{relogin}`` is filled at raise time with the profile-aware sign-in command: a bare
  29. # ``hermes auth`` from a named profile re-signs the ROOT store (93889b770da, #114012).
  30. _MISSING_ACCESS_TOKEN_MSG = "Codex auth is missing access_token. Run `{relogin}` to re-authenticate."
  31. _MISSING_REFRESH_TOKEN_MSG = "Codex auth is missing refresh_token. Run `{relogin}` to re-authenticate."
  32. _NO_CREDENTIALS_MSG = "No Codex credentials stored. Run `{relogin}` to authenticate."
  33. def _codex_relogin_command() -> str:
  34. from agent.turn_failure_copy import oauth_relogin_command
  35. return oauth_relogin_command("openai-codex")
  36. def _parse_retry_after_seconds(headers: Any) -> Optional[int]:
  37. """Best-effort parse of a ``Retry-After`` header into whole seconds."""
  38. from agent.retry_utils import parse_retry_after_seconds
  39. seconds = parse_retry_after_seconds(headers)
  40. return None if seconds is None else int(seconds)
  41. def _stripped(value: Any) -> str:
  42. return str(value or "").strip()
  43. def _clear_pool_entry_status(entry: dict[str, Any]) -> None:
  44. """Reset a pool entry's cooldown / last-error metadata to healthy."""
  45. from hermes_cli.auth import _POOL_STATUS_FIELDS
  46. for status_field in _POOL_STATUS_FIELDS:
  47. entry[status_field] = None
  48. def _codex_access_token_is_expiring(access_token: Any, skew_seconds: int) -> bool:
  49. exp = _decode_jwt_claims(access_token).get("exp")
  50. return isinstance(exp, (int, float)) and float(exp) <= (time.time() + max(0, int(skew_seconds)))
  51. def _codex_base_url() -> str:
  52. return os.getenv("HERMES_CODEX_BASE_URL", "").strip().rstrip("/") or DEFAULT_CODEX_BASE_URL
  53. def _codex_pool_route_base_url(entry_base_url: Optional[str] = "") -> str:
  54. """Base URL the chat route sends a pooled Codex credential to — the same rule
  55. ``runtime_provider._pool_entry_mode_and_url`` applies (``HERMES_CODEX_BASE_URL`` > ``model.base_url``
  56. while the row still carries the canonical URL > the row's own URL). A pooled gateway key belongs to
  57. that host only; composing it with the ambient default sends it to chatgpt.com (#121486)."""
  58. base = _stripped(entry_base_url).rstrip("/")
  59. try:
  60. from hermes_cli.config import load_config_readonly
  61. from hermes_cli.runtime_provider import _pool_entry_mode_and_url
  62. model_cfg = load_config_readonly().get("model")
  63. return _pool_entry_mode_and_url(
  64. "openai-codex", None, model_cfg if isinstance(model_cfg, dict) else {}, "", base)[1]
  65. except Exception:
  66. logger.debug("Codex pool route base resolution failed", exc_info=True)
  67. # Profile-scoped override only (never the raw process env: a multiplexed sibling's gateway).
  68. with suppress(Exception):
  69. from agent.secret_scope import get_secret_str
  70. base = _stripped(get_secret_str("HERMES_CODEX_BASE_URL", "")).rstrip("/") or base
  71. return base or DEFAULT_CODEX_BASE_URL
  72. def _codex_runtime_result(
  73. api_key: str, *, source: str, last_refresh: Optional[str], base_url: Optional[str] = None) -> dict[str, Any]:
  74. return {
  75. "provider": "openai-codex", "base_url": base_url or _codex_base_url(), "api_key": api_key,
  76. "source": source, "last_refresh": last_refresh, "auth_mode": "chatgpt"}
  77. def _load_auth_store_maybe_locked(lock: bool) -> dict[str, Any]:
  78. """Load the auth store, taking the cross-process lock unless the caller already holds it."""
  79. from hermes_cli.auth import _auth_store_lock, _load_auth_store
  80. if lock:
  81. with _auth_store_lock():
  82. return _load_auth_store()
  83. return _load_auth_store()
  84. def _read_codex_tokens(*, _lock: bool = True) -> dict[str, Any]:
  85. """Read Codex OAuth tokens from Hermes auth store (~/.hermes/auth.json)."""
  86. from hermes_cli.auth import _load_provider_state, _nonempty_str
  87. auth_store = _load_auth_store_maybe_locked(_lock)
  88. state = _load_provider_state(auth_store, "openai-codex")
  89. if not state:
  90. raise _codex_err(_NO_CREDENTIALS_MSG.format(relogin=_codex_relogin_command()),
  91. "codex_auth_missing", relogin=True)
  92. tokens = state.get("tokens")
  93. if not isinstance(tokens, dict):
  94. raise _codex_err(
  95. f"Codex auth state is missing tokens. Run `{_codex_relogin_command()}` to re-authenticate.",
  96. "codex_auth_invalid_shape", relogin=True)
  97. if not _nonempty_str(tokens.get("access_token")):
  98. raise _codex_err(_MISSING_ACCESS_TOKEN_MSG.format(relogin=_codex_relogin_command()),
  99. "codex_auth_missing_access_token", relogin=True)
  100. if not _nonempty_str(tokens.get("refresh_token")):
  101. raise _codex_err(_MISSING_REFRESH_TOKEN_MSG.format(relogin=_codex_relogin_command()),
  102. "codex_auth_missing_refresh_token", relogin=True)
  103. return {"tokens": tokens, "last_refresh": state.get("last_refresh")}
  104. def _sync_codex_pool_entries(
  105. auth_store: dict[str, Any], tokens: dict[str, str], last_refresh: Optional[str],
  106. previous_singleton_tokens: Optional[dict[str, str]] = None) -> None:
  107. """Mirror a fresh Codex re-auth into the credential_pool OAuth entries.
  108. ``device_code`` (the singleton-seeded entry from ``hermes setup`` / the model picker) is always
  109. synced. ``manual:device_code`` (``hermes auth add openai-codex``) is synced only when its
  110. access_token equals the PREVIOUS singleton token — a legacy alias of the singleton; an entry
  111. with its own token material is an independent account and must be left alone. ``manual:api_key``
  112. and any other source are independent credentials and are never overwritten by a re-auth.
  113. See #33000, #39236.
  114. The original #33538 fix refreshed every ``manual:device_code`` entry unconditionally. That worked when
  115. ``manual:device_code`` only meant "legacy alias of the singleton", but the same source string is now
  116. also produced by independent-account additions, and the broad sync silently clobbered distinct accounts
  117. with the latest-authenticated token pair. The access_token-match check distinguishes the two cases
  118. without changing the source-string contract.
  119. """
  120. access_token = tokens.get("access_token")
  121. if not access_token:
  122. return
  123. refresh_token = tokens.get("refresh_token")
  124. entries = _pool_entries(auth_store, "openai-codex")
  125. if entries is None:
  126. return
  127. # None/empty prev_at → no manual entry can be an alias (right default for a first-ever save).
  128. prev_at = (previous_singleton_tokens or {}).get("access_token") or None
  129. for entry in _codex_pool_dicts(entries):
  130. source = entry.get("source")
  131. is_alias = source == "manual:device_code" and bool(
  132. prev_at and entry.get("access_token") == prev_at)
  133. if not (source == "device_code" or is_alias):
  134. continue
  135. entry["access_token"] = access_token
  136. if refresh_token:
  137. entry["refresh_token"] = refresh_token
  138. if last_refresh:
  139. entry["last_refresh"] = last_refresh
  140. _clear_pool_entry_status(entry)
  141. def _save_codex_tokens(
  142. tokens: dict[str, str], last_refresh: str | None = None, label: str | None = None, *,
  143. set_active: bool = True, write_through: bool = False,
  144. ) -> None:
  145. """Save Codex OAuth tokens to the auth store the grant was resolved FROM.
  146. Codex refresh tokens are single-use with rotation-family reuse detection. A profile without its
  147. own ``providers.openai-codex`` block reads root's grant via the fallback, so a refresh under that
  148. profile must rotate ROOT's chain — singleton AND ``credential_pool`` entries — or root keeps the
  149. consumed refresh token, the next process replays it and OpenAI revokes the whole family
  150. (#87503). Root-only write-back: a profile copy would shadow root and disable the write-through
  151. on the next refresh (#74339). Mirrors the xAI source-aware save.
  152. Only a token REFRESH passes ``write_through=True``: a fresh login or import under a profile
  153. is the profile's own grant and must not overwrite the root account it was borrowing.
  154. ``set_active=False`` stores credentials for a side tool (image gen) without making Codex the
  155. active inference provider.
  156. """
  157. from hermes_cli.auth import (
  158. _auth_file_path, _load_auth_store, _provider_state_transaction, _same_path,
  159. _save_auth_store, _store_provider_state, _utc_now_z)
  160. if last_refresh is None:
  161. last_refresh = _utc_now_z()
  162. with _provider_state_transaction("openai-codex") as (auth_store, state, source_path):
  163. state = dict(state) if state else {}
  164. # Capture the previous singleton tokens BEFORE overwriting: the pool sync uses them to
  165. # tell legacy singleton-aliases (refresh) from independent ``auth add`` accounts (keep).
  166. previous_singleton_tokens = (
  167. state.get("tokens") if isinstance(state.get("tokens"), dict) else None)
  168. state.update(tokens=tokens, last_refresh=last_refresh, auth_mode="chatgpt")
  169. if label and str(label).strip():
  170. state["label"] = str(label).strip()
  171. target_store, target_path = auth_store, None
  172. if write_through and source_path is not None and not _same_path(source_path, _auth_file_path()):
  173. # Root-borrowed grant: the transaction already holds root's lock, so write the rotated
  174. # chain into ROOT's store (never set_active — a refresh is not a provider choice).
  175. target_store, target_path, set_active = _load_auth_store(source_path), source_path, False
  176. _store_provider_state(target_store, "openai-codex", state, set_active=set_active)
  177. _sync_codex_pool_entries(
  178. target_store, tokens, last_refresh, previous_singleton_tokens=previous_singleton_tokens)
  179. _save_auth_store(target_store, target_path=target_path)
  180. def _recover_codex_tokens_from_cli(
  181. reason: str, observed_access_token: Optional[str] = None) -> Optional[dict[str, str]]:
  182. """Adopt a valid Codex CLI token pair into Hermes auth, if available.
  183. Automatic adoption only; the interactive import offer in ``_login_openai_codex`` asks first and is
  184. not subject to ``auth.adopt_external_logins``.
  185. ``observed_access_token`` is the singleton access_token (or None) the caller saw when it decided
  186. the credential needs repair. Recovery repairs THAT credential and nothing else (#73667): a Codex
  187. Desktop/CLI login into another ChatGPT workspace must not replace it silently, and a concurrent
  188. explicit re-auth must not be overwritten, so the save is a compare-and-swap under the store lock.
  189. """
  190. from agent.credential_pool import _codex_principal_identity
  191. from agent.credential_sources import adopt_external_logins_enabled
  192. from hermes_cli.auth import _import_codex_cli_tokens, _provider_state_transaction, _save_codex_tokens
  193. if not adopt_external_logins_enabled():
  194. return None
  195. imported = _import_codex_cli_tokens()
  196. # Require BOTH tokens before adopting: persisting a payload without a usable refresh_token
  197. # would only break the next refresh cycle.
  198. if not (imported and _stripped(imported.get("access_token"))
  199. and _stripped(imported.get("refresh_token"))):
  200. return None
  201. observed = _stripped(observed_access_token) or None
  202. with _provider_state_transaction("openai-codex") as (_store, state, _source):
  203. stored = (state or {}).get("tokens")
  204. stored = stored if isinstance(stored, dict) else {}
  205. if (_stripped(stored.get("access_token")) or None) != observed:
  206. logger.info("Codex CLI recovery skipped (%s): the credential was re-authenticated meanwhile.", reason)
  207. return None
  208. known = _codex_principal_identity(observed)
  209. if known and _codex_principal_identity(imported["access_token"]) not in (None, known):
  210. logger.warning(
  211. "Codex CLI recovery refused (%s): the Codex CLI login belongs to a different ChatGPT "
  212. "workspace than the Hermes credential. Run `%s` to re-authenticate it.",
  213. reason, _codex_relogin_command())
  214. return None
  215. logger.info("Codex auth recovered from Codex CLI auth.json (%s).", reason)
  216. _save_codex_tokens(imported) # nested: the per-path lock is reentrant
  217. return dict(imported)
  218. def _refresh_payload_access_token(
  219. response: httpx.Response, *, provider: str, invalid_json: tuple[str, str],
  220. invalid_response: Optional[tuple[str, str]], missing_access: tuple[str, str],
  221. relogin_required: bool = True, invalid_json_relogin: Optional[bool] = None,
  222. strict_str: bool = True) -> tuple[dict[str, Any], str]:
  223. """Parse a 200 token-refresh response; return ``(payload, stripped access_token)``.
  224. Each ``(message, code)`` pair keeps the provider's historical wording; ``{exc}`` in
  225. *invalid_json*'s message is formatted with the JSON error. *strict_str* rejects non-string
  226. access tokens; otherwise they are ``str()``-coerced.
  227. """
  228. def _err(message: str, code: str, relogin: bool = relogin_required) -> AuthError:
  229. return AuthError(message, provider=provider, code=code, relogin_required=relogin)
  230. try:
  231. payload = response.json()
  232. except Exception as exc:
  233. relogin = relogin_required if invalid_json_relogin is None else invalid_json_relogin
  234. raise _err(invalid_json[0].format(exc=exc), invalid_json[1], relogin) from exc
  235. if not isinstance(payload, dict):
  236. if invalid_response is not None:
  237. raise _err(*invalid_response)
  238. payload = {}
  239. access = payload.get("access_token")
  240. if strict_str:
  241. access = access.strip() if isinstance(access, str) else ""
  242. else:
  243. access = _stripped(access)
  244. if not access:
  245. raise _err(*missing_access)
  246. return payload, access
  247. _SSL_TROUBLE_MARKERS = ("[SSL:", "_ssl.c", "UNEXPECTED_EOF")
  248. def _ssl_interop_hint(exc: BaseException) -> str:
  249. """Actionable hint for device-login transport errors that look like TLS middlebox interference.
  250. OpenSSL 3.5+ advertises post-quantum hybrid groups (e.g. X25519MLKEM768) by default, and some
  251. intercepting middleboxes reject the resulting larger TLS 1.3 ClientHello — while curl, using a
  252. different TLS stack, still works, so the failure masquerades as a Codex outage (#106384).
  253. httpx wraps the ``ssl.SSLError`` in a ``ConnectError``/``ConnectTimeout`` whose text usually
  254. repeats the OpenSSL message; the cause chain is checked too in case it doesn't.
  255. """
  256. import ssl
  257. chain = (exc, exc.__cause__, exc.__context__)
  258. if not any(
  259. isinstance(err, ssl.SSLError) or any(marker in str(err) for marker in _SSL_TROUBLE_MARKERS)
  260. for err in chain if err is not None
  261. ):
  262. return ""
  263. return (
  264. " This looks like a TLS handshake failure rather than a Codex outage: some networks reject"
  265. " the larger TLS 1.3 ClientHello that OpenSSL 3.5+ sends by default (post-quantum hybrid"
  266. " groups). Workaround: point OPENSSL_CONF at a config restricting Groups to classic curves"
  267. " (x25519:secp256r1:secp384r1:x448), or test with TLS 1.2 — see the Codex note in"
  268. " https://hermes-agent.nousresearch.com/docs/integrations/providers"
  269. )
  270. def _is_transient_transport_error(exc: BaseException) -> bool:
  271. """True when *exc* is transport-level (connection/TLS/socket) and safe to retry.
  272. httpx raises ``httpx.TransportError`` subclasses wrapping the original ``ssl``/``socket``
  273. error as ``__cause__``, so both spellings count. Anything else (decode errors, bugs) is
  274. not a network blip and must surface immediately.
  275. """
  276. err: Optional[BaseException] = exc
  277. seen: set[int] = set()
  278. while err is not None and id(err) not in seen:
  279. if isinstance(err, (httpx.TransportError, OSError)):
  280. return True
  281. seen.add(id(err))
  282. err = err.__cause__
  283. return False
  284. def _codex_login_post(url: str, *, failure: tuple[str, str], **kwargs: Any) -> httpx.Response:
  285. """One 15s POST for the device-login flow; transport errors become ``_codex_err(*failure)``.
  286. A transient transport blip (a dropped connection mid-flow) is retried twice with a small
  287. linear backoff before failing: losing the token exchange to a single SSL EOF wastes a
  288. device-code approval the user already completed in the browser (#114610).
  289. """
  290. attempt, attempts = 1, 3
  291. while True:
  292. try:
  293. with _codex_http_client(timeout=httpx.Timeout(15.0)) as client:
  294. return client.post(url, **kwargs)
  295. except Exception as exc:
  296. if attempt == attempts or not _is_transient_transport_error(exc):
  297. raise _codex_err(
  298. f"{failure[0]}: {exc}{_ssl_interop_hint(exc)}", failure[1]) from exc
  299. time.sleep(attempt)
  300. attempt += 1
  301. _CODEX_AUTH_BODY_MAX_BYTES = 1024 * 1024 # real OAuth/device-auth payloads are a few hundred bytes
  302. def _capped_byte_stream_class() -> type:
  303. """Build the capped stream subclass on first use.
  304. The base class is ``httpx.SyncByteStream``; naming it at module scope would resolve the
  305. lazy ``httpx`` proxy at import time and put httpx back on the interactive-CLI startup path
  306. (see ``auth_constants``).
  307. """
  308. cached = getattr(_capped_byte_stream_class, "cls", None)
  309. if cached is not None:
  310. return cached
  311. class _CappedByteStream(httpx.SyncByteStream):
  312. """Body stream that raises once more than ``_CODEX_AUTH_BODY_MAX_BYTES`` came off the wire.
  313. httpx type-checks ``response.stream`` against ``SyncByteStream``, so the cap has to be a
  314. stream subclass rather than a bare generator.
  315. """
  316. def __init__(self, response: httpx.Response) -> None:
  317. self._response, self._raw = response, response.stream
  318. def __iter__(self) -> Iterator[bytes]:
  319. total = 0
  320. for chunk in self._raw: # type: ignore[union-attr] # sync client only
  321. total += len(chunk)
  322. if total > _CODEX_AUTH_BODY_MAX_BYTES:
  323. self.close()
  324. raise _codex_err(
  325. f"Codex auth response from {self._response.url.host} exceeded "
  326. f"{_CODEX_AUTH_BODY_MAX_BYTES // 1024} KiB; refusing to parse it.",
  327. "codex_auth_response_too_large", relogin=False)
  328. yield chunk
  329. def close(self) -> None:
  330. self._raw.close() # type: ignore[union-attr]
  331. _capped_byte_stream_class.cls = _CappedByteStream # type: ignore[attr-defined]
  332. return _CappedByteStream
  333. def _cap_codex_response_body(response: httpx.Response) -> None:
  334. """httpx response hook: refuse to buffer an auth body above ``_CODEX_AUTH_BODY_MAX_BYTES``.
  335. Runs before ``client.post()`` reads the body, so a hostile or broken endpoint/proxy answering
  336. 200 with megabytes of "JSON" is cut off at the cap instead of being fully buffered and parsed
  337. (#55253). Same cap for every status: error bodies are small diagnostics too.
  338. """
  339. response.stream = _capped_byte_stream_class()(response)
  340. def _codex_http_client(**kwargs: Any) -> httpx.Client:
  341. """Build an ``httpx.Client`` for Codex OAuth/probe endpoints with Happy-Eyeballs racing and a
  342. 1 MiB response-body cap (``_cap_codex_response_body``).
  343. A host advertising AAAA records but blackholing IPv6 makes each serial connect eat the full
  344. timeout before IPv4 is tried (same failure mode as the chat transport). Best-effort: if the
  345. racing backend can't be installed (mocked client in tests), serial connect behavior remains.
  346. Same broken-IPv6 failure mode as the chat transport (#13834): a host that advertises AAAA records but
  347. blackholes IPv6 makes each serial connect attempt eat the full connect timeout before IPv4 is tried, so
  348. token refresh / device login / usage probes time out where the official Codex CLI (which races families
  349. per RFC 8305) works.
  350. """
  351. normalize_proxy_env_vars() # a bracketed-IPv6 NO_PROXY entry ([::1], Clash Verge/mihomo) makes the bare
  352. # trust_env client below raise InvalidURL at construction (#118159) — sanitize before building it.
  353. client = httpx.Client(event_hooks={"response": [_cap_codex_response_body]}, **kwargs)
  354. with suppress(Exception):
  355. from agent.process_bootstrap import enable_happy_eyeballs_on_client
  356. enable_happy_eyeballs_on_client(client)
  357. return client
  358. def _codex_quota_exhausted_error(retry_after: Optional[int]) -> AuthError:
  359. message = (
  360. f"Codex provider quota exhausted (429); retry after {retry_after}s. "
  361. "Credentials are still valid."
  362. if retry_after is not None else
  363. "Codex provider quota exhausted (429). Credentials are still valid; "
  364. "retry after the usage limit resets.")
  365. return _codex_err(message, CODEX_RATE_LIMITED_CODE, relogin=False)
  366. def _codex_refresh_failure_error(response: httpx.Response) -> AuthError:
  367. """Decode a non-200 Codex token-refresh response into a shaped AuthError."""
  368. from hermes_cli.auth import _nonempty_str
  369. code = "codex_refresh_failed"
  370. message = f"Codex token refresh failed with status {response.status_code}."
  371. try:
  372. err = response.json()
  373. if isinstance(err, dict):
  374. err_obj = err.get("error")
  375. # OpenAI shape: {"error": {"code": "...", "message": "...", "type": "..."}}
  376. if isinstance(err_obj, dict):
  377. nested_code = err_obj.get("code") or err_obj.get("type")
  378. if _nonempty_str(nested_code):
  379. code = nested_code.strip()
  380. nested_msg = err_obj.get("message")
  381. if _nonempty_str(nested_msg):
  382. message = f"Codex token refresh failed: {nested_msg.strip()}"
  383. # OAuth spec shape: {"error": "code_str", "error_description": "..."}
  384. elif _nonempty_str(err_obj):
  385. code = err_obj.strip()
  386. err_desc = err.get("error_description") or err.get("message")
  387. if _nonempty_str(err_desc):
  388. message = f"Codex token refresh failed: {err_desc.strip()}"
  389. except Exception:
  390. pass
  391. if code == "refresh_token_reused":
  392. message = (
  393. "Codex refresh token was already consumed by another client "
  394. "(e.g. Codex CLI or VS Code extension). "
  395. "Run `codex` in your terminal to generate fresh tokens, "
  396. f"then run `{_codex_relogin_command()}` to re-authenticate.")
  397. # A 401/403 from the token endpoint always means the refresh token is invalid/expired —
  398. # force relogin even if the body error code wasn't one of the known strings.
  399. relogin_required = (
  400. code in {"invalid_grant", "invalid_token", "invalid_request", "refresh_token_reused"}
  401. or response.status_code in {401, 403})
  402. return _codex_err(message, code, relogin=relogin_required)
  403. def refresh_codex_oauth_pure(
  404. access_token: str, refresh_token: str, *, timeout_seconds: float = 20.0) -> dict[str, Any]:
  405. """Refresh Codex OAuth tokens without mutating Hermes auth state."""
  406. from hermes_cli.auth import _nonempty_str, _utc_now_z
  407. del access_token # Access token is only used by callers to decide whether to refresh.
  408. if not _nonempty_str(refresh_token):
  409. raise _codex_err(_MISSING_REFRESH_TOKEN_MSG.format(relogin=_codex_relogin_command()),
  410. "codex_auth_missing_refresh_token", relogin=True)
  411. with _codex_http_client(
  412. timeout=httpx.Timeout(max(5.0, float(timeout_seconds))),
  413. headers={"Accept": "application/json", "User-Agent": hermes_user_agent()}) as client:
  414. response = client.post(
  415. CODEX_OAUTH_TOKEN_URL, headers={"Content-Type": "application/x-www-form-urlencoded"},
  416. data={
  417. "grant_type": "refresh_token", "refresh_token": refresh_token,
  418. "client_id": CODEX_OAUTH_CLIENT_ID})
  419. if response.status_code == 429:
  420. # Quota exhaustion on the token endpoint: the refresh token is still valid and re-auth
  421. # cannot lift a quota cap, so classify distinctly from auth failures ("retry later").
  422. raise _codex_quota_exhausted_error(
  423. _parse_retry_after_seconds(getattr(response, "headers", None)))
  424. if response.status_code != 200:
  425. raise _codex_refresh_failure_error(response)
  426. refresh_payload, refreshed_access = _refresh_payload_access_token(
  427. response, provider="openai-codex", invalid_response=None,
  428. invalid_json=("Codex token refresh returned invalid JSON.", "codex_refresh_invalid_json"),
  429. # A 200 with a non-JSON body is an edge/proxy misfire, not a revoked grant: never relogin.
  430. invalid_json_relogin=False,
  431. missing_access=(
  432. "Codex token refresh response was missing access_token.",
  433. "codex_refresh_missing_access_token"))
  434. updated = {
  435. "access_token": refreshed_access, "refresh_token": refresh_token.strip(),
  436. "last_refresh": _utc_now_z()}
  437. next_refresh = refresh_payload.get("refresh_token")
  438. if _nonempty_str(next_refresh):
  439. updated["refresh_token"] = next_refresh.strip()
  440. return updated
  441. def _refresh_codex_auth_tokens(tokens: dict[str, str], timeout_seconds: float) -> dict[str, str]:
  442. """Refresh Codex access token using the refresh token.
  443. The whole re-read -> endpoint POST -> write-back runs inside the SOURCE store's transaction:
  444. two profiles borrowing the same root grant otherwise both submit the same single-use refresh
  445. token (each holds only its own profile lock) and OpenAI revokes the family. A waiter that
  446. finds root already rotated by its peer adopts the stored pair instead of replaying the
  447. consumed token. Both locks wait out a full endpoint timeout so the waiter adopts, not times out.
  448. """
  449. from hermes_cli.auth import _provider_state_transaction, _save_codex_tokens, refresh_codex_oauth_pure
  450. lock_timeout = max(float(AUTH_LOCK_TIMEOUT_SECONDS), float(timeout_seconds) + 5.0)
  451. with _provider_state_transaction("openai-codex", lock_timeout) as (_store, state, _source):
  452. stored = (state or {}).get("tokens")
  453. stored = stored if isinstance(stored, dict) else {}
  454. stored_at, stored_rt = _stripped(stored.get("access_token")), _stripped(stored.get("refresh_token"))
  455. if stored_at and stored_rt and stored_rt != _stripped(tokens.get("refresh_token")):
  456. logger.info("Codex refresh token already rotated by a peer — adopting the stored pair.")
  457. return {**tokens, "access_token": stored_at, "refresh_token": stored_rt}
  458. try:
  459. refreshed = refresh_codex_oauth_pure(
  460. str(tokens.get("access_token", "") or ""), str(tokens.get("refresh_token", "") or ""),
  461. timeout_seconds=timeout_seconds)
  462. except AuthError as exc:
  463. # Self-heal cross-store rotation: refresh_tokens are single-use, so when the Codex CLI
  464. # (or another Hermes process) rotates the shared token this frozen copy fails with a
  465. # relogin-required error (invalid_grant / refresh_token_reused / 401). Adopt the
  466. # canonical fresh token from ~/.codex/auth.json before surfacing a hard 401. Transient
  467. # failures (429 quota) keep relogin_required=False — the stored token is still valid —
  468. # re-raise.
  469. if not getattr(exc, "relogin_required", False):
  470. raise
  471. imported = _recover_codex_tokens_from_cli(
  472. f"refresh_token rejected: {getattr(exc, 'code', None) or 'auth_error'}",
  473. observed_access_token=stored_at or None)
  474. if not imported:
  475. raise
  476. return imported
  477. updated_tokens = {
  478. **tokens, "access_token": refreshed["access_token"],
  479. "refresh_token": refreshed["refresh_token"]}
  480. # Nested transaction: the per-path lock is reentrant, and it re-reads under the held locks.
  481. _save_codex_tokens(updated_tokens, write_through=True)
  482. return updated_tokens
  483. def _import_codex_cli_tokens() -> Optional[dict[str, str]]:
  484. """Read ~/.codex/auth.json (Codex CLI file) tokens if valid and not expired; never writes."""
  485. from hermes_cli.auth import _codex_access_token_is_expiring
  486. codex_home = os.getenv("CODEX_HOME", "").strip() or str(Path.home() / ".codex")
  487. auth_path = Path(codex_home).expanduser() / "auth.json"
  488. if not auth_path.is_file():
  489. return None
  490. try:
  491. tokens = json.loads(auth_path.read_text(encoding="utf-8-sig")).get("tokens")
  492. if not (isinstance(tokens, dict) and tokens.get("access_token")
  493. and tokens.get("refresh_token")):
  494. return None
  495. # Importing stale tokens that can't be refreshed would leave the user with
  496. # "Login successful!" but no working credentials.
  497. if _codex_access_token_is_expiring(tokens["access_token"], 0):
  498. logger.debug("Codex CLI tokens at %s are expired — skipping import.", auth_path)
  499. return None
  500. return dict(tokens)
  501. except Exception:
  502. return None
  503. def resolve_codex_runtime_credentials(
  504. *, force_refresh: bool = False, refresh_if_expiring: bool = True,
  505. refresh_skew_seconds: int = CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
  506. read_only: bool = False) -> dict[str, Any]:
  507. """Resolve runtime credentials from Hermes's own Codex token store.
  508. ``read_only=True`` (status / doctor / pickers) reports the stored state as-is: no Codex CLI
  509. adoption, no token refresh, no auth-store write — and it wins over ``force_refresh``. A
  510. diagnostic that silently imports another program's rotating refresh token or spends one is a
  511. mutation the user never asked for (#68004).
  512. Falls back to the credential pool when the singleton (``providers.openai-codex.tokens``) has no
  513. usable access_token but the pool (``credential_pool.openai-codex``) does.
  514. This closes the divergence between the chat path (singleton-only via this function) and the auxiliary
  515. path (pool-first via ``auxiliary_client._resolve_codex_credential_and_base``). Without this fallback, a user whose tokens live only
  516. in the pool — for example after a manual pool seed, a partial re-auth, or pool-only restoration from a
  517. backup — gets a bare HTTP 401 ``Missing Authentication header`` from the wire instead of a usable
  518. credential. See issue #32992.
  519. """
  520. from hermes_cli.auth import (
  521. _auth_store_lock, _codex_access_token_is_expiring, _probe_codex_quota_restored,
  522. _read_codex_tokens)
  523. read_error: Optional[AuthError] = None
  524. data = None
  525. observed: Optional[str] = None
  526. try:
  527. if read_only:
  528. # A read-only report takes no store lock: ``_save_auth_store`` replaces auth.json
  529. # atomically, and materialising ``auth.lock`` is itself a write a diagnostic must not
  530. # make. No recovery follows a read-only read, so no observed token is needed.
  531. data = _read_codex_tokens(_lock=False)
  532. else:
  533. with _auth_store_lock():
  534. # Observe the singleton in the same locked snapshot the read validates, so recovery
  535. # can compare-and-swap against exactly the credential it is repairing (#73667).
  536. from hermes_cli.auth import _load_auth_store, _load_provider_state
  537. raw = (_load_provider_state(_load_auth_store(), "openai-codex") or {}).get("tokens")
  538. observed = raw.get("access_token") if isinstance(raw, dict) else None
  539. data = _read_codex_tokens(_lock=False)
  540. except AuthError as exc:
  541. read_error = exc
  542. if not read_only and exc.relogin_required and exc.code in {
  543. "codex_auth_missing_access_token", "codex_auth_missing_refresh_token",
  544. "codex_auth_invalid_shape"}:
  545. imported = _recover_codex_tokens_from_cli(
  546. str(exc.code or "auth_error"), observed_access_token=observed)
  547. if imported:
  548. data = {"tokens": imported, "last_refresh": imported.get("last_refresh")}
  549. if data is None:
  550. pool_token, pool_base = _pool_codex_credential()
  551. if pool_token and force_refresh and not read_only:
  552. # Pool-only setup: a forced refresh must rotate the pool entry, not resend its token.
  553. from agent.credential_pool import load_pool
  554. refreshed = load_pool("openai-codex").try_refresh_matching(api_key_hint=pool_token)
  555. pool_token = refreshed.runtime_api_key if refreshed is not None else ""
  556. if pool_token:
  557. # Report the host this row routes to, not the ambient default: a pooled gateway key
  558. # paired with chatgpt.com leaks to every consumer of this result (#121486).
  559. return _codex_runtime_result(pool_token, source="credential_pool", last_refresh=None,
  560. base_url=_codex_pool_route_base_url(pool_base))
  561. pool_rate_limit = _codex_pool_rate_limit_status()
  562. if pool_rate_limit:
  563. # Before surfacing the persisted cooldown, ask the usage endpoint whether the quota
  564. # reset early (banked reset redeemed, plan upgraded): ``last_error_reset_at`` can be
  565. # days in the future while the account is already usable again. Never from a
  566. # read-only caller: the picker fingerprint resolves on every cache-only read.
  567. if not read_only and _probe_codex_pool_entry_quota_restored(pool_rate_limit):
  568. logger.info("Codex quota restored upstream — clearing stale pool cooldown(s).")
  569. clear_codex_pool_quota_cooldowns()
  570. pool_token, pool_base = _pool_codex_credential()
  571. if pool_token:
  572. return _codex_runtime_result(
  573. pool_token, source="credential_pool", last_refresh=None,
  574. base_url=_codex_pool_route_base_url(pool_base))
  575. reset_at = pool_rate_limit.get("reset_at")
  576. in_future = isinstance(reset_at, (int, float)) and reset_at > time.time()
  577. raise _codex_quota_exhausted_error(int(reset_at - time.time()) if in_future else None)
  578. if read_error is not None:
  579. raise read_error
  580. raise _codex_err(_NO_CREDENTIALS_MSG.format(relogin=_codex_relogin_command()),
  581. "codex_auth_missing", relogin=True)
  582. tokens = dict(data["tokens"])
  583. access_token = _stripped(tokens.get("access_token"))
  584. refresh_timeout_seconds = env_float("HERMES_CODEX_REFRESH_TIMEOUT_SECONDS", 20)
  585. def _should_refresh(token: str) -> bool:
  586. if read_only:
  587. return False
  588. return bool(force_refresh) or (
  589. refresh_if_expiring and _codex_access_token_is_expiring(token, refresh_skew_seconds))
  590. if _should_refresh(access_token):
  591. # Re-read under lock to avoid racing with other Hermes processes
  592. lock_timeout = max(float(AUTH_LOCK_TIMEOUT_SECONDS), refresh_timeout_seconds + 5.0)
  593. with _auth_store_lock(timeout_seconds=lock_timeout):
  594. data = _read_codex_tokens(_lock=False)
  595. tokens = dict(data["tokens"])
  596. if _should_refresh(_stripped(tokens.get("access_token"))):
  597. tokens = _refresh_codex_auth_tokens(tokens, refresh_timeout_seconds)
  598. access_token = _stripped(tokens.get("access_token"))
  599. return _codex_runtime_result(
  600. access_token, source="hermes-auth-store", last_refresh=data.get("last_refresh"))
  601. def _is_codex_rate_limit_shaped(code: Any, reason: Any, message: Any) -> bool:
  602. """True when persisted pool-entry error metadata describes a 429/quota stop."""
  603. reason_l, message_l = str(reason or "").lower(), str(message or "").lower()
  604. return (
  605. code == 429
  606. or any(k in reason_l for k in ("rate_limit", "usage_limit", "quota"))
  607. or any(k in message_l for k in ("rate limit", "usage limit", "quota")))
  608. def _entry_is_rate_limit_exhausted(entry: dict[str, Any]) -> bool:
  609. """Pool entry frozen by a 429/quota stop (as opposed to an auth failure)."""
  610. return entry.get("last_status") == "exhausted" and _is_codex_rate_limit_shaped(
  611. entry.get("last_error_code"), entry.get("last_error_reason"),
  612. entry.get("last_error_message"))
  613. # Throttle for the live Codex quota probe. It runs on the hot credential-selection path while the
  614. # pool is exhausted, so without a floor a busy gateway would hammer the usage endpoint per call.
  615. CODEX_QUOTA_PROBE_MIN_INTERVAL_SECONDS = 300 # 5 minutes
  616. _codex_quota_probe_cache: dict[str, tuple[float, Optional[bool]]] = {}
  617. _codex_quota_probe_lock = threading.Lock()
  618. def _codex_quota_probe_cache_key(token: str) -> str:
  619. return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
  620. def _codex_usage_probe_url(base_url: Optional[str]) -> str:
  621. """Resolve the Codex usage endpoint for a probe.
  622. Mirrors the Codex CLI's PathStyle split: base URLs containing ``/backend-api`` use the ChatGPT
  623. ``/wham/usage`` path, everything else ``/api/codex/usage``. Kept local so this low-level auth
  624. module does not import the auxiliary account-usage module.
  625. """
  626. normalized = _stripped(base_url).rstrip("/") or _codex_base_url()
  627. normalized = normalized.removesuffix("/codex")
  628. prefix = normalized + ("/wham" if "/backend-api" in normalized else "/api/codex")
  629. return prefix + "/usage"
  630. def _probe_codex_quota_restored(
  631. access_token: Any, *, base_url: Optional[str] = None,
  632. min_interval_seconds: float = CODEX_QUOTA_PROBE_MIN_INTERVAL_SECONDS) -> Optional[bool]:
  633. """Ask the Codex usage endpoint whether this account's quota is usable again.
  634. Probes are throttled per access token (module-local cache) so the hot selection path can fire
  635. this freely.
  636. """
  637. from hermes_cli.auth import _codex_quota_probe_cache, _nonempty_str
  638. token = _stripped(access_token)
  639. # Real Codex access tokens are JWTs. Refusing to probe non-JWT tokens avoids pointless
  640. # network calls for corrupt/placeholder entries (and keeps hermetic test fixtures offline).
  641. if not token or not _decode_jwt_claims(token):
  642. return None
  643. cache_key = _codex_quota_probe_cache_key(token)
  644. now = time.monotonic()
  645. with _codex_quota_probe_lock:
  646. cached = _codex_quota_probe_cache.get(cache_key)
  647. if cached is not None and (now - cached[0]) < min_interval_seconds:
  648. return cached[1]
  649. # Reserve the slot immediately so concurrent selectors don't stampede the endpoint.
  650. _codex_quota_probe_cache[cache_key] = (now, None)
  651. result: Optional[bool] = None
  652. try:
  653. # Account/residency headers from the JWT (required for some account shapes).
  654. from agent.codex_headers import codex_account_headers
  655. headers = {
  656. "Authorization": f"Bearer {token}", "Accept": "application/json",
  657. "User-Agent": "codex-cli", **codex_account_headers(token)}
  658. with _codex_http_client(timeout=10.0) as client:
  659. response = client.get(_codex_usage_probe_url(base_url), headers=headers)
  660. if response.status_code == 200:
  661. payload = response.json() or {}
  662. # A model-scoped allowance (``additional_rate_limits``) at 100% still 429s that
  663. # model, so it counts against "restored" like the account-wide windows (#97315).
  664. windows = [payload.get("rate_limit") or {}] + [
  665. extra.get("rate_limit") or {}
  666. for extra in (payload.get("additional_rate_limits") or [])
  667. if isinstance(extra, dict)]
  668. worst_used: Optional[float] = None
  669. for rate_limit in windows:
  670. for key in ("primary_window", "secondary_window"):
  671. used = (rate_limit.get(key) or {}).get("used_percent")
  672. if isinstance(used, (int, float)):
  673. worst_used = max(worst_used or 0.0, float(used))
  674. if worst_used is not None:
  675. result = worst_used < 100.0
  676. elif response.status_code == 429:
  677. result = False
  678. except Exception:
  679. logger.debug("Codex quota probe failed", exc_info=True)
  680. result = None
  681. with _codex_quota_probe_lock:
  682. _codex_quota_probe_cache[cache_key] = (now, result)
  683. return result
  684. def _refresh_expired_codex_probe_token(
  685. access_token: Any, refresh_token: Any, *,
  686. min_interval_seconds: float = CODEX_QUOTA_PROBE_MIN_INTERVAL_SECONDS) -> Optional[dict[str, Any]]:
  687. """Refresh an EXPIRED stored access token so the quota probe can get a real answer.
  688. Exhausted pool entries are skipped by the proactive refresh chain (#44799), so by the time
  689. anything probes with the stored token it has expired; the usage endpoint answers
  690. ``401 token_expired``, the probe returns None, and the cooldown is kept until
  691. ``last_error_reset_at`` no matter what happened upstream (top-up, plan upgrade) — #89415.
  692. Returns the rotated token pair (callers MUST persist it: refresh tokens are single-use) or
  693. None when no refresh was needed/possible. The cooldown itself is left untouched.
  694. Shares the probe's per-token throttle: a refresh that keeps failing (revoked grant, network
  695. down) would otherwise POST to the token endpoint on every credential selection, while the
  696. probe itself is capped at one call per ``min_interval_seconds``. A failed attempt reserves
  697. the stale token's probe slot, so neither the refresh nor the doomed 401 probe fire again
  698. until the interval has elapsed.
  699. """
  700. from hermes_cli.auth import _codex_quota_probe_cache
  701. token, refresh = _stripped(access_token), _stripped(refresh_token)
  702. if not token or not refresh or not _codex_access_token_is_expiring(token, 0):
  703. return None
  704. cache_key = _codex_quota_probe_cache_key(token)
  705. now = time.monotonic()
  706. with _codex_quota_probe_lock:
  707. cached = _codex_quota_probe_cache.get(cache_key)
  708. if cached is not None and (now - cached[0]) < min_interval_seconds:
  709. return None
  710. try:
  711. return refresh_codex_oauth_pure(token, refresh)
  712. except Exception:
  713. logger.debug("Codex pre-probe token refresh failed", exc_info=True)
  714. with _codex_quota_probe_lock:
  715. _codex_quota_probe_cache[cache_key] = (now, None)
  716. return None
  717. def _probe_codex_pool_entry_quota_restored(entry: dict[str, Any]) -> Optional[bool]:
  718. """``_probe_codex_quota_restored`` for a persisted pool entry, refreshing an expired token first."""
  719. from hermes_cli.auth import _auth_store_lock, _load_auth_store, _save_auth_store
  720. token = _stripped(entry.get("access_token"))
  721. fresh = _refresh_expired_codex_probe_token(token, entry.get("refresh_token"))
  722. if fresh:
  723. token = fresh["access_token"]
  724. try:
  725. with _auth_store_lock():
  726. auth_store = _load_auth_store()
  727. for disk_entry in _codex_pool_dicts(_pool_entries(auth_store, "openai-codex")):
  728. if disk_entry.get("id") == entry.get("id"):
  729. disk_entry.update(fresh)
  730. _save_auth_store(auth_store)
  731. break
  732. except Exception:
  733. logger.debug("Failed to persist refreshed Codex pool tokens", exc_info=True)
  734. if not token:
  735. return None
  736. # The row keeps the canonical URL; a gateway key belongs to its route host (#121486).
  737. return _probe_codex_quota_restored(token, base_url=_codex_pool_route_base_url(entry.get("base_url")))
  738. def clear_codex_pool_quota_cooldowns(access_token: Optional[str] = None) -> int:
  739. """Clear rate-limit cooldowns on persisted openai-codex pool entries.
  740. Called after the upstream quota is KNOWN to be restored (a ``/usage reset`` redemption or a
  741. positive live probe) so auth.json stops freezing credentials behind a stale
  742. ``last_error_reset_at``. With *access_token* only the matching entry clears; otherwise every
  743. rate-limited entry does (a redeemed banked reset restores the whole account; a still-exhausted
  744. entry just re-freezes with fresh metadata on its next 429).
  745. """
  746. from agent.credential_pool import _borrowed_single_use_pool_root, _profile_owns_pool_provider
  747. from hermes_cli.auth import _auth_store_lock, _load_auth_store, _save_auth_store
  748. cleared = 0
  749. try:
  750. # Same owner rule as ``persist_pool_entries``: a profile with no Codex rows of its own
  751. # borrows the global-root pool, so the cooldown must clear where the rows actually live.
  752. target = None if _profile_owns_pool_provider("openai-codex") else _borrowed_single_use_pool_root()
  753. with _auth_store_lock(target_path=target):
  754. auth_store = _load_auth_store(target)
  755. for entry in _codex_pool_dicts(_pool_entries(auth_store, "openai-codex")):
  756. if access_token and str(entry.get("access_token") or "") != access_token:
  757. continue
  758. if _entry_is_rate_limit_exhausted(entry):
  759. _clear_pool_entry_status(entry)
  760. cleared += 1
  761. if cleared:
  762. _save_auth_store(auth_store, target_path=target)
  763. except Exception:
  764. logger.debug("Failed to clear Codex pool quota cooldowns", exc_info=True)
  765. return cleared
  766. def _codex_pool_dicts(entries: Optional[list[Any]]) -> Iterator[dict[str, Any]]:
  767. for entry in entries or ():
  768. if isinstance(entry, dict):
  769. yield entry
  770. def _codex_pool_rate_limit_status() -> Optional[dict[str, Any]]:
  771. """Return metadata for a pool-only Codex credential in quota cooldown.
  772. Reads through ``read_credential_pool`` so a named profile with no Codex rows of its own sees
  773. the global-root pool (the per-provider fallback every other pool read uses)."""
  774. from hermes_cli.auth import _nonempty_str, read_credential_pool
  775. from agent.credential_pool import _parse_absolute_timestamp
  776. try:
  777. now = time.time()
  778. for entry in _codex_pool_dicts(read_credential_pool("openai-codex")):
  779. token = entry.get("access_token")
  780. if not _nonempty_str(token) or not _entry_is_rate_limit_exhausted(entry):
  781. continue
  782. reset_at = _parse_absolute_timestamp(entry.get("last_error_reset_at"))
  783. if reset_at is None or reset_at > now:
  784. return {
  785. "label": entry.get("label"), "last_refresh": entry.get("last_refresh"),
  786. "reset_at": reset_at, "reason": entry.get("last_error_reason"),
  787. "message": entry.get("last_error_message"), "access_token": token.strip(),
  788. "refresh_token": entry.get("refresh_token"), "id": entry.get("id"),
  789. "base_url": entry.get("base_url")}
  790. except Exception:
  791. logger.debug("Codex pool rate-limit lookup failed", exc_info=True)
  792. return None
  793. def _pool_entries(auth_store: dict[str, Any], provider_id: str) -> Optional[list[Any]]:
  794. """``auth_store["credential_pool"][provider_id]`` when it is a list, else None."""
  795. pool = auth_store.get("credential_pool")
  796. entries = pool.get(provider_id) if isinstance(pool, dict) else None
  797. return entries if isinstance(entries, list) else None
  798. def _pool_codex_credential() -> tuple[str, str]:
  799. """``(access_token, row base_url)`` of the first pool entry with a non-empty access_token that is
  800. not in an exhaustion cooldown window, so the caller routes the token to the host that row belongs
  801. to; ``("", "")`` when none is usable.
  802. Fallback for ``resolve_codex_runtime_credentials`` when the singleton has no creds; reads
  803. through ``read_credential_pool`` so a profile inherits the global-root pool (#34143)."""
  804. from agent.credential_pool import _parse_absolute_timestamp
  805. from hermes_cli.auth import _nonempty_str, read_credential_pool
  806. try:
  807. for entry in _codex_pool_dicts(read_credential_pool("openai-codex")):
  808. token = entry.get("access_token")
  809. # Same normaliser as ``_codex_pool_rate_limit_status``: a millisecond epoch compared
  810. # raw reads as far-future here and as elapsed there, hiding a usable entry (#103349).
  811. reset_at = _parse_absolute_timestamp(entry.get("last_error_reset_at"))
  812. in_cooldown = reset_at is not None and reset_at > time.time()
  813. if _nonempty_str(token) and not in_cooldown:
  814. return token.strip(), _stripped(entry.get("base_url"))
  815. except Exception:
  816. logger.debug("Codex pool fallback lookup failed", exc_info=True)
  817. return "", ""
  818. def _login_openai_codex(args, pconfig: ProviderConfig, *, force_new_login: bool = False) -> None:
  819. """OpenAI Codex login: device code by default, browser PKCE when opted in (``--browser`` /
  820. ``auth.codex_login_flow``). Tokens stored in ~/.hermes/auth.json."""
  821. from hermes_cli.auth import (
  822. _codex_access_token_is_expiring, _import_codex_cli_tokens,
  823. _offer_existing_oauth_credentials, _print_login_success, _prompt_yes_no, _save_codex_tokens,
  824. _update_config_for_provider, resolve_codex_runtime_credentials)
  825. from hermes_cli.auth_codex_browser import codex_oauth_login
  826. del pconfig # kept for parity with other provider login helpers
  827. if not force_new_login:
  828. if _offer_existing_oauth_credentials(
  829. "openai-codex", resolve=resolve_codex_runtime_credentials,
  830. is_expiring=_codex_access_token_is_expiring, display_name="Codex",
  831. default_base_url=DEFAULT_CODEX_BASE_URL,
  832. expired_notice="Existing Codex credentials are expired. Starting fresh login..."):
  833. return
  834. cli_tokens = _import_codex_cli_tokens()
  835. if cli_tokens:
  836. print("Found existing Codex CLI credentials at ~/.codex/auth.json")
  837. print("Hermes will create its own session to avoid conflicts with Codex CLI / VS Code.")
  838. if _prompt_yes_no(
  839. "Import these credentials? (a separate login is recommended) [y/N]: ", default="n"):
  840. _save_codex_tokens(cli_tokens)
  841. config_path = _update_config_for_provider("openai-codex", _codex_base_url())
  842. print()
  843. print("Credentials imported. Note: if Codex CLI refreshes its token,")
  844. print("Hermes will keep working independently with its own session.")
  845. print(f" Config updated: {config_path} (model.provider=openai-codex)")
  846. return
  847. # Run a fresh OAuth flow — Hermes gets its own session (device code unless the user opted in
  848. # to the browser flow).
  849. print()
  850. creds = codex_oauth_login(args)
  851. _save_codex_tokens(creds["tokens"], creds.get("last_refresh"))
  852. config_path = _update_config_for_provider(
  853. "openai-codex", creds.get("base_url", DEFAULT_CODEX_BASE_URL))
  854. _print_login_success("openai-codex", config_path, show_auth_state=True)
  855. def _codex_login_rate_limited_error(response: httpx.Response, *, during: str = "") -> AuthError:
  856. """AuthError for a 429 from OpenAI's device-auth endpoints (throttle, not credential fault)."""
  857. # Upstream rate-limit / usage-quota exhaustion on the token endpoint. The stored refresh token is still
  858. # valid here — re-authenticating cannot lift a quota cap. Classify distinctly from auth failures so
  859. # callers surface a "retry later" notice instead of a misleading "run hermes auth" prompt (see issue
  860. # #32790).
  861. retry_after = _parse_retry_after_seconds(getattr(response, "headers", None))
  862. wait_hint = (
  863. f" Try again in about {retry_after}s." if retry_after is not None
  864. else " Wait a minute and run the login again.")
  865. return _codex_err(
  866. f"OpenAI is rate-limiting Codex login requests (HTTP 429){during}. "
  867. f"This is a temporary throttle on OpenAI's side, not a credential problem.{wait_hint}",
  868. CODEX_RATE_LIMITED_CODE)
  869. def _codex_request_device_code(issuer: str, client_id: str) -> dict[str, Any]:
  870. """Step 1 of the Codex device flow: request a user code, retrying capped on HTTP 429.
  871. OpenAI rate-limits this request when login is attempted too often from one IP/account — retry
  872. with capped backoff (honoring ``Retry-After``) before surfacing an actionable message.
  873. """
  874. max_attempts = 4
  875. for attempt in range(1, max_attempts + 1):
  876. resp = _codex_login_post(
  877. f"{issuer}/api/accounts/deviceauth/usercode", json={"client_id": client_id},
  878. headers={"Content-Type": "application/json"},
  879. failure=("Failed to request device code", "device_code_request_failed"))
  880. if resp.status_code != 429:
  881. break
  882. if attempt < max_attempts:
  883. # Exponential backoff (2s, 4s, 8s) capped, preferring the server's Retry-After.
  884. retry_after = _parse_retry_after_seconds(getattr(resp, "headers", None))
  885. delay = max(1, min(int(retry_after if retry_after is not None else 2 ** attempt), 60))
  886. print(f"OpenAI is rate-limiting login requests (429); retrying in {delay}s...")
  887. time.sleep(delay)
  888. if resp.status_code == 429:
  889. raise _codex_login_rate_limited_error(resp)
  890. if resp.status_code != 200:
  891. raise _codex_err(
  892. f"Device code request returned status {resp.status_code}.", "device_code_request_error")
  893. device_data = resp.json()
  894. device_data["interval"] = max(3, int(device_data.get("interval", "5")))
  895. if not device_data.get("user_code", "") or not device_data.get("device_auth_id", ""):
  896. raise _codex_err("Device code response missing required fields.", "device_code_incomplete")
  897. return device_data
  898. def _codex_poll_authorization_code(
  899. issuer: str, *, device_auth_id: str, user_code: str, poll_interval: int) -> dict[str, Any]:
  900. """Step 3 of the Codex device flow: poll until sign-in completes (403/404 = still pending)."""
  901. max_wait = 15 * 60 # 15 minutes
  902. max_consecutive_blips = 6 # survives transient drops, still fails fast on a dead network
  903. start = time.monotonic()
  904. code_resp = None
  905. try:
  906. with _codex_http_client(timeout=httpx.Timeout(15.0)) as client:
  907. consecutive_blips = 0
  908. while time.monotonic() - start < max_wait:
  909. time.sleep(poll_interval)
  910. try:
  911. poll_resp = client.post(
  912. f"{issuer}/api/accounts/deviceauth/token",
  913. json={"device_auth_id": device_auth_id, "user_code": user_code},
  914. headers={"Content-Type": "application/json"})
  915. except Exception as exc:
  916. if not _is_transient_transport_error(exc):
  917. raise _codex_err(
  918. f"Device auth polling request failed: {exc}{_ssl_interop_hint(exc)}",
  919. "device_code_poll_error") from exc
  920. consecutive_blips += 1
  921. if consecutive_blips >= max_consecutive_blips:
  922. raise _codex_err(
  923. f"Device auth polling request failed after {consecutive_blips} consecutive"
  924. f" transport errors: {exc}{_ssl_interop_hint(exc)}",
  925. "device_code_poll_error") from exc
  926. print("Transient network error while waiting for sign-in; retrying...")
  927. continue
  928. consecutive_blips = 0
  929. if poll_resp.status_code == 200:
  930. code_resp = poll_resp.json()
  931. break
  932. if poll_resp.status_code not in {403, 404}: # 403/404 = user hasn't finished yet
  933. raise _codex_err(
  934. f"Device auth polling returned status {poll_resp.status_code}.",
  935. "device_code_poll_error")
  936. except KeyboardInterrupt:
  937. print("\nLogin cancelled.")
  938. raise SystemExit(130)
  939. if code_resp is None:
  940. raise _codex_err("Login timed out after 15 minutes.", "device_code_timeout")
  941. return code_resp
  942. def _codex_exchange_authorization_code(
  943. issuer: str, client_id: str, code_resp: dict[str, Any]) -> dict[str, Any]:
  944. """Step 4 of the Codex device flow: swap the authorization code for tokens."""
  945. authorization_code = code_resp.get("authorization_code", "")
  946. code_verifier = code_resp.get("code_verifier", "")
  947. if not authorization_code or not code_verifier:
  948. raise _codex_err(
  949. "Device auth response missing authorization_code or code_verifier.",
  950. "device_code_incomplete_exchange")
  951. token_resp = _codex_login_post(
  952. CODEX_OAUTH_TOKEN_URL,
  953. data={
  954. "grant_type": "authorization_code", "code": authorization_code,
  955. "redirect_uri": f"{issuer}/deviceauth/callback", "client_id": client_id,
  956. "code_verifier": code_verifier},
  957. headers={"Content-Type": "application/x-www-form-urlencoded"},
  958. failure=("Token exchange failed", "token_exchange_failed"))
  959. if token_resp.status_code == 429:
  960. raise _codex_login_rate_limited_error(token_resp, during=" during token exchange")
  961. if token_resp.status_code != 200:
  962. raise _codex_err(
  963. f"Token exchange returned status {token_resp.status_code}.", "token_exchange_error")
  964. tokens = token_resp.json()
  965. if not tokens.get("access_token", ""):
  966. raise _codex_err(
  967. "Token exchange did not return an access_token.", "token_exchange_no_access_token")
  968. return tokens
  969. def _codex_device_code_login() -> dict[str, Any]:
  970. """Run the OpenAI device code login flow and return credentials dict."""
  971. from hermes_cli.auth import _utc_now_z
  972. issuer, client_id = "https://auth.openai.com", CODEX_OAUTH_CLIENT_ID
  973. device_data = _codex_request_device_code(issuer, client_id)
  974. user_code = device_data["user_code"]
  975. # Step 2: Show user the code
  976. print("To continue, follow these steps:\n")
  977. print(" 1. Open this URL in your browser:")
  978. print(f" \033[94m{issuer}/codex/device\033[0m\n")
  979. print(" 2. Enter this code:")
  980. print(f" \033[94m{user_code}\033[0m\n")
  981. print("Waiting for sign-in... (press Ctrl+C to cancel)")
  982. code_resp = _codex_poll_authorization_code(
  983. issuer, device_auth_id=device_data["device_auth_id"], user_code=user_code,
  984. poll_interval=device_data["interval"])
  985. tokens = _codex_exchange_authorization_code(issuer, client_id, code_resp)
  986. # Return tokens for the caller to persist (never writes to ~/.codex/)
  987. return {
  988. "tokens": {
  989. "access_token": tokens.get("access_token", ""),
  990. "refresh_token": tokens.get("refresh_token", "")},
  991. "base_url": _codex_base_url(), "last_refresh": _utc_now_z(), "auth_mode": "chatgpt",
  992. "source": "device-code"}