client_lifecycle.py 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002
  1. """Tool-resource teardown, wire-client lifecycle and credential refresh for ``AIAgent``.
  2. ``ClientLifecycleMixin`` owns task cleanup, the shared primary client, per-request client caches
  3. (owner-thread close vs stranger-thread abort), credential rotation and route-derived headers.
  4. """
  5. import logging
  6. import threading
  7. import time
  8. from contextlib import suppress
  9. from typing import Any, Optional
  10. from agent.lazy_forward import forward as _forward, forward_static as _forward_static, lazy_attr as _lazy_attr
  11. from hermes_cli.timeouts import get_provider_request_timeout
  12. from utils import base_url_host_matches, env_float
  13. logger = logging.getLogger("run_agent") # origin module's logger name: log records / caplog filters unchanged
  14. _QWEN_CODE_VERSION = "0.14.1" # Qwen Portal mimics the QwenCode CLI
  15. # Per-request cache slot attribute names (OpenAI-style and Anthropic clients).
  16. _OPENAI_SLOT = "_request_client_cache"
  17. _ANTHROPIC_SLOT = "_request_anthropic_client_cache"
  18. _NO_SOCKETS_SUFFIX = " — no sockets found; in-flight request may keep running until the provider finishes"
  19. def _routermint_headers() -> dict:
  20. """User-Agent RouterMint needs to avoid Cloudflare 1010 blocks."""
  21. from hermes_cli.version_info import get_version_info
  22. return {"User-Agent": f"HermesAgent/{get_version_info().base_version}"}
  23. def _qwen_portal_headers() -> dict:
  24. import platform as _plat
  25. _ua = f"QwenCode/{_QWEN_CODE_VERSION} ({_plat.system().lower()}; {_plat.machine()})"
  26. return {
  27. "User-Agent": _ua, "X-DashScope-CacheControl": "enable", "X-DashScope-UserAgent": _ua,
  28. "X-DashScope-AuthType": "qwen-oauth",
  29. }
  30. # Route-specific default headers; first host match wins (order preserved from the original chain).
  31. # Builders resolve their module lazily so run_agent keeps its import-time cost and avoids cycles.
  32. _ROUTE_DEFAULT_HEADERS = (
  33. ("openrouter.ai", lambda self, url: _lazy_attr("agent.auxiliary_client", "build_or_headers")()),
  34. ("ai-gateway.vercel.sh", lambda self, url: _lazy_attr("agent.auxiliary_client", "_ai_gateway_headers")()),
  35. ("integrate.api.nvidia.com", lambda self, url: _lazy_attr("agent.auxiliary_client", "build_nvidia_nim_headers")(url)),
  36. ("api.routermint.com", lambda self, url: _routermint_headers()),
  37. ("api.kimi.com", lambda self, url: _lazy_attr("agent.auxiliary_client", "_ai_gateway_headers")()),
  38. ("portal.qwen.ai", lambda self, url: _qwen_portal_headers()),
  39. ("chatgpt.com", lambda self, url: _lazy_attr("agent.codex_headers", "codex_cloudflare_headers")(
  40. self._client_kwargs.get("api_key", ""), base_url=url)),
  41. # Covers provider=xai and provider=xai-oauth (api.x.ai).
  42. ("x.ai", lambda self, url: _lazy_attr("tools.xai_http", "hermes_xai_default_headers")()),
  43. )
  44. def _reset_slot(cache: dict, *, in_use: bool = False) -> None:
  45. cache["client"] = None
  46. cache["key"] = None
  47. cache["poisoned"] = False
  48. cache["in_use"] = in_use
  49. def _valid_credential_pair(api_key: Any, base_url: Any) -> bool:
  50. return bool(isinstance(api_key, str) and api_key.strip() and isinstance(base_url, str) and base_url.strip())
  51. def _swap_fallback_clients(agent, fb_client, fb_provider: str, fb_model: str, fb_base_url: str, fb_api_mode: str) -> None:
  52. """Install the fallback client(s) in place, honoring request_timeout_seconds (None = SDK default)."""
  53. timeout = get_provider_request_timeout(fb_provider, fb_model)
  54. if fb_provider == "bedrock" and fb_api_mode in ("anthropic_messages", "bedrock_converse"):
  55. # Non-Mantle Bedrock: boto3-chain auth, no OpenAI/Anthropic SDK client to carry over.
  56. from agent.bedrock_adapter import bind_bedrock_runtime
  57. bind_bedrock_runtime(agent, fb_base_url, fb_api_mode)
  58. return
  59. # The SDK exposes an empty/stale api_key when a rotating source is installed.
  60. key_provider = vars(fb_client).get("_api_key_provider")
  61. credential = key_provider if callable(key_provider) else fb_client.api_key
  62. if fb_api_mode == "anthropic_messages":
  63. from agent.anthropic_adapter import build_anthropic_client
  64. from agent.anthropic_credentials import resolve_anthropic_token, anthropic_route_is_oauth
  65. is_anthropic = fb_provider == "anthropic"
  66. effective_key = credential or (resolve_anthropic_token(model=getattr(agent, "model", None)) if is_anthropic else None) or ""
  67. agent.api_key = agent._anthropic_api_key = effective_key
  68. agent._anthropic_base_url = fb_base_url
  69. agent._anthropic_client = build_anthropic_client(effective_key, fb_base_url, timeout=timeout)
  70. agent._is_anthropic_oauth = anthropic_route_is_oauth(fb_base_url, effective_key, provider=fb_provider)
  71. agent.client, agent._client_kwargs = None, {}
  72. return
  73. agent.api_key = credential
  74. agent.client = fb_client
  75. # Keep provider headers resolve_provider_client() baked into fb_client (SDK: _custom_headers), else
  76. # later request-client rebuilds drop them and User-Agent-sentinel providers (Kimi Coding) 403.
  77. fb_headers = getattr(fb_client, "_custom_headers", None) or getattr(fb_client, "default_headers", None)
  78. agent._client_kwargs = {"api_key": credential, "base_url": fb_base_url}
  79. if fb_headers:
  80. agent._client_kwargs["default_headers"] = dict(fb_headers)
  81. if timeout is not None:
  82. agent._client_kwargs["timeout"] = timeout
  83. # Rebuild now so the timeout applies to the very next request, not only after a rotation rebuild.
  84. agent._replace_primary_openai_client(reason="fallback_timeout_apply")
  85. class ClientLifecycleMixin:
  86. def _close_task_resources(self, task_id: str) -> None:
  87. """Release task resources without treating a shared environment as process ownership."""
  88. from run_agent import _quietly, cleanup_browser, cleanup_vm
  89. def kill_processes() -> None:
  90. from tools.process_registry import process_registry
  91. # A session can run several task IDs; delegated IDs also differ from session_id.
  92. # Never match the environment key (e.g. "default"), shared by parent and siblings.
  93. owners = getattr(self, "_process_owner_task_ids", ())
  94. for process in process_registry.list_sessions():
  95. if process["owner_task_id"] in owners and process["status"] == "running":
  96. # An explicitly persisted job (terminal persist_on_release=true) survives
  97. # agent close — session end, compression, error recovery (#41225). The
  98. # user can still stop it on purpose via process_manage kill.
  99. if process.get("persist_on_release"):
  100. continue
  101. process_registry.kill_process(
  102. process["session_id"], source="agent_close", consume_output=True,
  103. )
  104. def release_computer_use() -> None:
  105. from tools.computer_use.tool import release_computer_use_session
  106. release_computer_use_session(task_id)
  107. def forget_file_state() -> None:
  108. # File tools key their read stamps / writer claims by the per-turn task_id (cron:
  109. # ``cron:<job>:<uuid>``, subagents: ``subagent-N-xxxx``), which differs from session_id;
  110. # cleanup_vm(session_id) alone leaves a finished run looking like a live sibling (#114446).
  111. from tools.file_tools import clear_file_ops_cache
  112. for owner in getattr(self, "_process_owner_task_ids", ()):
  113. if owner and owner != task_id:
  114. clear_file_ops_cache(owner)
  115. for step in (kill_processes, lambda: cleanup_vm(task_id), lambda: cleanup_browser(task_id),
  116. release_computer_use, forget_file_state):
  117. _quietly(step)
  118. def _client_log_context(self) -> str:
  119. thread = threading.current_thread()
  120. return (
  121. f"thread={thread.name}:{thread.ident} provider={getattr(self, 'provider', 'unknown')} "
  122. f"base_url={getattr(self, 'base_url', 'unknown')} model={getattr(self, 'model', 'unknown')}"
  123. )
  124. def _anthropic_log_context(self) -> str:
  125. return f"provider={getattr(self, 'provider', None)} model={getattr(self, 'model', None)}"
  126. def _openai_client_lock(self) -> threading.RLock:
  127. if getattr(self, "_client_lock", None) is None:
  128. self._client_lock = threading.RLock()
  129. return self._client_lock
  130. @staticmethod
  131. def _is_openai_client_closed(client: Any) -> bool:
  132. """Check if an OpenAI client is closed.
  133. Handles both property and method forms of is_closed:
  134. - httpx.Client.is_closed is a bool property
  135. - openai.OpenAI.is_closed is a method returning bool
  136. Prior bug: getattr(client, "is_closed", False) returned the bound method,
  137. which is always truthy, causing unnecessary client recreation on every call.
  138. """
  139. from unittest.mock import Mock
  140. if isinstance(client, Mock):
  141. return False
  142. is_closed_attr = getattr(client, "is_closed", None)
  143. if is_closed_attr is not None:
  144. # Handle method (openai SDK) vs property (httpx)
  145. if callable(is_closed_attr):
  146. if is_closed_attr():
  147. return True
  148. elif bool(is_closed_attr):
  149. return True
  150. http_client = getattr(client, "_client", None)
  151. if http_client is not None:
  152. return bool(getattr(http_client, "is_closed", False))
  153. return False
  154. @staticmethod
  155. def _build_keepalive_http_client(base_url: str = "", *, verify: Any = True) -> Any:
  156. """Build the shared OpenAI httpx client used by main and aux paths."""
  157. from agent.process_bootstrap import build_keepalive_http_client
  158. return build_keepalive_http_client(base_url, verify=verify)
  159. _create_openai_client = _forward("agent.agent_runtime_helpers", "create_openai_client")
  160. _force_close_tcp_sockets = _forward_static("agent.agent_runtime_helpers", "force_close_tcp_sockets")
  161. _cleanup_dead_connections = _forward("agent.agent_runtime_helpers_dead_connections", "cleanup_dead_connections")
  162. _run_codex_stream = _forward("agent.codex_runtime", "run_codex_stream")
  163. _recover_with_credential_pool = _forward("agent.agent_runtime_helpers", "recover_with_credential_pool")
  164. def _close_openai_client(self, client: Any, *, reason: str, shared: bool) -> None:
  165. if client is None:
  166. return
  167. ctx = self._client_log_context()
  168. # Force-close TCP sockets first (no CLOSE-WAIT accumulation), then the graceful SDK close.
  169. force_closed = self._force_close_tcp_sockets(client)
  170. try:
  171. client.close()
  172. logger.info("OpenAI client closed (%s, shared=%s, tcp_force_closed=%d) %s", reason, shared, force_closed, ctx)
  173. except Exception as exc:
  174. logger.debug("OpenAI client close failed (%s, shared=%s) %s error=%s", reason, shared, ctx, exc)
  175. def _retire_shared_openai_client(self, client: Any, *, reason: str) -> None:
  176. """Ownership-safe retirement of a replaced shared OpenAI client: ``shutdown()`` sockets only, FDs to GC.
  177. ``close()`` releases FDs from the calling thread while other threads may still hold the fd in an SSL BIO;
  178. a recycled fd then gets a TLS record written into an unrelated file (SQLite-header corruption).
  179. The shared primary client has no single owning thread — worker threads from stale-killed attempts
  180. may still be unwinding their SSL BIOs, and the codex-direct / MoA paths stream on the shared client
  181. itself. If we release an FD while another thread's SSL layer still caches the raw integer fd, the
  182. kernel can recycle it into an unrelated ``open()`` (e.g. ``kanban.db``) and the unwinding TLS flush
  183. then writes an application-data record into that file — the SQLite-header corruption documented in
  184. #29507/#70773.
  185. """
  186. if client is None:
  187. return
  188. try:
  189. shutdown_count = self._force_close_tcp_sockets(client)
  190. logger.info(
  191. "Shared OpenAI client retired (%s, tcp_shutdown=%d, fd_release=deferred_to_gc) %s",
  192. reason, shutdown_count, self._client_log_context(),
  193. )
  194. except Exception as exc:
  195. logger.debug("Shared OpenAI client retire failed (%s) %s error=%s", reason, self._client_log_context(), exc)
  196. def _drain_transports_after_abandonment(self, *, reason: str) -> int:
  197. """FD-safe transport drain for an abandoned (timed-out) worker; returns sockets shut down.
  198. The worker may be blocked in an OpenSSL read; hard-closing from the timeout thread releases FDs under a
  199. live BIO (native corruption / SIGSEGV). Only ``shutdown()`` so the read sees EOF and the worker closes itself.
  200. See #94248.
  201. A delegation deadline abandons this agent's daemon worker while it may still be blocked inside an
  202. in-flight OpenSSL ``read`` (Codex Responses stream, httpx request). This helper only ``shutdown()``s
  203. pooled sockets (safe from any thread), settling blocked reads with EOF/EPIPE so the worker can
  204. unwind and run the real close from its own thread. See #70773, #94248.
  205. """
  206. drained = 0
  207. # Shared primary client (codex-direct / MoA stream on it directly).
  208. try:
  209. client = getattr(self, "client", None)
  210. if client is not None:
  211. drained += self._force_close_tcp_sockets(client)
  212. except Exception:
  213. logger.debug("Abandoned-worker drain: shared client sweep failed", exc_info=True)
  214. # Cached per-request wire clients: abort (shutdown + poison the reuse slot) so the unwinding
  215. # worker discards them instead of re-caching.
  216. for slot_attr, abort, label in (
  217. (_OPENAI_SLOT, self._abort_request_openai_client, "request"),
  218. (_ANTHROPIC_SLOT, self._abort_request_anthropic_client, "anthropic"),
  219. ):
  220. try:
  221. with self._openai_client_lock():
  222. cache = getattr(self, slot_attr, None)
  223. cached = cache["client"] if cache else None
  224. if cached is not None:
  225. abort(cached, reason=reason)
  226. except Exception:
  227. logger.debug("Abandoned-worker drain: %s client abort failed", label, exc_info=True)
  228. # Codex app-server session watches a private interrupt event.
  229. try:
  230. request_interrupt = getattr(getattr(self, "_codex_session", None), "request_interrupt", None)
  231. if callable(request_interrupt):
  232. request_interrupt()
  233. except Exception:
  234. logger.debug("Abandoned-worker drain: codex interrupt failed", exc_info=True)
  235. # Inline (cron-style) request abort hook, when registered.
  236. try:
  237. abort_active = getattr(self, "_active_request_abort", None)
  238. if callable(abort_active):
  239. abort_active(reason)
  240. except Exception:
  241. logger.debug("Abandoned-worker drain: active request abort failed", exc_info=True)
  242. logger.info(
  243. "Abandoned-worker transports drained (%s, tcp_shutdown=%d, fd_release=deferred_to_worker) %s",
  244. reason, drained, self._client_log_context(),
  245. )
  246. return drained
  247. def _replace_primary_openai_client(self, *, reason: str) -> bool:
  248. with self._openai_client_lock():
  249. old_client = getattr(self, "client", None)
  250. try:
  251. # MoA's ``client`` is an in-process facade, not an SDK client; generic rebuilds must preserve that.
  252. if (getattr(self, "provider", "") or "").strip().lower() == "moa":
  253. from agent.moa_loop import build_moa_facade
  254. new_client = build_moa_facade(self, self.model)
  255. else:
  256. new_client = self._create_openai_client(self._client_kwargs, reason=reason, shared=True)
  257. except Exception as exc:
  258. logger.warning(
  259. "Failed to rebuild shared primary client (%s) %s error=%s", reason, self._client_log_context(), exc,
  260. )
  261. return False
  262. self.client = new_client
  263. # Never hard-close the replaced shared client (another thread may still be unwinding on the old pool).
  264. # #70773: never hard-close the replaced shared client from here — the caller may not be the thread
  265. # whose request is still unwinding on the old pool (credential rotation and dead-connection cleanup
  266. # run on the turn thread while stale-killed workers unwind; the codex-direct path streams on the
  267. # shared client itself). Retire it instead: sockets are shut down (FD-safe), FD release deferred to
  268. # GC.
  269. self._retire_shared_openai_client(old_client, reason=f"replace:{reason}")
  270. return True
  271. def _ensure_primary_openai_client(self, *, reason: str) -> Any:
  272. with self._openai_client_lock():
  273. client = getattr(self, "client", None)
  274. if client is not None and not self._is_openai_client_closed(client):
  275. return client
  276. try:
  277. new_client = self._create_openai_client(self._client_kwargs, reason=reason, shared=True)
  278. except Exception as exc:
  279. logger.warning(
  280. "Failed to recreate closed OpenAI client (%s) %s error=%s", reason, self._client_log_context(), exc,
  281. )
  282. raise RuntimeError("Failed to recreate closed OpenAI client") from exc
  283. self.client = new_client
  284. logger.warning(
  285. "Detected closed shared OpenAI client; recreated before use (%s) %s", reason, self._client_log_context(),
  286. )
  287. self._close_openai_client(client, reason=f"replace:{reason}", shared=True)
  288. return new_client
  289. @staticmethod
  290. def _api_kwargs_have_image_parts(api_kwargs: dict) -> bool:
  291. """True when the outbound request still has native image parts (Chat ``messages`` / Responses ``input``)."""
  292. if not isinstance(api_kwargs, dict):
  293. return False
  294. def _contains_image(value: Any) -> bool:
  295. if isinstance(value, dict):
  296. return value.get("type") in {"image_url", "input_image"} or any(
  297. _contains_image(v) for v in value.values()
  298. )
  299. return isinstance(value, list) and any(_contains_image(v) for v in value)
  300. return any(
  301. _contains_image(item)
  302. for field in ("messages", "input")
  303. if isinstance(api_kwargs.get(field), list)
  304. for item in api_kwargs[field]
  305. )
  306. # ------------------------------------------------------------------ per-request client slots
  307. # One single-slot cache per client kind: {"client", "key", "poisoned", "in_use"}. Reuse keeps the warm httpx
  308. # pool between sequential calls; ``in_use`` keeps concurrent calls off one pool; ``poisoned`` marks a pool
  309. # whose sockets were shut from a stranger thread (never reuse it).
  310. # Reuse reasons: closes from the FD-owning worker's own finally after a response — the only closes that
  311. # attest a healthy pool. Poisoning still wins.
  312. _REQUEST_CLIENT_REUSE_REASONS = frozenset({"request_complete", "stream_request_complete"})
  313. def _request_slot(self, slot_attr: str) -> dict:
  314. cache = getattr(self, slot_attr, None) # lazy: tests build agents via AIAgent.__new__ without __init__
  315. if cache is None:
  316. setattr(self, slot_attr, cache := {})
  317. _reset_slot(cache)
  318. return cache
  319. def _checkout_request_slot(self, slot_attr: str, key: Any) -> tuple:
  320. """Return ``(reusable_client, stale_client)``; at most one is non-None."""
  321. with self._openai_client_lock():
  322. cache = self._request_slot(slot_attr)
  323. cached = cache["client"]
  324. if cached is None or cache["in_use"]:
  325. return None, None
  326. if not cache["poisoned"] and cache["key"] == key and not self._is_openai_client_closed(cached):
  327. cache["in_use"] = True
  328. return cached, None
  329. # Key changed / poisoned / externally closed — rebuild. in_use was False, so closing the stale
  330. # client from this thread is FD-safe (no worker owns it).
  331. _reset_slot(cache)
  332. return None, cached
  333. def _store_request_slot(self, slot_attr: str, client: Any, key: Any) -> None:
  334. with self._openai_client_lock():
  335. cache = self._request_slot(slot_attr)
  336. if cache["client"] is None:
  337. cache.update(client=client, key=key, poisoned=False, in_use=True)
  338. # else: a concurrent call holds the slot — hand this client out untracked (fully closed later).
  339. def _release_request_slot(self, slot_attr: str, client: Any, reason: str) -> bool:
  340. """Owner-thread release; True when the client stays cached (clean finish, not poisoned)."""
  341. with self._openai_client_lock():
  342. cache = self._request_slot(slot_attr)
  343. if cache["client"] is client:
  344. if reason in self._REQUEST_CLIENT_REUSE_REASONS and not cache["poisoned"]:
  345. cache["in_use"] = False
  346. return True
  347. _reset_slot(cache)
  348. return False
  349. def _take_request_slot(self, slot_attr: str) -> tuple:
  350. """Teardown: empty the slot and return ``(client, was_in_use)``."""
  351. with self._openai_client_lock():
  352. cache = getattr(self, slot_attr, None)
  353. client, in_use = (cache["client"], bool(cache["in_use"])) if cache else (None, False)
  354. if cache is not None:
  355. _reset_slot(cache)
  356. return client, in_use
  357. def _abort_request_slot_client(self, slot_attr: str, client: Any, *, reason: str) -> None:
  358. """Cross-thread abort (interrupt loop, stale detector): ``shutdown(SHUT_RDWR)`` without releasing FDs.
  359. ``close()`` from a non-owning thread races the live SSL BIO and corrupts unrelated FDs; shutdown unblocks
  360. the owner's recv/send so it closes from its own context. The slot is poisoned so the pool is never reused.
  361. """
  362. if client is None:
  363. return
  364. anthropic = slot_attr == _ANTHROPIC_SLOT
  365. label = "Anthropic" if anthropic else "OpenAI"
  366. context = self._anthropic_log_context() if anthropic else self._client_log_context()
  367. with self._openai_client_lock():
  368. cache = self._request_slot(slot_attr)
  369. if cache["client"] is client:
  370. cache["poisoned"] = True
  371. try:
  372. # Non-HTTP providers cancel without closing owner-thread file descriptors.
  373. if callable(getattr(type(client), "cancel", None)):
  374. client.cancel()
  375. return
  376. shutdown_count = self._force_close_tcp_sockets(client)
  377. # Zero sockets shut down means the worker stays blocked — WARN, not success.
  378. # tcp_force_closed=0 means the stranger-thread abort found no sockets to shut down — the worker
  379. # stays blocked in recv and the provider keeps the slot (#72975). Surface that as WARNING so it
  380. # cannot be mistaken for a successful abort in the logs.
  381. # See #72975.
  382. _log = logger.warning if shutdown_count == 0 else logger.info
  383. _log(
  384. "%s client aborted (%s, shared=False, tcp_force_closed=%d, deferred_close=stranger_thread) %s%s",
  385. label, reason, shutdown_count, context, _NO_SOCKETS_SUFFIX if shutdown_count == 0 else "",
  386. )
  387. except Exception as exc:
  388. logger.debug("%s client abort failed (%s, shared=False) %s error=%s", label, reason, context, exc)
  389. def _create_request_openai_client(self, *, reason: str, api_kwargs: Optional[dict] = None) -> Any:
  390. from unittest.mock import Mock
  391. primary_client = self._ensure_primary_openai_client(reason=reason)
  392. if self.provider == "moa" or isinstance(primary_client, Mock):
  393. return primary_client
  394. with self._openai_client_lock():
  395. request_kwargs = dict(self._client_kwargs)
  396. # No SDK retry loop: the outer loop owns retries/rotation/fallback, and SDK retries stretch a hung
  397. # request ~3x past our stale detector.
  398. request_kwargs["max_retries"] = 0
  399. if self._api_kwargs_have_image_parts(api_kwargs or {}):
  400. # Image requests get the route's image client headers on a request-scoped client (cache-keyed apart).
  401. from agent.provider_route_hooks import route_answer
  402. image_headers = route_answer("client_headers", provider=self.provider or "", scope=self,
  403. base_url=str(request_kwargs.get("base_url", "")),
  404. api_key=str(request_kwargs.get("api_key", "") or ""), has_images=True)
  405. if image_headers:
  406. request_kwargs["default_headers"] = {**(request_kwargs.get("default_headers") or {}), **image_headers}
  407. cached, stale = self._checkout_request_slot(_OPENAI_SLOT, request_kwargs)
  408. if cached is not None:
  409. return cached
  410. if stale is not None:
  411. self._close_openai_client(stale, reason=f"reuse_evict:{reason}", shared=False)
  412. client = self._create_openai_client(request_kwargs, reason=reason, shared=False)
  413. # Snapshot nested dicts (default_headers) so an aliased inner object can't mutate the cache key.
  414. snapshot = {k: dict(v) if isinstance(v, dict) else v for k, v in request_kwargs.items()}
  415. self._store_request_slot(_OPENAI_SLOT, client, snapshot)
  416. return client
  417. def _close_request_openai_client(self, client: Any, *, reason: str) -> None:
  418. if not self._release_request_slot(_OPENAI_SLOT, client, reason):
  419. self._close_openai_client(client, reason=reason, shared=False)
  420. def _close_cached_request_openai_client(self, *, reason: str) -> None:
  421. """Teardown hook: really close the cached per-request wire client."""
  422. client, in_use = self._take_request_slot(_OPENAI_SLOT)
  423. if client is None:
  424. return
  425. if in_use:
  426. # Checked out by a worker: close() here would release FDs from a stranger thread. Abort the
  427. # sockets; the worker's own finally does the real close.
  428. self._abort_request_openai_client(client, reason=f"{reason}_in_flight")
  429. else:
  430. self._close_openai_client(client, reason=reason, shared=False)
  431. def _abort_request_openai_client(self, client: Any, *, reason: str) -> None:
  432. self._abort_request_slot_client(_OPENAI_SLOT, client, reason=reason)
  433. def _request_anthropic_client_key(self) -> tuple:
  434. """Cache key over everything forcing a fresh client: credential, base URL/region, timeout, 1M-beta flag."""
  435. if getattr(self, "provider", None) == "bedrock":
  436. return ("bedrock", getattr(self, "_bedrock_region", "us-east-1") or "us-east-1",
  437. get_provider_request_timeout(self.provider, self.model))
  438. return (
  439. "direct", self._anthropic_api_key, getattr(self, "_anthropic_base_url", None),
  440. get_provider_request_timeout(self.provider, self.model), bool(getattr(self, "_oauth_1m_beta_disabled", False)),
  441. )
  442. def _build_direct_anthropic_client(self, token: str, base_url: Any) -> Any:
  443. """Native Anthropic client for ``token``/``base_url`` with the provider/model request timeout."""
  444. from agent.anthropic_adapter import build_anthropic_client
  445. return build_anthropic_client(token, base_url, timeout=get_provider_request_timeout(self.provider, self.model))
  446. def _anthropic_oauth_flag(self, token: str) -> bool:
  447. """OAuth flag only on native Anthropic routes; third-party Anthropic-protocol endpoints must not trip OAuth paths."""
  448. from agent.anthropic_credentials import anthropic_route_is_oauth
  449. return anthropic_route_is_oauth(getattr(self, "_anthropic_base_url", None), token, provider=self.provider)
  450. def _build_anthropic_client_for_key(self, key: tuple) -> Any:
  451. from agent.anthropic_adapter import build_anthropic_bedrock_client, build_anthropic_client
  452. if key[0] == "bedrock":
  453. return build_anthropic_bedrock_client(key[1], timeout=key[2])
  454. return build_anthropic_client(key[1], key[2], timeout=key[3], drop_context_1m_beta=key[4])
  455. def _create_request_anthropic_client(self, *, reason: str) -> Any:
  456. """Build (or reuse) a request-local Anthropic client for one in-flight call.
  457. The watchdog must never ``close()`` a client a worker is still reading (fd recycled under a live SSL BIO →
  458. TLS record in a SQLite header); per-request clients let the stranger ``shutdown()`` while the owner closes.
  459. """
  460. if self.api_mode == "anthropic_messages":
  461. self._try_refresh_anthropic_client_credentials()
  462. key = self._request_anthropic_client_key()
  463. cached, stale = self._checkout_request_slot(_ANTHROPIC_SLOT, key)
  464. if cached is not None:
  465. return cached
  466. if stale is not None:
  467. self._close_request_anthropic_client(stale, reason=f"reuse_evict:{reason}")
  468. client = self._build_anthropic_client_for_key(key)
  469. logger.debug("Anthropic request client created (%s, shared=False) %s", reason, self._anthropic_log_context())
  470. self._store_request_slot(_ANTHROPIC_SLOT, client, key)
  471. return client
  472. def _close_request_anthropic_client(self, client: Any, *, reason: str) -> None:
  473. """Owner-thread close: clean finish keeps the pool warm; otherwise force-close sockets (CLOSE-WAIT) + SDK close."""
  474. if client is None or self._release_request_slot(_ANTHROPIC_SLOT, client, reason):
  475. return
  476. try:
  477. self._force_close_tcp_sockets(client)
  478. client.close()
  479. logger.info("Anthropic client closed (%s, shared=False) %s", reason, self._anthropic_log_context())
  480. except Exception as exc:
  481. logger.debug(
  482. "Anthropic client close failed (%s, shared=False) %s error=%s", reason, self._anthropic_log_context(), exc,
  483. )
  484. def _close_cached_request_anthropic_client(self, *, reason: str) -> None:
  485. """Teardown hook: really close the cached per-request Anthropic client."""
  486. client, in_use = self._take_request_slot(_ANTHROPIC_SLOT)
  487. if client is None:
  488. return
  489. if in_use: # checked out by a worker — same reasoning as the OpenAI teardown hook
  490. self._abort_request_anthropic_client(client, reason=f"{reason}_in_flight")
  491. return
  492. with suppress(Exception):
  493. self._force_close_tcp_sockets(client)
  494. client.close()
  495. def _abort_request_anthropic_client(self, client: Any, *, reason: str) -> None:
  496. self._abort_request_slot_client(_ANTHROPIC_SLOT, client, reason=reason)
  497. # ------------------------------------------------------------------ credential refresh
  498. def _sync_client_kwargs_credentials(self) -> None:
  499. """Mirror ``self.api_key`` / ``self.base_url`` into the OpenAI-style client kwargs."""
  500. self._client_kwargs["api_key"] = self.api_key
  501. self._client_kwargs["base_url"] = self.base_url
  502. def _adopt_openai_credentials(self, api_key: str, base_url: str, *, reason: str) -> bool:
  503. """Apply a fresh key/base_url to the OpenAI-style kwargs and rebuild the shared client."""
  504. self.api_key, self.base_url = api_key.strip(), base_url.strip().rstrip("/")
  505. self._sync_client_kwargs_credentials()
  506. return self._replace_primary_openai_client(reason=reason)
  507. def _try_refresh_codex_client_credentials(self, *, force: bool = True) -> bool:
  508. if self.api_mode != "codex_responses" or self.provider not in {"openai-codex", "xai-oauth"}:
  509. return False
  510. # No silent account swap: a non-singleton credential (manual pool entry, explicit api_key=) must not be
  511. # replaced by the device_code singleton's tokens — the pool's reactive recovery owns that case.
  512. try:
  513. from hermes_cli import auth as _auth
  514. resolve = (
  515. _auth.resolve_codex_runtime_credentials if self.provider == "openai-codex"
  516. else _auth.resolve_xai_oauth_runtime_credentials
  517. )
  518. singleton_now = resolve(refresh_if_expiring=False)
  519. except Exception as exc:
  520. logger.debug("%s singleton read failed: %s", self.provider, exc)
  521. return False
  522. singleton_key = str(singleton_now.get("api_key") or "").strip()
  523. old_key = str(self.api_key or "").strip()
  524. if singleton_key and old_key and singleton_key != old_key:
  525. logger.debug(
  526. "%s singleton tokens differ from the active api_key; skipping singleton force-refresh to avoid "
  527. "silent account swap. Reactive credential rotation should go through the pool.", self.provider,
  528. )
  529. return False
  530. try:
  531. creds = resolve(force_refresh=force)
  532. except Exception as exc:
  533. logger.debug("%s credential refresh failed: %s", self.provider, exc)
  534. return False
  535. api_key, base_url = creds.get("api_key"), creds.get("base_url")
  536. if not _valid_credential_pair(api_key, base_url):
  537. return False
  538. # No NEW token minted (the resolver returns the same stale token when refresh fails) → False.
  539. if old_key and api_key.strip() == old_key:
  540. logger.debug("%s credential refresh returned the same token; refresh likely failed silently", self.provider)
  541. return False
  542. return self._adopt_openai_credentials(api_key, base_url, reason=f"{self.provider}_credential_refresh")
  543. def _try_refresh_nous_client_credentials(self, *, force: bool = True, require_account: str | None = None) -> bool:
  544. # Portal serves anthropic/* on the native Messages route, so either client kind may hold the expiring JWT.
  545. if self.provider != "nous" or self.api_mode not in ("chat_completions", "anthropic_messages"):
  546. return False
  547. try:
  548. from hermes_cli.auth import resolve_nous_runtime_credentials
  549. timeout = env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15)
  550. # Pass the bearer that just 401'd so a refresh already done by a sibling process is
  551. # adopted instead of rotating the grant again.
  552. creds = resolve_nous_runtime_credentials(
  553. timeout_seconds=timeout, force_refresh=force, stale_access_token=self.api_key or None,
  554. )
  555. except Exception as exc:
  556. logger.debug("Nous credential refresh failed: %s", exc)
  557. return False
  558. api_key, base_url = creds.get("api_key"), creds.get("base_url")
  559. if not _valid_credential_pair(api_key, base_url):
  560. return False
  561. if (str(api_key).strip() == str(self.api_key or "").strip()
  562. and str(base_url).strip().rstrip("/") == str(self.base_url or "").strip().rstrip("/")):
  563. return False # store holds the same key on the same route: nothing to adopt, no client rebuild
  564. if require_account is not None:
  565. try:
  566. from hermes_cli.auth_constants import _decode_jwt_claims
  567. new_account = _decode_jwt_claims(str(api_key)).get("sub")
  568. except Exception:
  569. new_account = None
  570. if str(new_account or "") != require_account:
  571. logger.info(
  572. "Nous pre-expiry adoption skipped: the store's key belongs to a different account "
  573. "than the one in hand; keeping the current credential."
  574. )
  575. return False
  576. if self.api_mode == "anthropic_messages":
  577. self.api_key, self.base_url = api_key.strip(), base_url.strip().rstrip("/")
  578. self._anthropic_api_key, self._anthropic_base_url = self.api_key, self.base_url
  579. self._rebuild_anthropic_client()
  580. return True
  581. # Nous requests should not inherit OpenRouter-only attribution headers.
  582. self._client_kwargs.pop("default_headers", None)
  583. return self._adopt_openai_credentials(api_key, base_url, reason="nous_credential_refresh")
  584. # Adopt a fresh key this many seconds before the one in hand expires. Wider than the store's
  585. # own refresh skew (120 s) so the keepalive has normally already minted the replacement.
  586. _NOUS_KEY_ADOPT_SKEW_S = 180
  587. def _adopt_nous_key_before_expiry(self) -> bool:
  588. """Swap in a fresh Nous agent key BEFORE the one in hand expires, so the request never 401s.
  589. The agent key is a JWT; its ``exp`` is read locally (no network). Inside the skew the store
  590. is re-read under the auth-store lock: the keepalive thread normally holds a fresh key already
  591. (adopt, no POST), otherwise ONE refresh runs and every peer adopts its result. Before this,
  592. every agent in a process learned about the hourly expiry from its own 401, all in the same
  593. minute (620 in one 200-subagent run), and the pool benched the sole credential for all of
  594. them. Returns True when a new key was adopted.
  595. Identity guard: the replacement must belong to the SAME account (``sub`` claim) as the key
  596. in hand. The store holds the logged-in singleton; an agent running on an explicitly supplied
  597. or pool-selected key for a different account must never be silently moved onto it (that
  598. changes who is billed). When either side lacks a ``sub`` nothing is adopted here; the
  599. reactive 401 path is unchanged.
  600. """
  601. if getattr(self, "provider", "") != "nous" or not getattr(self, "api_key", None):
  602. return False
  603. try:
  604. from hermes_cli.auth_constants import _decode_jwt_claims
  605. claims = _decode_jwt_claims(self.api_key)
  606. except Exception:
  607. return False
  608. exp, account = claims.get("exp"), claims.get("sub")
  609. if not account or not isinstance(exp, (int, float)) or exp - time.time() > self._NOUS_KEY_ADOPT_SKEW_S:
  610. return False
  611. try:
  612. return self._try_refresh_nous_client_credentials(force=False, require_account=str(account))
  613. except Exception:
  614. logger.debug("Nous key pre-expiry adoption failed", exc_info=True)
  615. return False
  616. def _resolve_env_credentials(self) -> Optional[tuple]:
  617. """Current ``.env``-sourced ``(api_key, base_url, default_base)`` for this provider, or ``None``.
  618. Covers registry api-key providers and named custom providers with ``key_env``.
  619. """
  620. try:
  621. from agent.credential_pool import get_env_prefer_dotenv
  622. from hermes_cli.auth import PROVIDER_REGISTRY
  623. except ImportError:
  624. return None
  625. pconfig = PROVIDER_REGISTRY.get(self.provider)
  626. if pconfig and getattr(pconfig, "auth_type", "") == "api_key" and getattr(pconfig, "api_key_env_vars", ()):
  627. # First non-empty env var wins (lazy: later vars are not read).
  628. api_key = next((k for k in (get_env_prefer_dotenv(v).strip() for v in pconfig.api_key_env_vars) if k), "")
  629. if not api_key:
  630. return None
  631. url_var = pconfig.base_url_env_var
  632. env_url = get_env_prefer_dotenv(url_var).strip().rstrip("/") if url_var else ""
  633. default_base = (pconfig.inference_base_url or "").strip().rstrip("/")
  634. base_url = env_url or default_base
  635. if self.provider == "actual":
  636. from hermes_cli.auth import normalize_actual_base_url
  637. from hermes_cli.runtime_provider import _config_base_url_for_provider, _get_model_config
  638. configured_base = _config_base_url_for_provider(_get_model_config(), "actual")
  639. base_url = normalize_actual_base_url(configured_base or base_url)
  640. elif self.provider in ("kimi-coding", "zai"):
  641. from hermes_cli import auth as _auth
  642. resolver = _auth._resolve_kimi_base_url if self.provider == "kimi-coding" else _auth._resolve_zai_base_url
  643. base_url = resolver(api_key, pconfig.inference_base_url, env_url).rstrip("/")
  644. elif self.provider == "custom":
  645. # Named custom provider: identity in config, credential in key_env; no key_env → nothing to watch.
  646. try:
  647. from hermes_cli.runtime_provider import _get_named_custom_provider
  648. except ImportError:
  649. return None
  650. custom_provider = _get_named_custom_provider(getattr(self, "requested_provider", "") or "")
  651. key_env = str((custom_provider or {}).get("key_env") or "").strip()
  652. api_key = get_env_prefer_dotenv(key_env).strip() if key_env else ""
  653. if not custom_provider or not api_key:
  654. return None
  655. # Custom providers pin base_url in config, so only key edits are adopted here.
  656. base_url = default_base = str(custom_provider.get("base_url") or "").strip().rstrip("/")
  657. else:
  658. return None
  659. if not base_url:
  660. return None
  661. return api_key, base_url, default_base
  662. def _should_adopt_env_credentials(self, api_key: str, base_url: str, default_base: str) -> bool:
  663. """Adopt only env *edits* (resolved value changed since last look), never drift from ``self.*``:
  664. pool rotation/failover and a config ``model.base_url`` legitimately move the session."""
  665. prev = getattr(self, "_env_creds_seen", None)
  666. current_base = (self.base_url or "").strip().rstrip("/")
  667. unchanged = base_url == current_base and api_key == self.api_key
  668. if prev is None:
  669. # First look: adopt only the boot-default case (anything else is unattributable on turn one), and
  670. # never stomp a pool-rotated key.
  671. pool_rotated = (
  672. api_key != self.api_key and getattr(self, "_credential_pool", None) is not None
  673. and getattr(self, "_credential_pool_entry_id", None)
  674. )
  675. return current_base == default_base and not unchanged and not pool_rotated
  676. # Adopt only while the session still runs on the registry default or the previously-seen env value.
  677. return (base_url, api_key) != prev and current_base in {default_base, prev[0]} and not unchanged
  678. def _try_refresh_env_client_credentials(self) -> bool:
  679. """Adopt ``~/.hermes/.env`` credential/base-url edits at the turn boundary (a Settings save updates ``.env``
  680. but a live worker keeps init-time values). Adoption rule: ``_should_adopt_env_credentials``.
  681. Covers api-key registry providers and named custom providers with a ``key_env`` (#67935) — the
  682. latter resolve to ``provider="custom"`` with no registry entry, so they are matched through the
  683. runtime provider's config lookup instead.
  684. """
  685. if self.api_mode != "chat_completions" or getattr(self, "_fallback_activated", False):
  686. return False
  687. resolved = self._resolve_env_credentials()
  688. if resolved is None:
  689. return False
  690. api_key, base_url, default_base = resolved
  691. if not self._should_adopt_env_credentials(api_key, base_url, default_base):
  692. self._env_creds_seen = (base_url, api_key)
  693. return False
  694. from hermes_cli.route_identity import normalize_route_base_url
  695. route_changed = normalize_route_base_url(self.base_url) != normalize_route_base_url(base_url)
  696. prior_api_key, prior_base_url = self.api_key, self.base_url
  697. prior_client_kwargs = dict(self._client_kwargs)
  698. self.api_key, self.base_url = api_key, base_url
  699. self._sync_client_kwargs_credentials()
  700. # A base-url change moves the route: recompute TLS material and default headers.
  701. self._reapply_route_client_config(route_changed=route_changed)
  702. if not self._replace_primary_openai_client(reason="env_credential_refresh"):
  703. # Leave the baseline un-advanced (retry next turn); roll the agent back to match the live client.
  704. self.api_key, self.base_url = prior_api_key, prior_base_url
  705. self._client_kwargs.clear()
  706. self._client_kwargs.update(prior_client_kwargs)
  707. return False
  708. # Rebind the pool entry id to the adopted key, or the next 429 quarantines the wrong credential.
  709. try:
  710. from agent.agent_runtime_helpers import sync_credential_pool_entry_id
  711. sync_credential_pool_entry_id(self)
  712. except Exception:
  713. logger.debug("sync_credential_pool_entry_id after env refresh failed", exc_info=True)
  714. self._env_creds_seen = (base_url, api_key)
  715. logger.info("Applied updated .env credentials for %s: endpoint %s", self.provider, self.base_url)
  716. return True
  717. def _try_refresh_vertex_client_credentials(self) -> bool:
  718. """Re-mint the Vertex OAuth2 token (~1h TTL; long sessions 401 on the expired bearer) and rebuild the client."""
  719. if self.api_mode != "chat_completions" or self.provider != "vertex":
  720. return False
  721. try:
  722. from agent.vertex_adapter import get_vertex_config
  723. token, base_url = get_vertex_config()
  724. except Exception as exc:
  725. logger.debug("Vertex credential refresh failed: %s", exc)
  726. return False
  727. ok = _valid_credential_pair(token, base_url) and self._adopt_openai_credentials(
  728. token, base_url, reason="vertex_credential_refresh",
  729. )
  730. if ok:
  731. logger.info("Vertex AI OAuth token refreshed")
  732. return ok
  733. def _try_refresh_provider_runtime_credentials(self, *, reason: str) -> bool:
  734. """Re-mint the live credential through the provider profile's ``refresh_runtime_credentials``
  735. and rebuild the shared client (caller enforces the single-shot guard). ``reason``:
  736. ``unauthorized`` (401) or ``stale`` (a non-auth error the profile reads as a stale credential)."""
  737. from agent.conversation_loop import _credential_refresh_profile
  738. from agent.provider_route_hooks import ask_on_error_path
  739. profile = _credential_refresh_profile(self)
  740. refreshed = ask_on_error_path(profile, "refresh_runtime_credentials", None, reason=reason) if profile else None
  741. if not (isinstance(refreshed, (tuple, list)) and len(refreshed) == 2 and refreshed[0]):
  742. return False
  743. token, base_url = refreshed
  744. self.api_key = token
  745. if base_url:
  746. self.base_url = base_url.rstrip("/")
  747. self._sync_client_kwargs_credentials()
  748. self._apply_client_headers_for_base_url(str(self.base_url or ""))
  749. return self._replace_primary_openai_client(reason=f"{profile.name}_credential_{reason}")
  750. def _try_refresh_anthropic_client_credentials(self) -> bool:
  751. # Only native Anthropic rotates OAuth tokens; other anthropic_messages providers (MiniMax, Alibaba, ...)
  752. # and Azure use static keys — a refresh would pick up the ~/.claude OAuth token and break auth.
  753. anthropic_base_url = getattr(self, "_anthropic_base_url", "") or ""
  754. if (
  755. self.api_mode != "anthropic_messages" or not hasattr(self, "_anthropic_api_key")
  756. or self.provider != "anthropic" or base_url_host_matches(anthropic_base_url, "azure.com")
  757. ):
  758. return False
  759. # Off the official hosts (a /anthropic proxy the resolver accepts, or a URL-bearing alias,
  760. # #28660) rotate only a credential the endpoint already holds: swapping a custom key for
  761. # ANTHROPIC_API_KEY / the OAuth token would leak it (#17829). Hostname match, not substring,
  762. # so ``proxy.example/anthropic.com`` stays foreign.
  763. official_host = not anthropic_base_url or any(
  764. base_url_host_matches(anthropic_base_url, host) for host in ("anthropic.com", "claude.com"))
  765. current_key = str(self._anthropic_api_key or "")
  766. if not official_host and not (current_key.startswith("sk-ant-") or getattr(self, "_is_anthropic_oauth", False)):
  767. return False
  768. try:
  769. from agent.anthropic_credentials import resolve_anthropic_token
  770. new_token = resolve_anthropic_token(model=self.model)
  771. except Exception as exc:
  772. logger.debug("Anthropic credential refresh failed: %s", exc)
  773. return False
  774. new_token = new_token.strip() if isinstance(new_token, str) else ""
  775. if not new_token or new_token == self._anthropic_api_key:
  776. return False
  777. with suppress(Exception):
  778. self._anthropic_client.close()
  779. try:
  780. base_url = getattr(self, "_anthropic_base_url", None)
  781. self._anthropic_client = self._build_direct_anthropic_client(new_token, base_url)
  782. except Exception as exc:
  783. logger.warning("Failed to rebuild Anthropic client after credential refresh: %s", exc)
  784. return False
  785. old_token = self._anthropic_api_key
  786. self._anthropic_api_key, self._is_anthropic_oauth = new_token, self._anthropic_oauth_flag(new_token)
  787. # Claude Code revokes the old token on refresh: every holder of it (agent.api_key, the compressor's
  788. # main_runtime, the fallback-restore snapshot) must move too, or compression 401s for the session's life.
  789. if self.api_key == old_token:
  790. self.api_key = new_token
  791. cc = getattr(self, "context_compressor", None)
  792. if cc is not None and getattr(cc, "api_key", None) == old_token:
  793. cc.api_key = new_token
  794. rt = getattr(self, "_primary_runtime", None) or {}
  795. if rt.get("anthropic_api_key") == old_token: # fallback restore rebuilds from the key + flag pair
  796. rt["is_anthropic_oauth"] = self._is_anthropic_oauth
  797. for k in ("api_key", "anthropic_api_key", "compressor_api_key"):
  798. if rt.get(k) == old_token:
  799. rt[k] = new_token
  800. # The turn's aux runtime was published before the first request triggered this refresh, so
  801. # same-turn `auto` aux calls (approvals, goal judge, plugin llm) would still send the revoked token.
  802. from agent.auxiliary_key_rotation import rotate_runtime_main_api_key
  803. rotate_runtime_main_api_key(old_token, new_token)
  804. return True
  805. # ------------------------------------------------------------------ route-derived client config
  806. def _apply_client_headers_for_base_url(self, base_url: str, *, apply_user_headers: bool = True) -> None:
  807. for host, build in _ROUTE_DEFAULT_HEADERS:
  808. if base_url_host_matches(base_url, host):
  809. self._client_kwargs["default_headers"] = build(self, base_url)
  810. break
  811. else:
  812. # No URL-specific headers — a profile's endpoint headers, else profile.default_headers, else clear.
  813. self._client_kwargs.pop("default_headers", None)
  814. from agent.provider_route_hooks import route_answer
  815. route_headers = route_answer("client_headers", provider=self.provider or "", base_url=base_url, scope=self,
  816. api_key=str(self._client_kwargs.get("api_key", "") or ""))
  817. if route_headers:
  818. self._client_kwargs["default_headers"] = dict(route_headers)
  819. with suppress(Exception):
  820. from providers import get_provider_profile
  821. profile = get_provider_profile(self.provider)
  822. if (not route_headers and profile and profile.default_headers
  823. and (profile_headers := dict(profile.default_headers))):
  824. self._client_kwargs["default_headers"] = profile_headers
  825. # User overrides win over URL/profile defaults for the same route; a swap to another endpoint must not
  826. # inherit them.
  827. if apply_user_headers:
  828. self._apply_user_default_headers()
  829. # Per-provider extra_headers last so they survive swaps/rebuilds. SECURITY: may carry credentials; never log.
  830. if self.api_mode not in ("anthropic_messages", "bedrock_converse"):
  831. try:
  832. from hermes_cli.config import apply_custom_provider_extra_headers_to_client_kwargs
  833. apply_custom_provider_extra_headers_to_client_kwargs(self._client_kwargs, base_url)
  834. except Exception:
  835. logger.debug("custom-provider extra_headers skipped", exc_info=True)
  836. def _apply_user_default_headers(self) -> None:
  837. """Merge config ``model.default_headers`` onto the OpenAI client (user wins; WAFs rejecting SDK headers).
  838. Delegates to ``agent.auxiliary_client`` so main and aux clients cannot drift. No-op for Anthropic/Bedrock."""
  839. if self.api_mode in ("anthropic_messages", "bedrock_converse"):
  840. return
  841. from agent.auxiliary_client import _apply_user_default_headers as _merge_user_headers
  842. merged = _merge_user_headers(self._client_kwargs.get("default_headers"))
  843. if merged:
  844. self._client_kwargs["default_headers"] = merged
  845. def _swap_credential(self, entry) -> bool:
  846. """Adopt *entry* as the live credential. Returns False, changing nothing, when the entry's
  847. route cannot serve this conversation's model (a conversation's model is never rewritten by a
  848. rotation; the caller treats a refused swap as "no entry")."""
  849. runtime_key = getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "")
  850. runtime_base = getattr(entry, "runtime_base_url", None) or getattr(entry, "base_url", None) or self.base_url
  851. from hermes_cli.providers import is_actual_route
  852. actual_route = is_actual_route(getattr(self, "provider", ""), runtime_base)
  853. if actual_route:
  854. from hermes_cli.auth import normalize_actual_base_url
  855. runtime_base = normalize_actual_base_url(runtime_base)
  856. stripped_base = runtime_base.rstrip("/") if isinstance(runtime_base, str) else runtime_base
  857. # Refuse BEFORE any state changes below: a refused swap must leave the agent exactly as it was.
  858. from hermes_cli.anon_auth import route_can_serve_model
  859. if not route_can_serve_model(getattr(self, "provider", None), stripped_base, getattr(self, "model", None)):
  860. logger.info("Credential %s skipped: its route cannot serve model %s", getattr(entry, "id", "?"), self.model)
  861. return False
  862. if actual_route:
  863. self.api_mode = "chat_completions"
  864. if hasattr(self, "_transport_cache"):
  865. self._transport_cache.clear()
  866. self._credential_pool_entry_id = getattr(entry, "id", None)
  867. from hermes_cli.route_identity import normalize_route_base_url
  868. route_changed = normalize_route_base_url(self.base_url) != normalize_route_base_url(runtime_base)
  869. if self.api_mode == "anthropic_messages":
  870. with suppress(Exception):
  871. self._anthropic_client.close()
  872. self._anthropic_api_key, self._anthropic_base_url = runtime_key, stripped_base
  873. self._anthropic_client = self._build_direct_anthropic_client(runtime_key, self._anthropic_base_url)
  874. self._is_anthropic_oauth = self._anthropic_oauth_flag(runtime_key)
  875. self.api_key, self.base_url = runtime_key, stripped_base
  876. return True
  877. self.api_key, self.base_url = runtime_key, stripped_base
  878. # Inlined (not _sync_client_kwargs_credentials): tests call this unbound on a SimpleNamespace agent.
  879. self._client_kwargs["api_key"] = self.api_key
  880. self._client_kwargs["base_url"] = self.base_url
  881. self._reapply_route_client_config(route_changed=route_changed)
  882. self._replace_primary_openai_client(reason="credential_rotation")
  883. return True
  884. def _reapply_route_client_config(self, *, route_changed: bool) -> None:
  885. """Recompute route-derived client kwargs (TLS material, default headers) for ``self.base_url``.
  886. Any rebuild that may have moved ``base_url`` must call this or the new endpoint inherits the old config.
  887. """
  888. self._client_kwargs.pop("ssl_verify", None)
  889. self._client_kwargs.pop("ssl_ca_cert", None)
  890. try:
  891. from hermes_cli.config import (
  892. apply_custom_provider_tls_to_client_kwargs, get_compatible_custom_providers, load_config_readonly,
  893. )
  894. apply_custom_provider_tls_to_client_kwargs(
  895. self._client_kwargs, str(self.base_url or ""), get_compatible_custom_providers(load_config_readonly()),
  896. )
  897. except Exception:
  898. logger.debug("custom-provider TLS resolution skipped on credential rotation", exc_info=True)
  899. self._apply_client_headers_for_base_url(self.base_url, apply_user_headers=not route_changed)
  900. def _anthropic_messages_create(self, api_kwargs: dict, *, client: Any = None):
  901. # A supplied request-local client was already refreshed in _create_request_anthropic_client.
  902. if client is None and self.api_mode == "anthropic_messages":
  903. self._try_refresh_anthropic_client_credentials()
  904. # Strips Responses-only kwargs that leak in under an api_mode-flip race.
  905. from agent.anthropic_adapter import create_anthropic_message
  906. # on_response: rate-limit + credits state live in response headers, which the parsed Message drops.
  907. return create_anthropic_message(
  908. client or self._anthropic_client, api_kwargs, log_prefix=getattr(self, "log_prefix", ""),
  909. prefer_stream=not bool(getattr(self, "_disable_streaming", False)),
  910. on_response=self._capture_anthropic_response_headers,
  911. )
  912. def _rebuild_anthropic_client(self) -> None:
  913. """Rebuild the Anthropic client after an interrupt/stale call (Bedrock SDK for bedrock; honors 1M-beta flag)."""
  914. self._anthropic_client = self._build_anthropic_client_for_key(self._request_anthropic_client_key())