turn_api_call.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. """The provider call for the conversation turn's retry loop: ``nous_rate_limit_guard`` (skip
  2. the attempt while another session's Nous Portal rate limit is active), ``perform_api_call``
  3. (streaming decision, MoA prepared-request handshake, LLM execution middleware wrapper, the
  4. redirect ``_model_request_active`` bracket and the response-vs-redirect crossing check) and
  5. ``handle_api_interrupt`` (``InterruptedError`` mid-call). Nothing here imports
  6. ``agent.conversation_loop`` at module level (cycle).
  7. """
  8. from __future__ import annotations
  9. from contextlib import nullcontext
  10. from dataclasses import dataclass
  11. import logging
  12. import time
  13. from typing import Any, Dict, Optional
  14. from agent.error_classifier import FailoverReason
  15. from agent.agent_runtime_helpers_placeholders import hidden_interrupt_placeholder_row
  16. from agent.message_metadata import append_message
  17. from agent.repetition_guard import REPETITION_LOOP_INTERRUPTED, is_runaway_repetition
  18. from agent.turn_failure_copy import site_copy, stamp_failure
  19. logger = logging.getLogger("agent.conversation_loop")
  20. def stop_thinking_spinner(agent: Any, thinking_spinner: Any) -> None:
  21. """Stop the spinner silently and clear the thinking callback; returns ``None`` so
  22. callers can rebind ``thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)``."""
  23. if thinking_spinner:
  24. thinking_spinner.stop("")
  25. if agent.thinking_callback:
  26. agent.thinking_callback("")
  27. @dataclass
  28. class ApiCallVerdict:
  29. """``action``: ``"fallthrough"`` (``response`` is ready for verification) or ``"break"``
  30. (a redirect crossed the response — rebuild armed on ``_retry`` or ``interrupted``)."""
  31. action: str
  32. response: Any
  33. thinking_spinner: Any
  34. interrupted: Any
  35. def _should_stream(agent: Any) -> bool:
  36. """Prefer streaming for liveness even without display consumers. Process providers
  37. must opt in; ACP transports and explicit session/provider disables stay non-streaming."""
  38. if getattr(agent, "_disable_streaming", False):
  39. return False
  40. _base = str(agent.base_url or "").lower()
  41. if _base.startswith(("acp://", "acp+tcp://")):
  42. return False
  43. from providers import get_provider_profile
  44. from hermes_cli.runtime_provider_backends import _is_external_process_provider
  45. profile = get_provider_profile(agent.provider)
  46. streaming = getattr(profile, "supports_streaming", None)
  47. if streaming is False:
  48. return False
  49. if _is_external_process_provider(agent.provider) and streaming is not True:
  50. return False
  51. if not agent._has_stream_consumers():
  52. if agent.provider == "moa":
  53. return False
  54. from unittest.mock import Mock
  55. if isinstance(getattr(agent, "client", None), Mock):
  56. return False
  57. return True
  58. def perform_api_call(
  59. agent: Any, *, api_kwargs: Any, _original_api_kwargs: Any, _llm_middleware_trace: Any,
  60. _moa_prepared_request: Any, _retry: Any, thinking_spinner: Any, retry_count: Any,
  61. api_call_count: Any, api_request_id: Any, effective_task_id: Any, turn_id: Any,
  62. interrupted: Any,
  63. ) -> ApiCallVerdict:
  64. """Issue the request (see ``_should_stream`` for the streaming decision)."""
  65. response = None
  66. def _verdict(action: str) -> ApiCallVerdict:
  67. return ApiCallVerdict(
  68. action=action, response=response, thinking_spinner=thinking_spinner,
  69. interrupted=interrupted,
  70. )
  71. def _stop_spinner():
  72. nonlocal thinking_spinner
  73. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  74. _use_streaming = _should_stream(agent)
  75. def _perform_api_call(next_api_kwargs):
  76. if agent.api_mode == "codex_responses":
  77. from agent.codex_responses_adapter import drops_replay_item_ids
  78. next_api_kwargs = agent._get_transport().preflight_kwargs(
  79. next_api_kwargs, allow_stream=False, drop_replay_item_ids=drops_replay_item_ids(agent.base_url),
  80. sanitize_harmony_tokens=agent._is_codex_backend(),
  81. )
  82. if _use_streaming:
  83. return agent._interruptible_streaming_api_call(
  84. next_api_kwargs, on_first_delta=_stop_spinner
  85. )
  86. from agent import relay_llm
  87. return relay_llm.execute(
  88. next_api_kwargs,
  89. agent._interruptible_api_call,
  90. session_id=str(agent.session_id or ""),
  91. name=str(agent.provider or "provider"),
  92. model_name=str(agent.model or ""),
  93. metadata={
  94. "api_mode": agent.api_mode,
  95. "api_request_id": api_request_id,
  96. "call_role": (
  97. "delegated"
  98. if getattr(agent, "is_subagent", False)
  99. else "fallback"
  100. if int(getattr(agent, "_fallback_index", 0) or 0) > 0
  101. else "primary"
  102. ),
  103. "retry_count": retry_count,
  104. },
  105. defer_logical_completion=True,
  106. )
  107. from hermes_cli.middleware import run_llm_execution_middleware
  108. # The ``_model_request_active`` bracket is taken under the redirect lock when one exists,
  109. # so redirect() can't observe a half-toggled flag.
  110. _model_request_active = getattr(agent, "_model_request_active", None)
  111. _redirect_lock = getattr(agent, "_pending_redirect_lock", None)
  112. _bracket = nullcontext() if _redirect_lock is None else _redirect_lock
  113. with _bracket:
  114. _redirect_before_request = bool(getattr(agent, "_pending_redirect", None))
  115. if not _redirect_before_request and _model_request_active is not None:
  116. _model_request_active.set()
  117. if _redirect_before_request:
  118. # A correction arrived after begin_iteration, during request preparation.
  119. # Rebuild before sending the stale prompt, using the usual bounded retry.
  120. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  121. if agent.clear_interrupt(preserve_redirect=True):
  122. _retry.restart_with_redirected_messages = True
  123. else:
  124. interrupted = True
  125. return _verdict("break")
  126. try:
  127. response = run_llm_execution_middleware(
  128. api_kwargs, _perform_api_call, original_request=_original_api_kwargs,
  129. task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
  130. session_id=agent.session_id or "", platform=agent.platform or "", model=agent.model,
  131. provider=agent.provider, base_url=agent.base_url, api_mode=agent.api_mode,
  132. api_call_count=api_call_count, middleware_trace=list(_llm_middleware_trace),
  133. )
  134. finally:
  135. with _bracket:
  136. if _model_request_active is not None:
  137. _model_request_active.clear()
  138. _redirect_crossed_response = (
  139. bool(agent._pending_redirect) if _redirect_lock is not None
  140. else agent._has_pending_redirect()
  141. )
  142. if _redirect_crossed_response:
  143. # Response and redirect can cross threads: discard the now-stale
  144. # response and rebuild from the correction rather than lose it.
  145. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  146. if agent.clear_interrupt(preserve_redirect=True):
  147. _retry.restart_with_redirected_messages = True
  148. else:
  149. interrupted = True
  150. return _verdict("break")
  151. return _verdict("fallthrough")
  152. @dataclass
  153. class ApiInterruptVerdict:
  154. """Always ``action == "break"`` (leave the retry loop): either a redirect restart was
  155. armed on ``_retry`` or the turn is ``interrupted`` with ``final_response`` set."""
  156. action: str
  157. thinking_spinner: Any
  158. interrupted: Any
  159. final_response: Any
  160. def handle_api_interrupt(
  161. agent: Any, *, _retry: Any, thinking_spinner: Any, messages: Any, conversation_history: Any,
  162. api_start_time: Any, interrupted: Any, final_response: Any,
  163. ) -> ApiInterruptVerdict:
  164. """``InterruptedError`` during the provider call: a pending redirect keeps its correction
  165. queued for the outer-loop rebuild; otherwise keep any streamed partial text so the next
  166. turn has a record of the half-finished reply."""
  167. from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX
  168. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  169. # redirect() cancelled only this request: keep the correction queued, clear the
  170. # cancellation bit, let the outer loop rebuild. Never materialize incomplete
  171. # signed/encrypted reasoning items.
  172. if agent._has_pending_redirect() and agent.clear_interrupt(preserve_redirect=True):
  173. _retry.restart_with_redirected_messages = True
  174. return ApiInterruptVerdict("break", thinking_spinner, interrupted, final_response)
  175. api_elapsed = time.time() - api_start_time
  176. agent._vprint(f"{agent.log_prefix}⚡ Interrupted during API call.", force=True)
  177. interrupted = True
  178. # A Stop during the empty-response nudge request leaves the synthetic assistant+nudge
  179. # pair after an executed tool result; strip it so the row appended below follows the tool
  180. # row (the finalizer then closes the tail with this exit's own reason).
  181. agent._drop_trailing_empty_response_scaffolding(messages)
  182. _partial = agent._strip_think_blocks(
  183. getattr(agent, "_current_streamed_assistant_text", "") or ""
  184. ).strip()
  185. if _partial and is_runaway_repetition(_partial):
  186. # The interrupted row is replayed next turn; looped bytes there re-seed the loop
  187. # (#112764). Same hidden shape as the redirect placeholder: nothing visible in the
  188. # transcript, a neutral api_content so the pre-call sanitizer does not re-heal it.
  189. append_message(messages, hidden_interrupt_placeholder_row())
  190. final_response = REPETITION_LOOP_INTERRUPTED
  191. elif _partial:
  192. append_message(messages, {
  193. "role": "assistant", "content": _partial, "display_metadata": {"interrupted": True},
  194. })
  195. final_response = _partial
  196. else:
  197. final_response = f"{INTERRUPT_WAITING_FOR_MODEL_PREFIX}{api_elapsed:.1f}s elapsed)."
  198. agent._persist_session(messages, conversation_history)
  199. return ApiInterruptVerdict("break", thinking_spinner, interrupted, final_response)
  200. @dataclass
  201. class NousRateGuardVerdict:
  202. """``action``: ``"fallthrough"`` (no active limit — make the call), ``"break"``
  203. (fallback armed on ``_retry``) or ``"return"`` (``result``: no fallback available)."""
  204. action: str
  205. active_system_prompt: Any
  206. retry_count: Any
  207. compression_attempts: Any
  208. result: Optional[dict[str, Any]] = None
  209. def nous_rate_limit_guard(
  210. agent: Any, *, _retry: Any, api_messages: Any, messages: Any, conversation_history: Any,
  211. active_system_prompt: Any, retry_count: Any, compression_attempts: Any, api_call_count: Any,
  212. ) -> NousRateGuardVerdict:
  213. """Skip the call if another session recorded a Nous Portal rate limit: every attempt (incl.
  214. SDK retries) counts against RPH. Never lets the guard itself break the agent loop."""
  215. from agent.conversation_loop import _arm_fallback_restart
  216. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> NousRateGuardVerdict:
  217. return NousRateGuardVerdict(
  218. action=action, active_system_prompt=active_system_prompt, retry_count=retry_count,
  219. compression_attempts=compression_attempts, result=result,
  220. )
  221. if agent.provider == "nous":
  222. # A gateway ``x-nous-model-switch`` recorded on the previous response moves this session
  223. # (and the config default, when it still names the free tier's model) before the next call.
  224. try:
  225. from hermes_cli.anon_auth import apply_model_switch
  226. apply_model_switch(agent)
  227. except Exception:
  228. pass
  229. try:
  230. from agent.nous_rate_guard import (
  231. nous_rate_limit_remaining, format_remaining as _fmt_nous_remaining
  232. )
  233. from hermes_cli import anon_auth
  234. _anonymous = anon_auth.is_anonymous_agent(agent)
  235. _nous_remaining = nous_rate_limit_remaining(anonymous=_anonymous)
  236. if _nous_remaining is not None and _nous_remaining > 0:
  237. reset = _fmt_nous_remaining(_nous_remaining)
  238. if _anonymous:
  239. _nous_msg = anon_auth.FREE_TIER_RATE_LIMIT_CHAT.format(
  240. reset=anon_auth.friendly_wait(_nous_remaining))
  241. else:
  242. _nous_msg = f"Your Nous account has hit its rate limit; it resets in {reset}."
  243. agent._buffer_vprint(f"⏳ {_nous_msg} Trying fallback...")
  244. agent._buffer_diagnostic_status(f"⏳ {_nous_msg}")
  245. if agent._try_activate_fallback():
  246. active_system_prompt = _arm_fallback_restart(
  247. agent, api_messages, active_system_prompt, _retry)
  248. retry_count = 0
  249. compression_attempts = 0
  250. return _verdict("break")
  251. # No fallback — surface the buffered rate-limit context that led here.
  252. agent._flush_status_buffer()
  253. agent._persist_session(messages, conversation_history)
  254. # The free tier's sentence already says what to do (wait, or sign in); the
  255. # fallback-provider advice is for an install that runs its own providers.
  256. return _verdict("return", stamp_failure({
  257. "final_response": (f"⏳ {_nous_msg}" if _anonymous
  258. else f"⏳ {_nous_msg}\n\n{site_copy('nous_rate_limit')}"),
  259. "messages": messages,
  260. "api_calls": api_call_count,
  261. "completed": False,
  262. "failed": True,
  263. "error": _nous_msg,
  264. # The free tier's card body and its sign-in door (agent/error_surface.py).
  265. **({"free_tier": {"kind": "rate_limited", "message": anon_auth.FREE_TIER_RATE_LIMIT_CARD.format(
  266. reset=anon_auth.friendly_wait(_nous_remaining))}} if _anonymous else {}),
  267. }, FailoverReason.rate_limit.value, True))
  268. except Exception:
  269. pass # Never let rate guard break the agent loop
  270. return _verdict("fallthrough")