turn_response_check.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352
  1. """Post-call response verification for the conversation turn's retry loop: stop the thinking
  2. spinner, validate the response shape (retry / eager fallback / terminal invalid-response
  3. result), derive ``finish_reason`` per api_mode, route content-policy refusals and
  4. ``length`` truncation, fold usage into the compressor, and mark the logical relay call
  5. complete. Nothing here imports ``agent.conversation_loop`` at module level (cycle) —
  6. loop-internal helpers resolve lazily so ``patch("agent.conversation_loop.X")`` keeps intercepting.
  7. """
  8. from __future__ import annotations
  9. from dataclasses import dataclass
  10. import logging
  11. import time
  12. from typing import Any, Dict, Optional
  13. from agent.error_classifier import FailoverReason
  14. from agent.turn_api_call import stop_thinking_spinner
  15. from agent.turn_failure_copy import invalid_response_failure_reason, provider_label_for, site_copy, stamp_failure
  16. from agent.turn_truncation import handle_content_policy_refusal, recover_from_truncation
  17. from agent.turn_usage import record_response_usage
  18. logger = logging.getLogger("agent.conversation_loop")
  19. @dataclass
  20. class ResponseCheckVerdict:
  21. """``action``: ``"break"`` (leave the retry loop — success, or a fallback/refusal restart
  22. armed on ``_retry``), ``"continue"`` (retry the API call) or ``"return"`` (``result`` is
  23. the turn's result dict). The other fields are the retry-loop locals rebound."""
  24. action: str
  25. thinking_spinner: Any
  26. messages: Any
  27. active_system_prompt: Any
  28. finish_reason: Any
  29. retry_count: Any
  30. compression_attempts: Any
  31. length_continue_retries: Any
  32. truncated_response_parts: Any
  33. truncated_tool_call_retries: Any
  34. _preflight_compression_blocked: Any
  35. _last_preflight_pressure: Any
  36. api_duration: Any
  37. result: Optional[dict[str, Any]] = None
  38. def _codex_finish_reason(response: Any) -> str:
  39. """Responses API max-output exhaustion is a normal Codex incomplete turn: route it to
  40. the Codex continuation path (``"incomplete"``), not the length rollback."""
  41. status = getattr(response, "status", None)
  42. if isinstance(status, str):
  43. status = status.strip().lower()
  44. incomplete_details = getattr(response, "incomplete_details", None)
  45. if isinstance(incomplete_details, dict):
  46. incomplete_reason = incomplete_details.get("reason")
  47. else:
  48. incomplete_reason = getattr(incomplete_details, "reason", None)
  49. if incomplete_reason is not None:
  50. incomplete_reason = str(incomplete_reason).strip().lower()
  51. if status == "incomplete" and incomplete_reason in {"max_output_tokens", "length"}:
  52. return "incomplete"
  53. if status == "incomplete" and incomplete_reason == "content_filter":
  54. return "content_filter"
  55. return "stop"
  56. def _derive_finish_reason(agent: Any, response: Any, messages: Any) -> str:
  57. if agent.api_mode == "codex_responses":
  58. finish_reason = _codex_finish_reason(response)
  59. # A function_call cut off by max_output_tokens is not a text turn to continue: the
  60. # Codex incomplete path would replay the partial and re-hit the same cap. Route it
  61. # to the length path so the same call is retried with a boosted budget (#91770).
  62. if finish_reason == "incomplete" and agent._get_transport().normalize_response(response).tool_calls:
  63. return "length"
  64. return finish_reason
  65. transport = agent._get_transport()
  66. if agent.api_mode == "anthropic_messages":
  67. return transport.response_finish_reason(response)
  68. normalized = transport.normalize_response(response) # Bedrock already normalized at dispatch
  69. finish_reason = normalized.finish_reason
  70. if agent.api_mode != "bedrock_converse" and agent._should_treat_stop_as_truncated(
  71. finish_reason, normalized, messages
  72. ):
  73. agent._vprint(
  74. f"{agent.log_prefix}⚠️ Treating suspicious Ollama/GLM stop response as truncated",
  75. force=True, diagnostic=True,
  76. )
  77. return "length"
  78. return finish_reason
  79. def check_api_response(
  80. agent: Any, *, response: Any, _retry: Any, thinking_spinner: Any, messages: Any,
  81. api_messages: Any, api_kwargs: Any, active_system_prompt: Any, conversation_history: Any,
  82. finish_reason: Any, retry_count: Any, max_retries: Any, compression_attempts: Any,
  83. max_compression_attempts: Any, length_continue_retries: Any, truncated_response_parts: Any,
  84. truncated_tool_call_retries: Any, current_turn_user_idx: Any, api_call_count: Any,
  85. api_request_id: Any, api_start_time: Any, effective_task_id: Any, turn_id: Any,
  86. _preflight_compression_blocked: Any, _last_preflight_pressure: Any,
  87. ) -> ResponseCheckVerdict:
  88. """Verify ``response`` in the original order. The retry buffer is NOT cleared on success
  89. (bytes back != usable content); ``_preflight_compression_blocked``/``_last_preflight_pressure``
  90. reset only when the usage fold re-arms the compression budget."""
  91. from agent.turn_recovery import validate_response_shape
  92. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ResponseCheckVerdict:
  93. return ResponseCheckVerdict(
  94. action=action, thinking_spinner=thinking_spinner, messages=messages,
  95. active_system_prompt=active_system_prompt, finish_reason=finish_reason,
  96. retry_count=retry_count, compression_attempts=compression_attempts,
  97. length_continue_retries=length_continue_retries,
  98. truncated_response_parts=truncated_response_parts,
  99. truncated_tool_call_retries=truncated_tool_call_retries,
  100. _preflight_compression_blocked=_preflight_compression_blocked,
  101. _last_preflight_pressure=_last_preflight_pressure, api_duration=api_duration,
  102. result=result,
  103. )
  104. api_duration = time.time() - api_start_time
  105. # Silent stop: the response box / tool messages that follow are more informative.
  106. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  107. if not agent.quiet_mode:
  108. agent._vprint(f"{agent.log_prefix}⏱️ API call completed in {api_duration:.2f}s")
  109. if agent.verbose_logging:
  110. resp_model = getattr(response, 'model', 'N/A') if response else 'N/A'
  111. logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}")
  112. response_invalid, error_details = validate_response_shape(agent, response)
  113. if response_invalid:
  114. _iv = retry_invalid_response(
  115. agent, response=response, error_details=error_details, _retry=_retry,
  116. thinking_spinner=thinking_spinner, messages=messages, api_messages=api_messages,
  117. api_kwargs=api_kwargs, active_system_prompt=active_system_prompt,
  118. conversation_history=conversation_history, retry_count=retry_count,
  119. max_retries=max_retries, compression_attempts=compression_attempts,
  120. api_call_count=api_call_count, api_request_id=api_request_id,
  121. api_start_time=api_start_time, api_duration=api_duration,
  122. effective_task_id=effective_task_id, turn_id=turn_id,
  123. )
  124. thinking_spinner = _iv.thinking_spinner
  125. active_system_prompt = _iv.active_system_prompt
  126. retry_count = _iv.retry_count
  127. compression_attempts = _iv.compression_attempts
  128. if _iv.action != "fallthrough":
  129. return _verdict(_iv.action, _iv.result)
  130. agent._turn_received_provider_response = True
  131. finish_reason = _derive_finish_reason(agent, response, messages)
  132. from hermes_cli.observability.shared_metrics_harness import record_reply_finish
  133. record_reply_finish(agent, response, finish_reason)
  134. # HTTP-200 refusals are deterministic: one fallback try, else return the refusal.
  135. if finish_reason == "content_filter":
  136. _rv = handle_content_policy_refusal(
  137. agent, response, _retry, thinking_spinner=thinking_spinner, messages=messages,
  138. api_messages=api_messages, api_kwargs=api_kwargs,
  139. active_system_prompt=active_system_prompt, conversation_history=conversation_history,
  140. api_call_count=api_call_count, effective_task_id=effective_task_id, turn_id=turn_id,
  141. api_request_id=api_request_id, api_start_time=api_start_time, retry_count=retry_count,
  142. max_retries=max_retries,
  143. )
  144. thinking_spinner = None
  145. active_system_prompt = _rv.active_system_prompt
  146. if _rv.action == "return":
  147. return _verdict("return", _rv.result)
  148. retry_count = 0
  149. compression_attempts = 0
  150. return _verdict("break")
  151. if finish_reason == "length":
  152. _tv = recover_from_truncation(
  153. agent, response, finish_reason, _retry, messages=messages,
  154. conversation_history=conversation_history, api_kwargs=api_kwargs,
  155. api_call_count=api_call_count, effective_task_id=effective_task_id,
  156. current_turn_user_idx=current_turn_user_idx,
  157. length_continue_retries=length_continue_retries,
  158. truncated_response_parts=truncated_response_parts,
  159. truncated_tool_call_retries=truncated_tool_call_retries, retry_count=retry_count,
  160. compression_attempts=compression_attempts,
  161. )
  162. messages = _tv.messages
  163. length_continue_retries = _tv.length_continue_retries
  164. truncated_response_parts = _tv.truncated_response_parts
  165. truncated_tool_call_retries = _tv.truncated_tool_call_retries
  166. retry_count = _tv.retry_count
  167. compression_attempts = _tv.compression_attempts
  168. if _tv.action in ("return", "break", "continue"):
  169. return _verdict(_tv.action, _tv.result)
  170. # Fold provider usage into compressor / anchors / session counters / state.db
  171. # (agent/turn_usage.py). A rearmed budget also clears the preflight-block latch.
  172. _usage_outcome = record_response_usage(
  173. agent, response, messages=messages, api_call_count=api_call_count,
  174. api_duration=api_duration, compression_attempts=compression_attempts,
  175. max_compression_attempts=max_compression_attempts,
  176. )
  177. compression_attempts = _usage_outcome.compression_attempts
  178. if _usage_outcome.rearmed:
  179. _preflight_compression_blocked = False
  180. _last_preflight_pressure = None
  181. _retry.has_retried_429 = False
  182. # Clearing Nous rate-limit state proves the limit reset so other sessions may resume.
  183. if agent.provider == "nous":
  184. try:
  185. from agent.nous_rate_guard import clear_nous_rate_limit
  186. from hermes_cli.anon_auth import is_anonymous_agent
  187. clear_nous_rate_limit(anonymous=is_anonymous_agent(agent))
  188. except Exception:
  189. pass
  190. from agent import relay_llm
  191. relay_llm.complete_logical_call(api_request_id, outcome="success")
  192. agent._touch_activity(f"API call #{api_call_count} completed")
  193. return _verdict("break")
  194. @dataclass
  195. class InvalidResponseVerdict:
  196. """``action``: ``"continue"`` (retry the API call after backoff), ``"break"`` (fallback
  197. armed / redirect pending) or ``"return"`` (``result``: terminal invalid-response result or
  198. interrupt during backoff). Rebinds ``thinking_spinner``/``active_system_prompt``/
  199. ``retry_count``/``compression_attempts``."""
  200. action: str
  201. thinking_spinner: Any
  202. active_system_prompt: Any
  203. retry_count: Any
  204. compression_attempts: Any
  205. result: Optional[dict[str, Any]] = None
  206. def retry_invalid_response(
  207. agent: Any, *, response: Any, error_details: Any, _retry: Any, thinking_spinner: Any,
  208. messages: Any, api_messages: Any, api_kwargs: Any, active_system_prompt: Any,
  209. conversation_history: Any, retry_count: Any, max_retries: Any, compression_attempts: Any,
  210. api_call_count: Any, api_request_id: Any, api_start_time: Any, api_duration: Any,
  211. effective_task_id: Any, turn_id: Any,
  212. ) -> InvalidResponseVerdict:
  213. """Malformed/empty provider response: fire the error hook, stop the spinner, eager
  214. fallback (empty responses often mean rate limiting), terminal result at max retries,
  215. else jittered backoff that preserves a pending redirect."""
  216. from agent.conversation_loop import _arm_fallback_restart
  217. from agent.retry_utils import jittered_backoff
  218. from agent.turn_recovery import (
  219. classify_codex_soft_failure, describe_invalid_response, interruptible_backoff_sleep,
  220. )
  221. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> InvalidResponseVerdict:
  222. return InvalidResponseVerdict(
  223. action=action, thinking_spinner=thinking_spinner,
  224. active_system_prompt=active_system_prompt, retry_count=retry_count,
  225. compression_attempts=compression_attempts, result=result,
  226. )
  227. agent._invoke_api_request_error_hook(
  228. task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
  229. api_call_count=api_call_count, api_start_time=api_start_time, api_kwargs=api_kwargs,
  230. error_type="InvalidAPIResponse",
  231. error_message=", ".join(error_details) or "Invalid API response",
  232. status_code=getattr(getattr(response, "error", None), "code", None),
  233. retry_count=retry_count, max_retries=max_retries, retryable=True, reason="invalid_response",
  234. )
  235. # Retry status is buffered and only surfaced if every retry+fallback exhausts.
  236. thinking_spinner = stop_thinking_spinner(agent, thinking_spinner)
  237. # Codex reports quota exhaustion as HTTP 200 ``status=failed`` — the SDK never raises, so the
  238. # exception path's credential-pool rotation never sees it. Same-provider recovery for the
  239. # pool-recoverable reasons FIRST (a healthy sibling account beats burning cross-provider
  240. # fallback); content-policy and other failures keep the fallback/retry path (#24159).
  241. _soft, _soft_ctx = classify_codex_soft_failure(agent, response)
  242. if _soft is not None and (_soft.reason in (FailoverReason.rate_limit, FailoverReason.billing) or _soft.is_auth):
  243. _recovered, _retry.has_retried_429 = agent._recover_with_credential_pool(
  244. status_code=None, has_retried_429=_retry.has_retried_429, classified_reason=_soft.reason,
  245. error_context=_soft_ctx, billing_unverified=_soft.billing_unverified,
  246. )
  247. if _recovered:
  248. agent._buffer_diagnostic_status(f"🔄 Codex soft failure ({_soft.reason.value}) — switched to the next pool credential, retrying...")
  249. return _verdict("continue")
  250. retry_count += 1
  251. # Eager fallback: empty/malformed responses often mean rate limiting.
  252. if agent._fallback_index < len(agent._fallback_chain):
  253. agent._buffer_diagnostic_status("⚠️ Empty/malformed response — switching to fallback...")
  254. if agent._try_activate_fallback():
  255. active_system_prompt = _arm_fallback_restart(
  256. agent, api_messages, active_system_prompt, _retry)
  257. retry_count = 0
  258. compression_attempts = 0
  259. return _verdict("break")
  260. error_msg, provider_name, _failure_hint = describe_invalid_response(
  261. agent, response, api_duration
  262. )
  263. agent._buffer_vprint(f"⚠️ Invalid API response (attempt {retry_count}/{max_retries}): {', '.join(error_details)}")
  264. agent._buffer_vprint(f" 🏢 Provider: {provider_name}")
  265. agent._buffer_vprint(f" 📝 Provider message: {agent._clean_error_message(error_msg)}")
  266. agent._buffer_vprint(f" ⏱️ {_failure_hint}")
  267. if retry_count >= max_retries:
  268. if agent._has_pending_fallback():
  269. agent._buffer_diagnostic_status(f"⚠️ Max retries ({max_retries}) for invalid responses — trying fallback...")
  270. if agent._try_activate_fallback():
  271. active_system_prompt = _arm_fallback_restart(
  272. agent, api_messages, active_system_prompt, _retry)
  273. retry_count = 0
  274. compression_attempts = 0
  275. return _verdict("break")
  276. # Terminal — flush buffered retry trace so user sees what happened.
  277. agent._flush_status_buffer()
  278. agent._emit_diagnostic_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.")
  279. logger.error("%sInvalid API response after %d retries.", agent.log_prefix, max_retries)
  280. agent._persist_session(messages, conversation_history)
  281. # "model=<id>" is describe_invalid_response's OpenRouter fallback, not a provider name.
  282. _label = (
  283. provider_label_for(agent.provider)
  284. if provider_name in ("Unknown", "") or provider_name.startswith("model=")
  285. else provider_name
  286. )
  287. _final_response = site_copy(
  288. "invalid_response", label=_label, attempts=max_retries, detail=_failure_hint,
  289. )
  290. return _verdict("return", stamp_failure({
  291. "final_response": _final_response,
  292. "messages": messages,
  293. "completed": False,
  294. "api_calls": api_call_count,
  295. "error": f"Invalid API response after {max_retries} retries: {_failure_hint}",
  296. "failed": True,
  297. }, invalid_response_failure_reason(response), True))
  298. wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0)
  299. agent._buffer_vprint(f"⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...")
  300. logger.warning("Invalid API response (retry %d/%d): %s | Provider: %s", retry_count, max_retries, ', '.join(error_details), provider_name)
  301. # A redirect cancels only the live request; the helper preserves the pending
  302. # correction (restart_with_redirected_messages) instead of clear_interrupt()-ing it.
  303. _interrupted = interruptible_backoff_sleep(
  304. agent, wait_time, _retry, messages=messages, conversation_history=conversation_history,
  305. api_call_count=api_call_count,
  306. abort_message="Interrupt detected during retry wait, aborting.",
  307. interrupt_text=f"Operation interrupted during retry ({_failure_hint}, attempt {retry_count}/{max_retries}).",
  308. activity_label=f"retry backoff ({retry_count}/{max_retries})",
  309. )
  310. if _interrupted is not None:
  311. return _verdict("return", _interrupted)
  312. if _retry.restart_with_redirected_messages:
  313. return _verdict("break") # rebuild this iteration from the correction
  314. return _verdict("continue")