turn_empty_response.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315
  1. """Empty / thinking-only final-response recovery ladder for the conversation turn loop.
  2. Runs when the model returned no visible text after ``<think>`` blocks. Ladder order is
  3. load-bearing: partial-stream recovery → reuse prior turn content (housekeeping tools only)
  4. → one post-tool-call nudge → thinking-only prefill continuation (×2) → empty-response
  5. retries (budgeted, deterministic-empty short-circuit) → fallback provider → terminal
  6. ``(empty)`` sentinel. Nothing here imports ``agent.conversation_loop`` at module level.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import re
  11. from dataclasses import dataclass
  12. from typing import Any, Dict, List, Optional
  13. from agent import empty_response_guard as _empty_guard
  14. from agent.message_metadata import append_message
  15. from agent.message_sanitization import _sanitize_surrogates
  16. from agent.turn_context_compaction import _refund_api_call
  17. from agent.turn_recovery import interruptible_backoff_sleep
  18. logger = logging.getLogger("agent.conversation_loop")
  19. _INLINE_THINK_RE = re.compile(r'<think>|<thinking>|<reasoning>', re.IGNORECASE)
  20. @dataclass
  21. class EmptyResponseVerdict:
  22. """Outcome of ``recover_empty_response``.
  23. ``action``: ``"break"`` (turn is done — ``final_response`` is set), ``"continue"``
  24. (re-enter the OUTER turn loop: a nudge/prefill row was appended, a retry wait
  25. elapsed, or a fallback was activated and preflight must re-run), ``"return"``
  26. (interrupted during a retry wait — return ``result``) or ``"fallthrough"``
  27. (unreachable: every path exits; kept for the contract)."""
  28. action: str
  29. result: Optional[dict[str, Any]]
  30. final_response: Any
  31. turn_exit_reason: Any
  32. active_system_prompt: Any
  33. preflight_compression_blocked: bool
  34. api_call_count: int
  35. def _retry_empty(
  36. agent: Any, response: Any, finish_reason: str, empty_candidate: bool, *, messages: Any,
  37. conversation_history: Any, api_call_count: int, observed_generation: bool = False,
  38. ) -> tuple:
  39. """Budgeted empty-response retry. Each empty attempt re-bills the full input, so the
  40. signature is recorded and deterministic empties stop burning paid retries (fails
  41. open: missing usage or any output keeps the budget). Returns
  42. ``(action_or_None, interrupt_result, deterministic_empty)``."""
  43. from agent.retry_utils import jittered_backoff
  44. if empty_candidate:
  45. _empty_guard.record_empty_attempt(
  46. agent, finish_reason=finish_reason, response=response, observed_generation=observed_generation,
  47. )
  48. budget = (
  49. _empty_guard.empty_retry_budget(agent, response)
  50. if empty_candidate else _empty_guard.DEFAULT_EMPTY_RETRY_BUDGET
  51. )
  52. deterministic = empty_candidate and _empty_guard.deterministic_empty(agent)
  53. if not (empty_candidate and agent._empty_content_retries < budget and not deterministic):
  54. return None, None, deterministic
  55. agent._empty_content_retries += 1
  56. n = agent._empty_content_retries
  57. wait_time = jittered_backoff(n, base_delay=5.0, max_delay=60.0)
  58. logger.warning(
  59. "Empty response (no content or reasoning) — retry %d/%d in %.1fs (model=%s)",
  60. n, budget, wait_time, agent.model,
  61. )
  62. _budget_note = (
  63. " — high-cost request, reduced retry budget"
  64. if budget < _empty_guard.DEFAULT_EMPTY_RETRY_BUDGET else ""
  65. )
  66. agent._buffer_diagnostic_status(
  67. f"⚠️ Empty response from model — retrying ({n}/{budget}) in {wait_time:.0f}s{_budget_note}"
  68. )
  69. _interrupted = interruptible_backoff_sleep(
  70. agent, wait_time, None,
  71. messages=messages,
  72. conversation_history=conversation_history,
  73. api_call_count=api_call_count,
  74. abort_message="Interrupt detected during empty-response retry wait, aborting.",
  75. interrupt_text=(
  76. f"Operation interrupted: retrying empty response from model (retry {n}/{budget})."
  77. ),
  78. activity_label=f"empty response retry backoff ({n}/{budget})",
  79. )
  80. if _interrupted is not None:
  81. return "return", _interrupted, deterministic
  82. return "continue", None, deterministic
  83. def _terminal_empty(agent: Any, assistant_message: Any, finish_reason: str, messages: Any) -> str:
  84. """Retries and fallback exhausted: persist the ``(empty)`` sentinel row and return the
  85. delivery text. Reasoning is never copied into the delivery text here — the persisted row
  86. keeps the sentinel so later "continue" turns don't replay it and loop on empties."""
  87. _streak_cost = _empty_guard.streak_cost_usd(agent)
  88. if _streak_cost is not None:
  89. agent._buffer_diagnostic_status(
  90. f"ℹ️ Estimated cost of these empty attempts: ~${_streak_cost:.2f} (input tokens are billed "
  91. f"per attempt even when no answer is produced)"
  92. )
  93. agent._flush_status_buffer()
  94. reasoning_text = agent._extract_reasoning(assistant_message)
  95. agent._drop_trailing_empty_response_scaffolding(messages)
  96. assistant_msg = agent._build_assistant_message(assistant_message, finish_reason)
  97. assistant_msg["content"] = "(empty)"
  98. assistant_msg["_empty_terminal_sentinel"] = True
  99. append_message(messages, assistant_msg)
  100. if not reasoning_text:
  101. logger.warning(
  102. "Empty response (no content or reasoning) after %d retries. No fallback available. "
  103. "model=%s provider=%s",
  104. agent._empty_content_retries, agent.model,
  105. agent.provider,
  106. )
  107. agent._emit_diagnostic_status(
  108. "❌ Model returned no content after all retries"
  109. + (" and fallback attempts." if agent._fallback_chain else
  110. ". No fallback providers configured.")
  111. )
  112. return "(empty)"
  113. logger.warning(
  114. "Reasoning-only response after %d retries; keeping it out of the terminal result "
  115. "(model=%s provider=%s)",
  116. agent._empty_content_retries, agent.model, agent.provider,
  117. )
  118. agent._emit_diagnostic_status(
  119. "⚠️ Model produced reasoning but no visible response after all retries. Returning empty."
  120. )
  121. return "(empty)"
  122. def housekeeping_reply_line(agent: Any) -> Optional[str]:
  123. """The line written beside a housekeeping-only tool round (think blocks stripped), the turn's
  124. reply when the follow-up adds nothing; None when there is none (substantive tools mean it was
  125. narration)."""
  126. fallback = getattr(agent, '_last_content_with_tools', None)
  127. if not (fallback and getattr(agent, '_last_content_tools_all_housekeeping', False)):
  128. return None
  129. return _sanitize_surrogates(agent._strip_think_blocks(fallback).strip())
  130. def take_housekeeping_reply_line(agent: Any) -> Optional[str]:
  131. """Consume ``housekeeping_reply_line`` as the turn's final response."""
  132. final_response = housekeeping_reply_line(agent)
  133. if final_response is None:
  134. return None
  135. agent._last_content_with_tools = None
  136. agent._last_content_tools_all_housekeeping = False
  137. # Do NOT modify the assistant message content (injected text poisoned history).
  138. # The final IS the response streamed before the housekeeping tool: name that identity
  139. # so a client settles the text it already shows instead of painting it a second time.
  140. # A held turn never showed it: its rewrite goes out as a new final.
  141. if getattr(agent, "_hold_stream", False) is not True:
  142. agent._response_was_previewed = True
  143. agent._reused_response_text = final_response
  144. return final_response
  145. def recover_empty_response(
  146. agent: Any, assistant_message: Any, response: Any, finish_reason: str, *, final_response: Any,
  147. messages: list[dict[str, Any]], api_messages: Any, conversation_history: Any,
  148. active_system_prompt: Any, api_call_count: int, turn_exit_reason: Any,
  149. preflight_compression_blocked: bool,
  150. ) -> EmptyResponseVerdict:
  151. """Recover from a final response with no visible content (see module docstring for
  152. the ladder). Role alternation is preserved: the post-tool nudge appends the empty
  153. assistant row BEFORE the user-level hint (APIs reject tool→user)."""
  154. from agent.conversation_loop import _EMPTY_TOOL_RESPONSE_NUDGE, _sync_failover_system_message
  155. _turn_exit_reason = turn_exit_reason
  156. _preflight_compression_blocked = preflight_compression_blocked
  157. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> EmptyResponseVerdict:
  158. return EmptyResponseVerdict(
  159. action=action, result=result, final_response=final_response,
  160. turn_exit_reason=_turn_exit_reason, active_system_prompt=active_system_prompt,
  161. preflight_compression_blocked=_preflight_compression_blocked,
  162. api_call_count=api_call_count,
  163. )
  164. # Partial stream recovery: content streamed before the connection died becomes the
  165. # final response instead of fallback or retries.
  166. _partial_streamed = getattr(agent, "_current_streamed_assistant_text", "") or ""
  167. if agent._has_content_after_think_block(_partial_streamed):
  168. _turn_exit_reason = "partial_stream_recovery"
  169. _recovered = agent._strip_think_blocks(_partial_streamed).strip()
  170. logger.info(
  171. "Partial stream content delivered (%d chars) — using as final response",
  172. len(_recovered),
  173. )
  174. agent._emit_diagnostic_status("↻ Stream interrupted — using delivered content " "as final response")
  175. final_response = _recovered
  176. # A streamed fragment isn't a confirmed preview: gateway fallback delivery
  177. # sends the text plus the abnormal-turn explanation.
  178. agent._response_was_previewed = False
  179. return _verdict("break")
  180. # Prior turn had real content + ONLY housekeeping tools: model is done, reuse it.
  181. # With substantive tools it was mid-task narration and the empty reply is a choke;
  182. # let the post-tool nudge handle it.
  183. if (final := take_housekeeping_reply_line(agent)) is not None:
  184. _turn_exit_reason = "fallback_prior_turn_content"
  185. logger.info("Empty follow-up after tool calls — using prior turn content as final response")
  186. agent._emit_diagnostic_status("↻ Empty response after tool calls — using earlier content as final answer")
  187. agent._empty_content_retries = 0
  188. final_response = final
  189. return _verdict("break")
  190. # Post-tool-call empty (no prior content, or only mid-task narration): nudge once.
  191. _prior_was_tool = any(m.get("role") == "tool" for m in messages[-5:])
  192. # Ollama puts <think> in content, not reasoning_content, so _has_structured misses
  193. # it; detect here to route to prefill.
  194. _has_inline_thinking = bool(_INLINE_THINK_RE.search(final_response or ""))
  195. if (
  196. _prior_was_tool
  197. and not getattr(agent, "_post_tool_empty_retried", False)
  198. and not _has_inline_thinking # thinking model still working — let prefill handle
  199. ):
  200. agent._post_tool_empty_retried = True
  201. # Clear stale narration so it doesn't resurface on a later empty response.
  202. agent._last_content_with_tools = None
  203. agent._last_content_tools_all_housekeeping = False
  204. logger.info("Empty response after tool calls — nudging model " "to continue processing")
  205. agent._buffer_diagnostic_status("⚠️ Model returned empty after tool calls — " "nudging to continue")
  206. # tool → assistant("(empty)") → user keeps the sequence valid.
  207. _nudge_msg = agent._build_assistant_message(assistant_message, finish_reason)
  208. _nudge_msg["content"] = "(empty)"
  209. _nudge_msg["_empty_recovery_synthetic"] = True
  210. append_message(messages, _nudge_msg)
  211. append_message(messages, {
  212. "role": "user", "content": _EMPTY_TOOL_RESPONSE_NUDGE, "_empty_recovery_synthetic": True
  213. })
  214. return _verdict("continue")
  215. # Thinking-only prefill: append the reasoning as-is and continue so the model sees
  216. # its own reasoning and writes text.
  217. _has_structured = bool(
  218. getattr(assistant_message, "reasoning", None)
  219. or getattr(assistant_message, "reasoning_content", None)
  220. or getattr(assistant_message, "reasoning_details", None)
  221. or _has_inline_thinking
  222. )
  223. if _has_structured and agent._thinking_prefill_retries < 2:
  224. agent._thinking_prefill_retries += 1
  225. logger.info(
  226. "Thinking-only response (no visible content) — prefilling to continue (%d/2)",
  227. agent._thinking_prefill_retries,
  228. )
  229. agent._buffer_diagnostic_status(
  230. f"↻ Thinking-only response — prefilling to continue ({agent._thinking_prefill_retries}/2)"
  231. )
  232. interim_msg = agent._build_assistant_message(assistant_message, "incomplete")
  233. interim_msg["_thinking_prefill"] = True
  234. append_message(messages, interim_msg)
  235. agent._session_messages = messages
  236. return _verdict("continue")
  237. # Empty-response retries: truly empty replies AND reasoning-only replies after
  238. # prefill exhaustion.
  239. _truly_empty = not agent._strip_think_blocks(final_response).strip()
  240. _empty_candidate = _truly_empty and (not _has_structured or agent._thinking_prefill_retries >= 2)
  241. action, interrupt_result, _deterministic_empty = _retry_empty(
  242. agent, response, finish_reason, _empty_candidate, messages=messages,
  243. conversation_history=conversation_history, api_call_count=api_call_count,
  244. observed_generation=_has_structured,
  245. )
  246. if action is not None:
  247. return _verdict(action, interrupt_result)
  248. if _truly_empty and _deterministic_empty:
  249. logger.warning(
  250. "Repeated empty response detected (model=%s provider=%s finish_reason=%s) — "
  251. "skipping remaining retries",
  252. agent.model, agent.provider, finish_reason,
  253. )
  254. agent._buffer_diagnostic_status(
  255. "⚠️ Model is repeatedly returning empty content — skipping further retries "
  256. "to avoid repeat charges"
  257. )
  258. # Exhausted retries — try the next provider in the chain before "(empty)".
  259. if _truly_empty and agent._fallback_chain:
  260. logger.warning(
  261. "Empty response after %d retries — attempting fallback (model=%s, provider=%s)",
  262. agent._empty_content_retries, agent.model, agent.provider,
  263. )
  264. agent._buffer_diagnostic_status("⚠️ Model returning empty responses — " "switching to fallback provider...")
  265. if agent._try_activate_fallback():
  266. active_system_prompt = _sync_failover_system_message(agent, api_messages, active_system_prompt)
  267. agent._empty_content_retries = 0
  268. agent._buffer_diagnostic_status(f"↻ Switched to fallback: {agent.model} " f"({agent.provider})")
  269. logger.info(
  270. "Fallback activated after empty responses: now using %s on %s",
  271. agent.model, agent.provider,
  272. )
  273. # OUTER loop: `continue` re-runs preflight against the fallback's window;
  274. # `break` would end the turn without calling the fallback.
  275. _preflight_compression_blocked = False
  276. # The fallback hop is a provider switch, not a model turn: refund the empty
  277. # call so a mid-turn fallback doesn't eat the iteration budget (#77305).
  278. api_call_count = _refund_api_call(agent, api_call_count)
  279. return _verdict("continue")
  280. _turn_exit_reason = "empty_response_exhausted"
  281. final_response = _terminal_empty(agent, assistant_message, finish_reason, messages)
  282. return _verdict("break")