turn_truncation.py 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776
  1. """Truncation recovery (``finish_reason == "length"``) for the conversation turn loop.
  2. Handles thinking-budget exhaustion, repetition-dominated truncation, content-filter stream
  3. stalls escalated to the fallback chain, text continuation nudges (up to 4, with the ceiling
  4. exit that drops the fragment trail), truncated tool-call retries with max_tokens boosts, and
  5. the final roll-back. Nothing here imports ``agent.conversation_loop`` at module level
  6. (cycle); loop-internal helpers are imported lazily so tests patching them keep working.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import re
  11. from dataclasses import dataclass
  12. from typing import Any, Dict, List, Optional, Tuple
  13. from agent.error_classifier import FailoverReason
  14. from agent.message_metadata import append_message
  15. from agent.message_sanitization import close_interrupted_tool_sequence
  16. from agent.repetition_guard import is_repetition_dominated
  17. from agent.turn_api_call import stop_thinking_spinner
  18. from agent.turn_failure_copy import content_policy_copy, provider_label_for, site_copy, stamp_failure
  19. from agent.turn_retry_state import TurnRetryState
  20. from agent.usage_pricing import normalize_usage
  21. from hermes_constants import PARTIAL_STREAM_STUB_ID
  22. logger = logging.getLogger("agent.conversation_loop")
  23. # codex_responses only reaches ``finish_reason == "length"`` for a tool call cut off by
  24. # max_output_tokens (turn_response_check.py::_derive_finish_reason); text truncation stays on
  25. # the Codex incomplete continuation, so the text branch below never double-continues it.
  26. _CONTINUABLE_MODES = {"chat_completions", "bedrock_converse", "anthropic_messages", "codex_responses"}
  27. _THINK_TAG_RE = re.compile(r'<(?:think|thinking|reasoning|REASONING_SCRATCHPAD)[^>]*>', re.IGNORECASE)
  28. _TRUNCATED_FINAL = site_copy("truncated")
  29. _FIRST_TRUNCATED_FINAL = _TRUNCATED_FINAL
  30. # #106260: a stream that died on a context-overflow error after partial delivery must not seed a
  31. # continuation — the transcript already cannot fit, and appending the partial stub grows every
  32. # later request into the same overflow. End the turn via the recovery contract instead.
  33. _CONTEXT_OVERFLOW_PARTIAL_FINAL = (
  34. "The request no longer fits the model's context window, so the partial "
  35. "response was not continued. Continue in a fresh session (/new; gateway "
  36. "chats are reset automatically)."
  37. )
  38. def collapse_continuation_trail(
  39. agent: Any, messages: list[dict[str, Any]], current_turn_user_idx: Any, *,
  40. finish_reason: str, parts: Optional[list[str]] = None,
  41. ) -> str:
  42. """Drop this turn's ``_length_continuation_fragment``/``_nudge`` rows and append one
  43. assistant row holding the joined, think-stripped partial (a held turn's rewrite); returns
  44. that text ("" none).
  45. ``parts=None`` (retry exhaustion, #119001): the text comes from the fragment rows and
  46. nothing happens without a valid turn index or a trail — an unanswered synthetic nudge
  47. must never be persisted, and an earlier turn's rows must never be read. Explicit
  48. ``parts`` (the continuation ceiling) always appends, scanning from 0 without an index.
  49. """
  50. idx = current_turn_user_idx
  51. valid_idx = isinstance(idx, int) and idx >= 0
  52. if parts is None and not (valid_idx and idx < len(messages)):
  53. return ""
  54. turn_start = idx + 1 if valid_idx else 0
  55. fragment_parts: list[str] = []
  56. retained: list[Any] = []
  57. found_trail = False
  58. for message in messages[turn_start:]:
  59. if isinstance(message, dict) and (
  60. message.get("_length_continuation_fragment") or message.get("_length_continuation_nudge")
  61. ):
  62. found_trail = True
  63. content = message.get("content")
  64. if message.get("_length_continuation_fragment") and isinstance(content, str) and content:
  65. fragment_parts.append(content)
  66. continue
  67. retained.append(message)
  68. if parts is None and not found_trail:
  69. return ""
  70. messages[turn_start:] = retained
  71. from agent.conversation_loop import _join_truncated_parts
  72. raw = fragment_parts if parts is None else parts
  73. join_parts = []
  74. for item in raw:
  75. if isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str):
  76. join_parts.append(item)
  77. elif isinstance(item, str) and item:
  78. join_parts.append((item, False))
  79. partial = agent._strip_think_blocks(
  80. _join_truncated_parts(join_parts)
  81. ).strip()
  82. if partial and getattr(agent, "_hold_stream", False) is True:
  83. # A held turn showed none of these fragments: the stitched text reaches the user and the
  84. # stored row only through the turn's transform snapshot (fail closed: the withheld notice).
  85. from agent.turn_finalizer import apply_llm_output_transform
  86. partial, _, _ = apply_llm_output_transform(agent, partial, turn_id=getattr(agent, "_current_turn_id", "") or "")
  87. if partial:
  88. append_message(messages, {"role": "assistant", "content": partial, "finish_reason": finish_reason})
  89. agent._session_messages = messages
  90. return partial
  91. _THINKING_EXHAUSTED = (
  92. "💭 Reasoning exhausted the output token budget — no visible response was produced.",
  93. "⚠️ **Thinking Budget Exhausted**\n\nThe model used all its output tokens on reasoning "
  94. "and had none left for the actual response.\n\nTo fix this:\n"
  95. "→ Lower reasoning effort: `/reasoning low` or `/reasoning minimal`\n"
  96. "→ Or switch to a larger/non-reasoning model with `/model`",
  97. "Model used all output tokens on reasoning with none left "
  98. "for the response. Try lowering reasoning effort or increasing max_tokens.",
  99. )
  100. def repetition_copy(stopping: str, outcome: str, refusal: str) -> tuple[str, str, str]:
  101. """(log line, user copy, error) for a repetition-dominated abort; only the clauses naming
  102. where the turn stopped differ between the length path and the stop path."""
  103. return (
  104. f"🔁 Response dominated by repeated text — stopping {stopping}.",
  105. "⚠️ **Response Stopped — Repetition Detected**\n\nThe model fell into a repetition loop while "
  106. f"writing this response, {outcome}\n\n→ Switch to a different model with `/model`\n"
  107. "→ Or resend your message (your conversation history is preserved)",
  108. f"Model output entered a repetition loop{refusal} degenerate response.",
  109. )
  110. _REPETITION_DOMINATED = repetition_copy(
  111. "instead of continuing a degenerate response",
  112. "so continuing would only produce more repeated text. The partial response was discarded.",
  113. " and was truncated mid-loop; refusing to continue a",
  114. )
  115. _REPETITION_STREAM_CUT = repetition_copy(
  116. "the stream mid-loop",
  117. "so the stream was stopped instead of running on. The partial response was discarded.",
  118. " and the stream was cut mid-loop; discarding the",
  119. )
  120. _CEILING_NO_TEXT = (
  121. "⚠️ **No visible answer was produced.** The model hit its output-token limit on every "
  122. "continuation attempt — its reasoning consumed the entire budget each time.\n\nTo fix this:\n"
  123. "→ Lower reasoning effort: `/reasoning low` or `/reasoning none`\n→ Or raise max_tokens for this model"
  124. )
  125. # Below this many free tokens the prompt itself filled the window: a continuation nudge +
  126. # fragment costs ~100 tokens per attempt, so retrying only shrinks the room (#106120).
  127. _MIN_CONTINUATION_HEADROOM = 512
  128. _WINDOW_FILLED = (
  129. "⚠️ **Context window full.** The prompt used {prompt:,} of this model's {ctx:,}-token "
  130. "context window, leaving no room to answer in. This is a context-window limit, not an "
  131. "output-length limit.\n\nTo fix this:\n→ Compress the conversation with `/compress` or start "
  132. "a new session\n→ Or raise the model's context window (e.g. Ollama `num_ctx`)"
  133. )
  134. def _prompt_filled_window(agent: Any, response: Any) -> Optional[tuple[int, int]]:
  135. """``(prompt_tokens, context_length)`` when this response's usage shows the prompt left
  136. less than ``_MIN_CONTINUATION_HEADROOM`` in the window compression resolves for the
  137. model; ``None`` (keep continuing) when either number is unknown."""
  138. ctx = int(getattr(getattr(agent, "context_compressor", None), "context_length", 0) or 0)
  139. usage = getattr(response, "usage", None)
  140. if not (ctx and usage):
  141. return None
  142. prompt = normalize_usage(usage, provider=agent.provider, api_mode=agent.api_mode).prompt_tokens
  143. return (prompt, ctx) if prompt and ctx - prompt < _MIN_CONTINUATION_HEADROOM else None
  144. def normalize_response_for_agent(agent: Any, response: Any) -> Any:
  145. """One OpenAI-style message from any transport; Anthropic strips the OAuth tool prefix."""
  146. if agent.api_mode == "anthropic_messages":
  147. return agent._get_transport().normalize_response(
  148. response, strip_tool_prefix=agent._is_anthropic_oauth
  149. )
  150. return agent._get_transport().normalize_response(response)
  151. def partial_result(
  152. messages: list[dict[str, Any]], api_call_count: int, final_response: str,
  153. error: Optional[str] = None, *, failed: bool = False, compression_exhausted: bool = False,
  154. ) -> dict[str, Any]:
  155. """Typed incomplete-turn result (``partial`` unless ``failed``); ``error`` defaults to
  156. ``final_response``. ``compression_exhausted`` carries the #98722 typed bit the gateway
  157. consumes to reset/move future input to a clean session (see run_turn.py)."""
  158. result = {
  159. "final_response": final_response,
  160. "messages": messages,
  161. "api_calls": api_call_count,
  162. "completed": False,
  163. ("failed" if failed else "partial"): True,
  164. "error": final_response if error is None else error,
  165. }
  166. if compression_exhausted:
  167. result["compression_exhausted"] = True
  168. return result
  169. @dataclass
  170. class TruncationVerdict:
  171. """Outcome of ``recover_from_truncation``.
  172. ``action``: ``"return"`` (end the turn with ``result``), ``"break"`` (a
  173. ``_retry.restart_with_*`` flag is set — restart the API call), ``"continue"``
  174. (re-issue the same call immediately) or ``"fallthrough"`` (unreachable in practice:
  175. every path exits, kept for the contract). The remaining fields are the loop locals
  176. the handler may have rebound."""
  177. action: str
  178. result: Optional[dict[str, Any]]
  179. messages: list[dict[str, Any]]
  180. length_continue_retries: int
  181. truncated_response_parts: list[tuple[str, bool]]
  182. truncated_tool_call_retries: int
  183. retry_count: int
  184. compression_attempts: int
  185. @dataclass(kw_only=True)
  186. class _Trunc(TruncationVerdict):
  187. """Working state for the truncation phases — the verdict itself, plus the read-only
  188. call context; phases mutate the loop-local fields and ``done()`` stamps the action."""
  189. agent: Any
  190. response: Any
  191. finish_reason: str
  192. conversation_history: Any
  193. api_call_count: int
  194. effective_task_id: Any
  195. current_turn_user_idx: Any
  196. action: str = "fallthrough"
  197. result: Optional[dict[str, Any]] = None
  198. window_filled: Optional[tuple[int, int]] = None # (prompt_tokens, context_length)
  199. def done(self, action: str, result: Optional[dict[str, Any]] = None) -> TruncationVerdict:
  200. self.action, self.result = action, result
  201. return self
  202. def end_turn(
  203. self, final_response: str, error: Optional[str] = None, *,
  204. result_messages: Optional[list[dict[str, Any]]] = None, cleanup: bool = True,
  205. failed: bool = False, compression_exhausted: bool = False,
  206. failure: tuple[str, bool] = ("truncated", True),
  207. ) -> TruncationVerdict:
  208. """Persist and end the turn as partial (or ``failed``).
  209. ``compression_exhausted`` forwards the #98722 typed bit so the gateway can
  210. move future input off a bloated session (run_turn.py consumes it). ``failure`` is
  211. the ``(failure_reason, retryable)`` verdict for the UI descriptor.
  212. """
  213. agent = self.agent
  214. if cleanup:
  215. agent._cleanup_task_resources(self.effective_task_id)
  216. agent._persist_session(self.messages, self.conversation_history)
  217. return self.done("return", stamp_failure(partial_result(
  218. self.messages if result_messages is None else result_messages, self.api_call_count,
  219. final_response, error, failed=failed, compression_exhausted=compression_exhausted,
  220. ), *failure))
  221. @property
  222. def is_stub(self) -> bool:
  223. return getattr(self.response, "id", "") == PARTIAL_STREAM_STUB_ID
  224. def _abort_reason(agent: Any, content: Any, has_tool_calls: bool) -> Optional[tuple]:
  225. """``(vprint, user response, error)`` when continuation must NOT be attempted:
  226. thinking exhausted the budget (reasoning blocks with no visible text after them —
  227. ``content=None`` from non-<think> models is normal truncation), or a repetition loop
  228. burned the budget on one fragment (reasoning stripped first)."""
  229. if has_tool_calls:
  230. return None
  231. if content and _THINK_TAG_RE.search(content) and not agent._has_content_after_think_block(content):
  232. return _THINKING_EXHAUSTED
  233. visible = agent._strip_think_blocks(content) if isinstance(content, str) else content
  234. if visible and is_repetition_dominated(visible):
  235. return _REPETITION_DOMINATED
  236. return None
  237. def _content_filter_fallback(st: _Trunc, _retry: TurnRetryState) -> Optional[TruncationVerdict]:
  238. """Content-filter stream stall → fallback. ``_content_filter_terminated`` is
  239. content-deterministic, so escalate before retrying the primary; without a fallback
  240. fall through to normal continuation (best-effort, may loop)."""
  241. agent = st.agent
  242. if not (
  243. getattr(st.response, "_content_filter_terminated", False)
  244. and agent._fallback_index < len(agent._fallback_chain)
  245. ):
  246. return None
  247. agent._vprint(
  248. f"{agent.log_prefix}🛡️ Content filter terminated stream — activating fallback provider...",
  249. force=True, diagnostic=True,
  250. )
  251. agent._emit_diagnostic_status("Content filter terminated stream; switching to fallback...")
  252. if agent._try_activate_fallback():
  253. # Roll partial content back to the last clean turn so the fallback gets a
  254. # coherent continuation point; unmark survivors (their text left the partial).
  255. if st.truncated_response_parts:
  256. st.messages = agent._get_messages_up_to_last_assistant(st.messages)
  257. for _frag in st.messages:
  258. if isinstance(_frag, dict):
  259. _frag.pop("_length_continuation_fragment", None)
  260. _frag.pop("_length_continuation_nudge", None)
  261. agent._session_messages = st.messages
  262. st.length_continue_retries = 0
  263. st.truncated_response_parts = []
  264. st.retry_count = 0
  265. st.compression_attempts = 0
  266. _retry.primary_recovery_attempted = False
  267. _retry.restart_with_rebuilt_messages = True
  268. return st.done("break")
  269. agent._vprint(
  270. f"{agent.log_prefix}⚠️ No fallback provider configured — retrying with same provider "
  271. f"(may re-hit filter)...",
  272. force=True, diagnostic=True,
  273. )
  274. return None
  275. def _continue_text(st: _Trunc, _retry: TurnRetryState, assistant_message: Any) -> TruncationVerdict:
  276. """Text truncation (no tool calls): append the fragment + a continuation nudge (up to
  277. 4), then the ceiling exit that drops the fragment trail and keeps the stitched partial.
  278. Never appends an interim assistant row with NO visible content — strict providers
  279. reject it with 400 — only the nudge."""
  280. from agent.conversation_loop import _get_continuation_prompt, _join_truncated_parts
  281. agent = st.agent
  282. messages = st.messages
  283. st.length_continue_retries += 1
  284. n = st.length_continue_retries
  285. _interim_content = getattr(assistant_message, "content", None)
  286. if not _interim_content and not st.is_stub:
  287. # Thinking-only truncation: continuing with thinking ON re-burns the budget.
  288. agent._ephemeral_reasoning_off = True
  289. if _interim_content:
  290. interim_msg = agent._build_assistant_message(assistant_message, st.finish_reason)
  291. interim_msg["_length_continuation_fragment"] = True # ceiling exit drops these
  292. append_message(messages, interim_msg)
  293. st.truncated_response_parts.append((_interim_content, st.is_stub))
  294. filled = st.window_filled
  295. if n < 4 and filled is None:
  296. _dropped_tools = getattr(st.response, "_dropped_tool_names", None)
  297. if st.is_stub and _dropped_tools:
  298. agent._vprint(
  299. f"{agent.log_prefix}↻ Stream interrupted mid "
  300. f"tool-call ({', '.join(_dropped_tools[:3])}) — requesting chunked retry ({n}/4)...", diagnostic=True,
  301. )
  302. elif st.is_stub:
  303. agent._vprint(f"{agent.log_prefix}↻ Stream interrupted — requesting continuation ({n}/4)...", diagnostic=True)
  304. else:
  305. agent._vprint(f"{agent.log_prefix}↻ Requesting continuation ({n}/4)...", diagnostic=True)
  306. append_message(messages, {
  307. "role": "user", "content": _get_continuation_prompt(st.is_stub, _dropped_tools),
  308. "_length_continuation_nudge": True,
  309. })
  310. agent._session_messages = messages
  311. _retry.restart_with_length_continuation = True
  312. return st.done("break")
  313. # Unanswered continue nudges made every later turn re-truncate: drop the trail.
  314. partial_response = collapse_continuation_trail(
  315. agent, messages, st.current_turn_user_idx, finish_reason="length",
  316. parts=st.truncated_response_parts,
  317. )
  318. # The one-shot reasoning-off override must not leak into the next turn.
  319. agent._ephemeral_reasoning_off = False
  320. agent._vprint(
  321. f"{agent.log_prefix}⚠️ Not continuing — each attempt would only grow the prompt."
  322. if filled is not None else
  323. f"{agent.log_prefix}⚠️ Response still truncated after {n} continuation attempts — "
  324. + ("keeping the partial response received so far." if partial_response
  325. else "no visible text was produced."),
  326. force=True, diagnostic=True,
  327. )
  328. if filled is not None:
  329. notice = _WINDOW_FILLED.format(prompt=filled[0], ctx=filled[1])
  330. return st.end_turn(
  331. f"{partial_response}\n\n{notice}" if partial_response else notice,
  332. f"Prompt used {filled[0]} of {filled[1]} context tokens; no room to answer",
  333. )
  334. return st.end_turn(
  335. partial_response or _CEILING_NO_TEXT,
  336. "Response remained truncated after 4 continuation attempts",
  337. )
  338. def _model_output_limit(agent: Any) -> Optional[int]:
  339. """The model's real max output tokens when Hermes knows it, else None."""
  340. if getattr(agent, "api_mode", None) != "anthropic_messages":
  341. return None
  342. # Local: only Anthropic-Messages turns need the adapter module.
  343. from agent.anthropic_adapter import _get_anthropic_max_output
  344. return _get_anthropic_max_output(getattr(agent, "model", None) or "")
  345. def boosted_output_cap(agent: Any, requested_cap: Optional[int], n: int, base: Optional[int] = None) -> int:
  346. """Output budget for truncation retry ``n`` (1-based): ``base·2ⁿ``, never below the
  347. failed request's cap, at most ``max(32768, 2×cap)``, and never above the model's
  348. known output limit. ``base`` defaults to max_tokens, else the cap actually sent.
  349. A ceiling equal to the requested cap would re-send the same budget (#72770); a
  350. ceiling past the model limit only buys a provider 400 (#79715).
  351. """
  352. if base is None:
  353. base = agent.max_tokens or requested_cap or 4096
  354. anchor = requested_cap or base
  355. limit = _model_output_limit(agent)
  356. if limit and anchor >= limit:
  357. return anchor # already at the model ceiling: doubling cannot help
  358. boost = min(max(base * (2 ** n), requested_cap or 0), max(32768, anchor * 2))
  359. return min(boost, limit) if limit else boost
  360. def _retry_truncated_tool_call(st: _Trunc, api_kwargs: Any) -> TruncationVerdict:
  361. """Truncated tool call: re-run the same call (up to 4×) with a boosted max_tokens —
  362. a real output-cap truncation needs it, harmless for a network stall — else refuse to
  363. execute incomplete arguments."""
  364. agent = st.agent
  365. if st.truncated_tool_call_retries < 4:
  366. st.truncated_tool_call_retries += 1
  367. n = st.truncated_tool_call_retries
  368. if st.is_stub:
  369. agent._buffer_vprint(f"⚠️ Stream interrupted mid tool-call — retrying ({n}/4)...")
  370. else:
  371. agent._buffer_vprint(f"⚠️ Truncated tool call detected — retrying API call ({n}/4)...")
  372. agent._ephemeral_max_output_tokens = boosted_output_cap(
  373. agent, agent._requested_output_cap_from_api_kwargs(api_kwargs), n
  374. )
  375. return st.done("continue") # don't append the broken response
  376. agent._flush_status_buffer()
  377. _failure = FailoverReason.timeout.value if st.is_stub else "truncated"
  378. if st.is_stub and getattr(st.response, "_clean_eof", False):
  379. # #102766: no transport error — the server (or a proxy) closed the stream cleanly
  380. # without a finish_reason, so "check your network" copy / a timeout stamp would mislead.
  381. agent._vprint(
  382. f"{agent.log_prefix}⚠️ Server kept closing the stream mid tool-call after 4 retries — the action was not executed.",
  383. force=True, diagnostic=True,
  384. )
  385. _final_response = site_copy("stream_closed_tool_call", label=provider_label_for(agent.provider))
  386. _failure = "truncated"
  387. elif st.is_stub:
  388. agent._vprint(
  389. f"{agent.log_prefix}⚠️ Stream kept dropping mid tool-call after 4 retries — the action was not executed.",
  390. force=True, diagnostic=True,
  391. )
  392. _final_response = site_copy("stream_dropped_tool_call", label=provider_label_for(agent.provider))
  393. else:
  394. agent._vprint(
  395. f"{agent.log_prefix}⚠️ Truncated tool call response detected again — refusing to execute incomplete tool arguments.",
  396. force=True, diagnostic=True,
  397. )
  398. _final_response = _TRUNCATED_FINAL
  399. agent._cleanup_task_resources(st.effective_task_id)
  400. # Prior tool batches can leave a tool-result tail; this path never reaches finalize_turn.
  401. close_interrupted_tool_sequence(st.messages, _final_response)
  402. return st.end_turn(
  403. _final_response, cleanup=False,
  404. failure=(_failure, True),
  405. )
  406. def recover_from_truncation(
  407. agent: Any, response: Any, finish_reason: str, _retry: TurnRetryState, *,
  408. messages: list[dict[str, Any]], conversation_history: Any, api_kwargs: Any, api_call_count: int,
  409. effective_task_id: Any, current_turn_user_idx: Any, length_continue_retries: int,
  410. truncated_response_parts: list[tuple[str, bool]], truncated_tool_call_retries: int, retry_count: int,
  411. compression_attempts: int,
  412. ) -> TruncationVerdict:
  413. """Recover from a truncated response. Order is load-bearing: thinking exhaustion and
  414. repetition abort BEFORE any continuation; a content-filter stall escalates to the
  415. fallback chain BEFORE the primary is retried; text continuation (no tool calls) then
  416. truncated tool-call retry; finally roll back to the last complete assistant turn."""
  417. st = _Trunc(
  418. agent=agent, response=response, finish_reason=finish_reason,
  419. conversation_history=conversation_history, api_call_count=api_call_count,
  420. effective_task_id=effective_task_id, current_turn_user_idx=current_turn_user_idx,
  421. messages=messages, length_continue_retries=length_continue_retries,
  422. truncated_response_parts=truncated_response_parts,
  423. truncated_tool_call_retries=truncated_tool_call_retries, retry_count=retry_count,
  424. compression_attempts=compression_attempts,
  425. )
  426. if getattr(response, "_runaway_repetition", False):
  427. # The streaming call cut a live repetition loop: a continuation would only re-enter it,
  428. # whatever the partial or its tool calls look like.
  429. line, user_response, error = _REPETITION_STREAM_CUT
  430. agent._vprint(f"{agent.log_prefix}{line}", force=True, diagnostic=True)
  431. return st.end_turn(user_response, error)
  432. st.window_filled = _prompt_filled_window(agent, response)
  433. if st.is_stub and getattr(response, "_clean_eof", False):
  434. _banner = ("Response truncated — server ended the stream without ever sending finish_reason "
  435. "(no transport error — the server or a proxy closed the stream cleanly)")
  436. elif st.is_stub:
  437. _banner = "Response truncated — stream ended before completion"
  438. elif st.window_filled:
  439. _banner = (f"Response truncated (finish_reason='length') - the prompt filled the context window "
  440. f"({st.window_filled[0]:,}/{st.window_filled[1]:,} tokens)")
  441. else:
  442. _banner = "Response truncated (finish_reason='length') - model hit max output tokens"
  443. agent._vprint(f"{agent.log_prefix}⚠️ {_banner}", force=True, diagnostic=True)
  444. # #106260: a context-overflow error after partial delivery must not seed a
  445. # continuation. _partial_stream_stub marks such stubs _overflow_terminal and
  446. # leaves content empty; continuing would only re-send a larger request into
  447. # the same overflow. The stub path never raises, so this class never reached
  448. # recover_from_overflow's compress-and-retry on main either — ending the turn
  449. # replaces a growth loop, not a compression attempt.
  450. if getattr(st.response, "_overflow_terminal", False):
  451. agent._flush_status_buffer()
  452. agent._vprint(
  453. f"{agent.log_prefix}⚠️ Stream ended on a context-overflow error after "
  454. "partial delivery — not continuing (the request no longer fits the model's "
  455. "context window).",
  456. force=True, diagnostic=True,
  457. )
  458. # Prior tool batches can leave a tool-result tail; this path never reaches
  459. # finalize_turn (same as the truncated-tool-call terminal above).
  460. close_interrupted_tool_sequence(st.messages, _CONTEXT_OVERFLOW_PARTIAL_FINAL)
  461. # Carry the #98722 typed exhaustion bit so the gateway resets/moves future
  462. # input to a clean session instead of leaving this bloated one authoritative
  463. # for the next turn.
  464. return st.end_turn(
  465. _CONTEXT_OVERFLOW_PARTIAL_FINAL,
  466. error=_CONTEXT_OVERFLOW_PARTIAL_FINAL,
  467. failed=True,
  468. compression_exhausted=True,
  469. failure=("context_overflow", False),
  470. )
  471. _trunc_msg = normalize_response_for_agent(agent, response)
  472. _trunc_content = getattr(_trunc_msg, "content", None) if _trunc_msg else None
  473. _trunc_has_tool_calls = bool(getattr(_trunc_msg, "tool_calls", None)) if _trunc_msg else False
  474. abort = _abort_reason(agent, _trunc_content, _trunc_has_tool_calls)
  475. if abort is not None:
  476. line, user_response, error = abort
  477. agent._vprint(f"{agent.log_prefix}{line}", force=True, diagnostic=True)
  478. return st.end_turn(user_response, error)
  479. if agent.api_mode in _CONTINUABLE_MODES:
  480. cf = _content_filter_fallback(st, _retry)
  481. if cf is not None:
  482. return cf
  483. if _trunc_msg is not None:
  484. if not _trunc_has_tool_calls:
  485. return _continue_text(st, _retry, _trunc_msg)
  486. return _retry_truncated_tool_call(st, api_kwargs)
  487. if len(messages) > 1:
  488. agent._vprint(f"{agent.log_prefix} ⏪ Rolling back to last complete assistant turn", diagnostic=True)
  489. return st.end_turn(
  490. _TRUNCATED_FINAL, result_messages=agent._get_messages_up_to_last_assistant(messages)
  491. )
  492. # First message was truncated - mark as failed
  493. agent._flush_status_buffer()
  494. agent._vprint(f"{agent.log_prefix}❌ First response truncated - cannot recover", force=True, diagnostic=True)
  495. return st.end_turn(_FIRST_TRUNCATED_FINAL, cleanup=False, failed=True)
  496. _CODEX_REPLAY_KEYS = (
  497. "content", "reasoning", "reasoning_content", "reasoning_details",
  498. "codex_reasoning_items", "codex_message_items",
  499. )
  500. # Third return value of ``continue_codex_incomplete``: the reasoning-only stall was handed to a
  501. # fallback provider — the caller re-syncs the system prompt identity and continues the turn.
  502. CODEX_FALLBACK_ACTIVATED = "codex_fallback_activated"
  503. def continue_codex_incomplete(
  504. agent: Any, assistant_message: Any, finish_reason: str, *, messages: list[dict[str, Any]],
  505. conversation_history: Any, api_call_count: int, response: Any = None,
  506. ) -> Optional[Any]:
  507. """Codex Responses ``status=incomplete`` continuation (max 3 per turn).
  508. Appends the interim assistant message (deduped on visible content only — opaque
  509. provider state drifts per continuation; ``codex_reasoning_items`` are merged, not
  510. overwritten, because the earlier response holds the only native-compaction
  511. checkpoint) and, when a bare retry would be byte-identical, a user-role nudge — only
  512. after an assistant row, to preserve role alternation. Returns ``None`` to continue
  513. the turn loop, ``CODEX_FALLBACK_ACTIVATED`` when a reasoning-only stall was handed to
  514. the next fallback provider, or the terminal ``partial`` result once retries are exhausted.
  515. Reasoning-only stall ladder (#67321): a response with neither visible text nor a tool
  516. call advances ``_codex_reasoning_only_streak`` (a visible partial resets it; the aggregate
  517. ``_codex_incomplete_retries`` stays the cap for partials). Encrypted reasoning replays
  518. byte-for-byte, so after replay (1) and nudge (2) the third consecutive reasoning-only
  519. response goes to the configured fallback with the semantic ``incomplete_response`` reason
  520. instead of ending on the sentinel; when that response consumed the last iteration the
  521. fallback gets exactly one grace call (``_budget_grace_call`` is consumed by the next
  522. iteration, and the streak restarts from 0, so a second grace call is unreachable).
  523. When ``response`` hit ``max_output_tokens`` with no visible text (reasoning ate the
  524. whole budget), the next attempt goes out with reasoning off and a doubled output
  525. cap — the same one-shot overrides the chat-completions length path uses — because
  526. re-sending the identical budget and effort re-burns the budget identically (#90393)."""
  527. from agent.conversation_loop import _CODEX_INCOMPLETE_NUDGE
  528. from agent.turn_response_check import _codex_finish_reason
  529. agent._codex_incomplete_retries += 1
  530. n = agent._codex_incomplete_retries
  531. interim_msg = agent._build_assistant_message(assistant_message, finish_reason)
  532. interim_has_content = bool((interim_msg.get("content") or "").strip())
  533. _reasoning = interim_msg.get("reasoning")
  534. interim_has_reasoning = isinstance(_reasoning, str) and bool(_reasoning.strip())
  535. interim_has_codex_reasoning = bool(interim_msg.get("codex_reasoning_items"))
  536. interim_has_codex_message_items = bool(interim_msg.get("codex_message_items"))
  537. reasoning_only = not interim_has_content and not getattr(assistant_message, "tool_calls", None)
  538. agent._codex_reasoning_only_streak = agent._codex_reasoning_only_streak + 1 if reasoning_only else 0
  539. streak = agent._codex_reasoning_only_streak
  540. if interim_has_content or interim_has_reasoning or interim_has_codex_reasoning or interim_has_codex_message_items:
  541. last_msg = messages[-1] if messages else None
  542. last_is_dict = isinstance(last_msg, dict)
  543. last_interim_visible = agent._interim_assistant_visible_text(last_msg) if last_is_dict else ""
  544. current_interim_visible = agent._interim_assistant_visible_text(interim_msg)
  545. if last_interim_visible or current_interim_visible:
  546. same_visible_output = last_interim_visible == current_interim_visible
  547. else:
  548. # Neither has text eligible for interim delivery: compare raw content+reasoning.
  549. same_visible_output = last_is_dict and (
  550. (last_msg.get("content") or "") == (interim_msg.get("content") or "")
  551. and (last_msg.get("reasoning") or "") == (interim_msg.get("reasoning") or "")
  552. )
  553. if (
  554. last_is_dict
  555. and last_msg.get("role") == "assistant"
  556. and last_msg.get("finish_reason") == "incomplete"
  557. and same_visible_output
  558. ):
  559. # Duplicate: refresh replay state in place, no re-emitted commentary.
  560. for _key in _CODEX_REPLAY_KEYS:
  561. if _key not in interim_msg:
  562. continue
  563. if _key == "codex_reasoning_items":
  564. from agent.native_compaction import merge_interim_reasoning_items
  565. last_msg[_key] = merge_interim_reasoning_items(last_msg.get(_key), interim_msg[_key])
  566. else:
  567. last_msg[_key] = interim_msg[_key]
  568. else:
  569. append_message(messages, interim_msg)
  570. agent._emit_interim_assistant_message(interim_msg)
  571. if reasoning_only and streak >= 3:
  572. if agent._try_activate_fallback(reason=FailoverReason.incomplete_response):
  573. # The trigger may have consumed the turn budget; without a grace call the loop
  574. # exits before the fallback is ever asked.
  575. if api_call_count >= agent.max_iterations or agent.iteration_budget.remaining <= 0:
  576. agent._budget_grace_call = True
  577. agent._codex_incomplete_retries = 0
  578. agent._codex_reasoning_only_streak = 0
  579. if not agent.quiet_mode:
  580. agent._vprint(
  581. f"{agent.log_prefix}↻ Codex reasoning-only stall after {streak} attempts — "
  582. f"switching to fallback {agent.model} ({agent.provider})", diagnostic=True,
  583. )
  584. agent._emit_diagnostic_wait("↻ model stuck on internal reasoning — switching to fallback provider")
  585. agent._session_messages = messages
  586. return CODEX_FALLBACK_ACTIVATED
  587. # No fallback left: fall through to the terminal sentinel.
  588. elif n < 3 or reasoning_only:
  589. # A reasoning-only streak below 3 continues even once partials used up the aggregate
  590. # cap, so the mixed partial-then-stall variant reaches the ladder above.
  591. # If the interim has nothing the Responses converter will replay, a bare retry is
  592. # byte-identical; a replayable interim holding only a ``compaction`` checkpoint
  593. # ALSO re-sends identically. One bare retry, then always nudge.
  594. interim_replayable = interim_has_content or interim_has_codex_reasoning or interim_has_codex_message_items
  595. if not interim_replayable or n >= 2:
  596. _last_msg = messages[-1] if messages else None
  597. if isinstance(_last_msg, dict):
  598. _already_nudged = (
  599. _last_msg.get("role") == "user" and _last_msg.get("content") == _CODEX_INCOMPLETE_NUDGE
  600. )
  601. # Alternation guard: the nudge may only follow an assistant row.
  602. if not _already_nudged and _last_msg.get("role") == "assistant":
  603. append_message(messages, {"role": "user", "content": _CODEX_INCOMPLETE_NUDGE})
  604. if not interim_has_content and _codex_finish_reason(response) == "incomplete":
  605. agent._ephemeral_reasoning_off = True
  606. # No configured cap means the provider's own ceiling was hit: the observed
  607. # output_tokens IS that ceiling, so seed the escalation from it (else 4096).
  608. usage = getattr(response, "usage", None)
  609. observed = getattr(usage, "output_tokens", None) if not isinstance(usage, dict) else usage.get("output_tokens")
  610. agent._ephemeral_max_output_tokens = boosted_output_cap(
  611. agent, None, n, base=agent.max_tokens or int(observed or 0) or 4096
  612. )
  613. if not agent.quiet_mode:
  614. agent._vprint(f"{agent.log_prefix}↻ Codex response incomplete; continuing turn ({n}/3)", diagnostic=True)
  615. # Spinner/heartbeat notice: these retries can take minutes and otherwise look
  616. # like infinite thinking.
  617. # #70773: same FD-recycle corruption vector as #67142. The shared OpenAI client's connection pool
  618. # must NOT be closed from this watchdog/poll thread — worker threads from previous stale-killed
  619. # attempts may still be unwinding their SSL BIOs. The request-local client is already closed above
  620. # via _close_request_client_once. The shared client will be replaced lazily by
  621. # _ensure_primary_openai_client on the next request.
  622. # Surface the continuation on the live spinner/status line (CLI/TUI/Desktop) and gateway heartbeat:
  623. # each of these retries can spend minutes waiting on the provider, and without a distinct notice the
  624. # user only sees a generic thinking spinner ("infinite thinking", #64434).
  625. agent._emit_diagnostic_wait(
  626. f"↻ model returned reasoning with no final answer — asking it to continue ({n}/3)"
  627. )
  628. agent._session_messages = messages
  629. return None
  630. agent._codex_incomplete_retries = 0
  631. agent._codex_reasoning_only_streak = 0
  632. agent._persist_session(messages, conversation_history)
  633. return partial_result(
  634. messages, api_call_count, "Codex response remained incomplete after 3 continuation attempts"
  635. )
  636. @dataclass
  637. class RefusalVerdict:
  638. """Outcome of ``handle_content_policy_refusal``: ``"break"`` (fallback activated —
  639. restart armed on ``_retry``; caller resets retry/compression counters) or
  640. ``"return"`` (the typed content-policy result in ``result``). ``active_system_prompt``
  641. is the possibly re-synced system prompt."""
  642. action: str
  643. result: Optional[dict[str, Any]]
  644. active_system_prompt: Any
  645. def handle_content_policy_refusal(
  646. agent: Any, response: Any, _retry: TurnRetryState, *, thinking_spinner: Any,
  647. messages: list[dict[str, Any]], api_messages: Any, api_kwargs: Any, active_system_prompt: Any,
  648. conversation_history: Any, api_call_count: int, effective_task_id: Any, turn_id: Any,
  649. api_request_id: Any, api_start_time: float, retry_count: int, max_retries: int,
  650. ) -> RefusalVerdict:
  651. """HTTP-200 refusal (``finish_reason`` ``content_filter`` / ``guardrail_intervened``).
  652. Deterministic for the unchanged prompt — never retried: one configured-fallback try,
  653. else surface the refusal (explanation may live only in the reasoning channel)."""
  654. from agent.conversation_loop import _arm_fallback_restart, _content_policy_blocked_result
  655. _refusal_result = normalize_response_for_agent(agent, response)
  656. _refusal_text = (getattr(_refusal_result, "content", None) or "").strip()
  657. if not _refusal_text:
  658. _refusal_text = (agent._extract_reasoning(_refusal_result) or "").strip()
  659. # Anthropic stop_reason=refusal carries its reason on stop_details (category + optional explanation),
  660. # not in a content block — without it a classifier halt reads as "(no text)" (#113689).
  661. _stop_details = (getattr(_refusal_result, "provider_data", None) or {}).get("stop_details")
  662. if not _refusal_text and isinstance(_stop_details, dict):
  663. _refusal_text = str(_stop_details.get("explanation") or "").strip() or (
  664. f"provider refusal category: {_stop_details['category']}" if _stop_details.get("category") else ""
  665. )
  666. agent._invoke_api_request_error_hook(
  667. task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
  668. api_call_count=api_call_count, api_start_time=api_start_time, api_kwargs=api_kwargs,
  669. error_type="ContentPolicyBlocked",
  670. error_message=_refusal_text or "model declined to respond (content_filter)",
  671. status_code=None, retry_count=retry_count, max_retries=max_retries, retryable=False,
  672. reason=FailoverReason.content_policy_blocked.value,
  673. )
  674. stop_thinking_spinner(agent, thinking_spinner)
  675. if agent._has_pending_fallback():
  676. agent._buffer_diagnostic_status("⚠️ Model declined to respond (safety refusal) — trying fallback...")
  677. if agent._try_activate_fallback():
  678. active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
  679. return RefusalVerdict("break", None, active_system_prompt)
  680. agent._flush_status_buffer()
  681. _refusal_log = _refusal_text[:500] + "..." if len(_refusal_text) > 500 else _refusal_text
  682. # native_stop_reason tells an Anthropic classifier refusal (``refusal``) from a Bedrock guardrail
  683. # block (``end_turn``); both arrive here as content_filter.
  684. logger.warning(
  685. "%sModel declined to respond (finish_reason=content_filter). model=%s provider=%s "
  686. "native_stop_reason=%s stop_details=%s refusal=%s",
  687. agent.log_prefix, agent.model, agent.provider,
  688. getattr(response, "stop_reason", None) or "n/a", _stop_details or "n/a",
  689. _refusal_log or "(no text)",
  690. )
  691. agent._emit_diagnostic_status("⚠️ The model declined to respond to this request (safety refusal).")
  692. _refusal_response = "⚠️ " + content_policy_copy(
  693. label=provider_label_for(agent.provider),
  694. summary=_refusal_text or "the model returned no explanation",
  695. )
  696. agent._cleanup_task_resources(effective_task_id)
  697. agent._persist_session(messages, conversation_history)
  698. return RefusalVerdict("return", _content_policy_blocked_result(
  699. messages, api_call_count, final_response=_refusal_response,
  700. error_detail=_refusal_text or "model declined (content_filter)",
  701. ), active_system_prompt)