turn_loop_errors.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171
  1. """Outer-loop exception handler for the conversation turn loop: phase-aware classification
  2. (interpreter shutdown, deterministic local post-processing bug vs API-path failure),
  3. unanswered tool_call error results, and the per-turn error cap that stops the loop from
  4. spinning until the budget is gone. Nothing here imports ``agent.conversation_loop`` at
  5. module level (cycle); loop-internal constants resolve lazily.
  6. """
  7. from __future__ import annotations
  8. from dataclasses import dataclass
  9. import logging
  10. import os
  11. import sys
  12. from typing import Any
  13. from agent.message_metadata import append_message
  14. from agent.turn_failure_copy import short_detail, site_copy
  15. logger = logging.getLogger("agent.conversation_loop")
  16. @dataclass
  17. class OuterErrorVerdict:
  18. """``action`` is ``"break"`` (turn ends with ``final_response``/``turn_exit_reason`` set)
  19. or ``"fallthrough"`` (retry the iteration). The other fields are the loop locals the
  20. handler rebinds."""
  21. action: str
  22. _outer_error_count: Any
  23. _turn_exit_reason: Any
  24. failed: Any
  25. final_response: Any
  26. def handle_outer_loop_error(
  27. agent: Any, *, e: Any, _outer_error_count: Any, api_call_count: Any, messages: Any,
  28. conversation_history: Any, _turn_exit_reason: Any, failed: Any, final_response: Any,
  29. ) -> OuterErrorVerdict:
  30. """Handle an exception that escaped the response-processing block. Shutdown and
  31. local-processing errors are deterministic and end the turn; API-path errors retry until
  32. ``min(_MAX_OUTER_LOOP_ERRORS, max_iterations)`` escaped exceptions. The assistant
  33. message is never appended here: a prefill/interim assistant may already be the tail
  34. (assistant→assistant); ``finalize_turn`` appends only when safe."""
  35. from agent.conversation_loop import (
  36. _API_CALL_MODULES, _LOCAL_PROCESSING_MODULES, _MAX_OUTER_LOOP_ERRORS,
  37. _is_interpreter_shutdown_error, _ra,
  38. )
  39. def _verdict(action: str) -> OuterErrorVerdict:
  40. return OuterErrorVerdict(
  41. action=action, _outer_error_count=_outer_error_count,
  42. _turn_exit_reason=_turn_exit_reason, failed=failed, final_response=final_response,
  43. )
  44. # Count every escaped exception before classification so permanent failures
  45. # terminate even with an unlimited turn budget.
  46. _outer_error_count += 1
  47. # Interpreter shutdown makes every executor op raise: break.
  48. # Phase-aware error classification. The huge outer try/except spans both the actual API request and all
  49. # local post-processing of the returned assistant message. Deterministic local bugs (e.g. passing a
  50. # multimodal content list into a regex helper after a vision turn or context compaction) should not be
  51. # retried: they will fail identically on every iteration and only burn the iteration budget. We classify
  52. # an error as local by inspecting the traceback: if the exception propagated through any of the known
  53. # local post-processing helpers and never entered the interruptible API-call helpers, it is almost
  54. # certainly a local processing bug. (#66267) Interpreter shutdown: if the process is tearing down, every
  55. # executor-backed operation (API call, tool dispatch, memory sync) raises ``RuntimeError: cannot
  56. # schedule new futures after interpreter shutdown``. Retrying is pointless — the executor is gone for
  57. # good — and each retry just spams another traceback. Break immediately so the turn exits cleanly.
  58. # (#93217)
  59. if sys.is_finalizing() or _is_interpreter_shutdown_error(e):
  60. error_msg = f"Interpreter is shutting down — cannot continue (API call #{api_call_count}): {e}"
  61. try:
  62. agent._safe_print(f"❌ {error_msg}", diagnostic=True)
  63. except (OSError, ValueError):
  64. pass
  65. logger.warning(error_msg)
  66. # Best-effort persist — the dying executor may raise the same error; don't let
  67. # it mask the shutdown exit. finalize_turn retries.
  68. try:
  69. agent._persist_session(messages, conversation_history)
  70. except Exception:
  71. pass
  72. _turn_exit_reason = "interpreter_shutdown"
  73. failed = True
  74. _sid = getattr(agent, "session_id", None)
  75. final_response = site_copy(
  76. "interpreter_shutdown", resume=f" (CLI: `hermes --resume {_sid}`)" if _sid else "",
  77. )
  78. return _verdict("break")
  79. # Deterministic local post-processing bugs (traceback via local helpers, never API
  80. # helpers) aren't retried.
  81. tb_module_names: set[str] = set()
  82. _tb = e.__traceback__
  83. while _tb is not None:
  84. tb_module_names.add(os.path.splitext(os.path.basename(_tb.tb_frame.f_code.co_filename))[0])
  85. _tb = _tb.tb_next
  86. _is_local_processing_error = bool(tb_module_names & _LOCAL_PROCESSING_MODULES) and not (
  87. tb_module_names & _API_CALL_MODULES
  88. )
  89. if _is_local_processing_error:
  90. error_msg = f"Error during local message processing after API call #{api_call_count}: {e!s}"
  91. else:
  92. error_msg = f"Error during API call #{api_call_count}: {e!s}"
  93. # Honor the _vprint contract: suppress_status_output silences hard failures;
  94. # quiet_mode -q still shows them. Traceback is logged below.
  95. if getattr(agent, "suppress_status_output", False):
  96. logger.error(error_msg)
  97. else:
  98. try:
  99. agent._safe_print(f"❌ {error_msg}", diagnostic=True)
  100. except (OSError, ValueError):
  101. logger.error(error_msg)
  102. # ERROR level with traceback so outer-loop failures land in agent.log AND errors.log.
  103. logger.exception("Outer loop error in API call #%d", api_call_count)
  104. # An appended assistant tool_calls message needs a role="tool" result per
  105. # tool_call_id; fill in error results for unanswered ones.
  106. for idx in range(len(messages) - 1, -1, -1):
  107. msg = messages[idx]
  108. if not isinstance(msg, dict):
  109. break
  110. if msg.get("role") == "tool":
  111. continue
  112. if msg.get("role") == "assistant" and msg.get("tool_calls"):
  113. answered_ids = {
  114. m["tool_call_id"]
  115. for m in messages[idx + 1:]
  116. if isinstance(m, dict) and m.get("role") == "tool"
  117. }
  118. for tc in msg["tool_calls"]:
  119. if tc and isinstance(tc, dict) and tc["id"] not in answered_ids:
  120. append_message(messages, {
  121. "role": "tool",
  122. "name": _ra().AIAgent._get_tool_call_name_static(tc),
  123. "tool_call_id": tc["id"],
  124. "content": f"Error executing tool: {error_msg}",
  125. })
  126. break
  127. # Non-tool errors are already printed; a synthetic message would pollute history
  128. # and risk breaking role alternation.
  129. # Local errors are deterministic: stop early instead of retrying until the budget is
  130. # gone; a small per-turn cap prevents infinite spinning.
  131. _outer_error_cap = min(_MAX_OUTER_LOOP_ERRORS, max(1, agent.max_iterations))
  132. if (
  133. _is_local_processing_error
  134. or api_call_count >= agent.max_iterations - 1
  135. or _outer_error_count >= _outer_error_cap
  136. ):
  137. # finalize_turn stamps failure_reason=loop_error from the exit reason so the desktop
  138. # card stops reading these as "unknown"/retryable. The deterministic local bug keeps
  139. # ``failed`` as it was (an incomplete, not failed, turn): flipping it made every such
  140. # child run a strike against the kanban dispatcher breaker.
  141. detail = short_detail(e)
  142. if _is_local_processing_error:
  143. _turn_exit_reason = f"local_processing_error({error_msg[:80]})"
  144. final_response = site_copy("local_processing_error", detail=detail)
  145. elif _outer_error_count >= _outer_error_cap:
  146. failed = True
  147. _turn_exit_reason = f"repeated_outer_errors({error_msg[:80]})"
  148. final_response = site_copy("loop_error", detail=detail)
  149. else:
  150. _turn_exit_reason = f"error_near_max_iterations({error_msg[:80]})"
  151. final_response = site_copy("loop_error", detail=detail)
  152. return _verdict("break")
  153. return _verdict("fallthrough")