turn_tool_round.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. """One tool-calling round of the conversation turn loop: validate/cap/dedupe the model's
  2. tool calls, persist the tool-call turn BEFORE any side effect, execute the tools, honour
  3. guardrail halts / persistence failures, then compress after tool results. Nothing here
  4. imports ``agent.conversation_loop`` at module level (cycle) — loop-internal helpers resolve
  5. lazily.
  6. """
  7. from __future__ import annotations
  8. from contextlib import suppress
  9. from dataclasses import dataclass
  10. import logging
  11. from typing import Any, Dict, Optional, Tuple
  12. from agent.message_metadata import append_message
  13. from agent.message_sanitization import coalesce_tool_call_id, normalize_provider_tool_call_ids
  14. from agent.turn_preflight import compress_after_tool_results
  15. from agent.turn_tool_validation import validate_tool_calls
  16. logger = logging.getLogger("agent.conversation_loop")
  17. # Post-response housekeeping tools: a round made only of these mutes tool progress.
  18. _HOUSEKEEPING_TOOLS = frozenset({"memory", "todo_list", "skill_manage", "session_search"})
  19. # Rounds whose visible line is the turn's reply if the follow-up is empty (no empty-response nudge):
  20. # housekeeping, and setup_beat, whose card already blocked for the user's answer.
  21. _LINE_IS_REPLY_TOOLS = _HOUSEKEEPING_TOOLS | {"setup_beat"}
  22. @dataclass
  23. class ToolRoundVerdict:
  24. """``action``: ``"continue"`` (tools ran, next API call), ``"break"`` (turn ends:
  25. persistence failure, guardrail halt, post-tool compression end) or ``"return"``
  26. (``result`` is the turn's result dict). The other fields are the loop locals the round
  27. rebinds."""
  28. action: str
  29. messages: Any
  30. conversation_history: Any
  31. active_system_prompt: Any
  32. compression_attempts: Any
  33. final_response: Any
  34. failed: Any
  35. _turn_exit_reason: Any
  36. truncated_tool_call_retries: Any
  37. current_turn_user_idx: Any
  38. result: Optional[dict[str, Any]] = None
  39. def run_tool_round(
  40. agent: Any, *, assistant_message: Any, finish_reason: Any, messages: Any,
  41. conversation_history: Any, api_call_count: Any, effective_task_id: Any, user_message: Any,
  42. system_message: Any, active_system_prompt: Any, compression_attempts: Any,
  43. max_compression_attempts: Any, final_response: Any, failed: Any, _turn_exit_reason: Any,
  44. truncated_tool_call_retries: Any, current_turn_user_idx: Any,
  45. ) -> ToolRoundVerdict:
  46. """Execute one tool round in the exact original order. Persist-before-execute is a
  47. durability invariant: resume must see the executed block if a destructive tool restarts
  48. Hermes; a failed canonical append ends the turn rather than running tools from
  49. process-only state."""
  50. from agent.conversation_loop import _invalid_tool_name_error_content
  51. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ToolRoundVerdict:
  52. return ToolRoundVerdict(
  53. action=action, messages=messages, conversation_history=conversation_history,
  54. active_system_prompt=active_system_prompt, compression_attempts=compression_attempts,
  55. final_response=final_response, failed=failed, _turn_exit_reason=_turn_exit_reason,
  56. truncated_tool_call_retries=truncated_tool_call_retries,
  57. current_turn_user_idx=current_turn_user_idx, result=result,
  58. )
  59. if not agent.quiet_mode:
  60. agent._vprint(f"{agent.log_prefix}🔧 Processing {len(assistant_message.tool_calls)} tool call(s)...")
  61. if agent.verbose_logging:
  62. for tc in assistant_message.tool_calls:
  63. raw_args = tc.function.arguments
  64. args_preview = raw_args[:200] if isinstance(raw_args, str) else repr(raw_args)[:200]
  65. logging.debug("Tool call: %s with args: %s...", tc.function.name, args_preview)
  66. _tvv = validate_tool_calls(
  67. agent, assistant_message, finish_reason, messages=messages,
  68. conversation_history=conversation_history, api_call_count=api_call_count,
  69. effective_task_id=effective_task_id,
  70. )
  71. if _tvv.action == "return":
  72. return _verdict("return", _tvv.result)
  73. if _tvv.action == "continue":
  74. return _verdict("continue")
  75. # Normalize only this unpersisted turn. Each local bridge entry must pass
  76. # through the same scope, approvals, state and scheduling as a singleton.
  77. from agent.tool_call_batches import expand_local_tool_batches
  78. assistant_message.tool_calls = expand_local_tool_batches(
  79. assistant_message.tool_calls, provider_data=getattr(assistant_message, "provider_data", None))
  80. # Post-call guardrails.
  81. assistant_message.tool_calls = agent._deduplicate_tool_calls(
  82. agent._cap_delegate_task_calls(assistant_message.tool_calls)
  83. )
  84. # Filtering can turn a mixed batch into an all-provider one; re-check the final batch
  85. # before it is staged (idempotent for already-normalized ids).
  86. normalize_provider_tool_call_ids(assistant_message.tool_calls)
  87. # Mixed batch: the assistant message keeps EVERY emitted call (each tool_call needs a
  88. # matching result) while only valid ones dispatch.
  89. _invalid_batch_calls = [
  90. tc for tc in assistant_message.tool_calls if tc.function.name not in agent.valid_tool_names
  91. ] if _tvv.mixed_invalid_batch else []
  92. assistant_msg, duplicate_previous_interim = stage_tool_call_message(
  93. agent, assistant_message=assistant_message, finish_reason=finish_reason, messages=messages
  94. )
  95. append_message(messages, assistant_msg)
  96. # Mixed batch: error-result invalid calls and drop them from execution.
  97. if _invalid_batch_calls:
  98. for tc in _invalid_batch_calls:
  99. append_message(messages, {
  100. "role": "tool",
  101. "name": tc.function.name,
  102. "tool_call_id": coalesce_tool_call_id(tc),
  103. "content": _invalid_tool_name_error_content(
  104. tc.function.name, agent.valid_tool_names
  105. ),
  106. })
  107. assistant_message.tool_calls = [
  108. tc for tc in assistant_message.tool_calls if tc.function.name in agent.valid_tool_names
  109. ]
  110. # Persist the tool-call turn before any tool side effects so resume sees the executed
  111. # block if a destructive tool restarts Hermes.
  112. try:
  113. _tool_turn_persisted = agent._flush_messages_to_session_db(messages, conversation_history)
  114. except Exception as exc:
  115. _tool_turn_persisted = False
  116. from hermes_state import classify_persistence_error
  117. agent._last_persistence_error_cause = classify_persistence_error(exc)
  118. logger.warning(
  119. "Incremental tool-call persistence failed before execution "
  120. "(session=%s): %s",
  121. agent.session_id or "none",
  122. exc,
  123. )
  124. if _tool_turn_persisted is False:
  125. # Canonical append failed: never project the row or run tools from process-only
  126. # state; break rather than retry. No recorded cause means genuinely unknown.
  127. if getattr(agent, "_last_persistence_error_cause", None) is None:
  128. agent._last_persistence_error_cause = "unknown"
  129. _turn_exit_reason = "session_persistence_failed"
  130. final_response = ""
  131. failed = True
  132. return _verdict("break")
  133. # A UI must never observe an assistant/tool-call row that is only an in-memory
  134. # projection: emit interim commentary after the DB append.
  135. if not duplicate_previous_interim:
  136. agent._emit_interim_assistant_message(assistant_msg)
  137. # Flush open streaming boxes before tools so early content doesn't wrap tool feed
  138. # lines. Display callback only — TTS (_stream_callback) must NOT receive None (EOS).
  139. if agent.stream_delta_callback:
  140. with suppress(Exception):
  141. agent.stream_delta_callback(None)
  142. # Opt-in (compression.prepare_ahead): when this response put the session near the trigger,
  143. # write the next compaction summary while the tools run. No-op otherwise; never raises.
  144. from agent.prepared_compaction import maybe_prepare
  145. maybe_prepare(agent, messages, origin="tool-batch")
  146. agent._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count)
  147. from hermes_cli.observability.shared_metrics_harness import finish_tool_round
  148. finish_tool_round(agent)
  149. if getattr(agent, "_incremental_persistence_failed", False):
  150. # Tool result could not be made canonical: never send the in-memory result to
  151. # the model or project later events from this turn.
  152. _turn_exit_reason = "session_persistence_failed"
  153. final_response = ""
  154. failed = True
  155. return _verdict("break")
  156. if agent._tool_guardrail_halt_decision is not None:
  157. decision = agent._tool_guardrail_halt_decision
  158. _turn_exit_reason = "guardrail_halt"
  159. final_response = agent._toolguard_controlled_halt_response(decision)
  160. agent._emit_diagnostic_status(f"⚠️ Tool guardrail halted {decision.tool_name}: {decision.code}")
  161. append_message(messages, {"role": "assistant", "content": final_response})
  162. # Emit the halt so it isn't mistaken for a crash; the stream callback is still
  163. # alive, so SSE/TUI clients see the explanation. A held turn shows it as the final only.
  164. if final_response:
  165. agent._safe_print(f"\n{final_response}\n")
  166. if agent.stream_delta_callback and not getattr(agent, "_hold_stream", False):
  167. with suppress(Exception):
  168. agent.stream_delta_callback(final_response)
  169. agent.stream_delta_callback(None)
  170. return _verdict("break")
  171. # Reset per-turn retry counters so one truncation can't poison the turn.
  172. truncated_tool_call_retries = 0
  173. # Defer the paragraph break: _fire_stream_delta() prepends one "\n\n" when real
  174. # text arrives, so tool iterations don't stack blank lines.
  175. agent._stream_needs_break = True
  176. # Refund the iteration when the ONLY tool was execute_code (programmatic tool
  177. # calling) — cheap RPC-style calls shouldn't eat the budget.
  178. if {tc.function.name for tc in assistant_message.tool_calls} == {"execute_code"}:
  179. agent.iteration_budget.refund()
  180. _ptc = compress_after_tool_results(
  181. agent, messages=messages, system_message=system_message, user_message=user_message,
  182. active_system_prompt=active_system_prompt, conversation_history=conversation_history,
  183. compression_attempts=compression_attempts,
  184. max_compression_attempts=max_compression_attempts, effective_task_id=effective_task_id,
  185. final_response=final_response, turn_exit_reason=_turn_exit_reason,
  186. current_turn_user_idx=current_turn_user_idx,
  187. )
  188. messages = _ptc.messages
  189. active_system_prompt = _ptc.active_system_prompt
  190. conversation_history = _ptc.conversation_history
  191. compression_attempts = _ptc.compression_attempts
  192. final_response = _ptc.final_response
  193. _turn_exit_reason = _ptc.turn_exit_reason
  194. current_turn_user_idx = _ptc.current_turn_user_idx
  195. if _ptc.end_turn:
  196. return _verdict("break")
  197. # Save session log incrementally (so progress is visible even if interrupted)
  198. agent._session_messages = messages
  199. # Touch activity so slow post-tool work plus a slow follow-up API call can't exceed
  200. # the gateway inactivity timeout (HERMES_AGENT_TIMEOUT).
  201. # Touch activity before continuing so the gateway's inactivity monitor never sees a stale timestamp
  202. # between tool completion and the start of the next API call. Without this, a tool-call result (which
  203. # takes ~0s to process) followed by slow post-tool processing (compression, persist) and a slow
  204. # follow-up API call can exceed the gateway inactivity timeout (HERMES_AGENT_TIMEOUT, default 1800s) and
  205. # the gateway kills the session before the next activity touch fires (#69559, #69131).
  206. agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}")
  207. return _verdict("continue")
  208. def stage_tool_call_message(
  209. agent: Any, *, assistant_message: Any, finish_reason: Any, messages: Any
  210. ) -> tuple[dict[str, Any], bool]:
  211. """Build the assistant tool-call row and update the per-turn fallback/mute state.
  212. Drops a bare bracketed marker beside a call (#78148), classifies housekeeping-only
  213. rounds, keeps visible content as the empty-follow-up fallback, pops thinking-only
  214. prefills (resetting their counters), re-arms the post-tool nudge and the
  215. dropped-tool-call stall budget. Returns ``(assistant_msg, duplicate_previous_interim)``;
  216. the flag suppresses re-emitting interim commentary the previous ``incomplete`` row
  217. already showed."""
  218. from agent.conversation_loop import _STALE_MARKER_RE
  219. assistant_msg = agent._build_assistant_message(assistant_message, finish_reason)
  220. turn_content = assistant_message.content or ""
  221. # A bare bracketed token (e.g. ``[memory]``) beside a function call is protocol
  222. # scaffolding; persisting it lets the post-tool fallback replay it forever (#78148).
  223. if assistant_message.tool_calls and _STALE_MARKER_RE.fullmatch(turn_content.strip()):
  224. logger.warning(
  225. "Discarding bare tool-call marker from assistant content: %s", turn_content
  226. )
  227. turn_content = ""
  228. assistant_msg["content"] = ""
  229. # Classify tools regardless of visible content: a substantive tool-only turn must
  230. # invalidate any older housekeeping fallback (so a two-turn-old housekeeping
  231. # narration isn't attributed to the preceding tool turn), and clear the mute flag a
  232. # prior housekeeping turn set, else _vprint suppresses this turn's tool progress.
  233. _all_housekeeping = all(
  234. tc.function.name in _HOUSEKEEPING_TOOLS for tc in assistant_message.tool_calls
  235. )
  236. # The line written with a blocking card is the reply when the follow-up comes back empty.
  237. _line_is_reply = all(tc.function.name in _LINE_IS_REPLY_TOOLS for tc in assistant_message.tool_calls)
  238. if assistant_message.tool_calls and not _all_housekeeping:
  239. agent._last_content_with_tools = None
  240. agent._last_content_tools_all_housekeeping = False
  241. agent._mute_post_response = False
  242. # Content + tool_calls in one turn: keep the content as a fallback final response in
  243. # case the follow-up turn after tools is empty. Mute only when EVERY tool call is
  244. # post-response housekeeping; substantive tools keep output on.
  245. if turn_content and agent._has_content_after_think_block(turn_content):
  246. agent._last_content_with_tools = turn_content
  247. agent._last_content_tools_all_housekeeping = _line_is_reply
  248. if _all_housekeeping and agent._has_stream_consumers():
  249. agent._mute_post_response = True
  250. elif agent._should_emit_quiet_tool_messages() and not getattr(agent, "_hold_stream", False):
  251. clean = agent._strip_think_blocks(turn_content).strip()
  252. if clean:
  253. agent._vprint(f" ┊ 💬 {clean}")
  254. # Pop thinking-only prefill message(s) before appending (same rationale as the
  255. # final-response path). Tool calls after a prefill recovery reset the prefill
  256. # counter, so each tool-call success is a fresh start, not a cumulative burn.
  257. _had_prefill = False
  258. while messages and isinstance(messages[-1], dict) and messages[-1].get("_thinking_prefill"):
  259. messages.pop()
  260. _had_prefill = True
  261. if _had_prefill:
  262. agent._thinking_prefill_retries = 0
  263. agent._empty_content_retries = 0
  264. # Re-arm the post-tool nudge so it can fire on a LATER tool round; a landed tool call
  265. # recovers any dropped-tool-call stall, so refresh that budget per stall.
  266. agent._post_tool_empty_retried = False
  267. agent._dropped_toolcall_retries = 0
  268. previous_msg = messages[-1] if messages else None
  269. current_interim_visible = agent._interim_assistant_visible_text(assistant_msg)
  270. duplicate_previous_interim = (
  271. bool(current_interim_visible)
  272. and isinstance(previous_msg, dict)
  273. and previous_msg.get("role") == "assistant"
  274. and previous_msg.get("finish_reason") == "incomplete"
  275. and agent._interim_assistant_visible_text(previous_msg) == current_interim_visible
  276. )
  277. return assistant_msg, duplicate_previous_interim