turn_tool_validation.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. """Tool-call validation for the conversation turn loop: unknown tool names (with
  2. auto-repair and the 3-strike partial exit) and malformed JSON arguments (retry, then
  3. recovery tool results).
  4. Role alternation is preserved on every path: an invalid batch is answered with tool-role
  5. error results (never a user message), and the exits close any open tool-result tail
  6. (#48879). Nothing here imports ``agent.conversation_loop`` at module level (cycle).
  7. """
  8. from __future__ import annotations
  9. import json
  10. import logging
  11. from dataclasses import dataclass
  12. from typing import Any, Dict, List, Optional
  13. from agent.message_metadata import append_message
  14. from agent.message_sanitization import (close_interrupted_tool_sequence, coalesce_tool_call_id,
  15. normalize_provider_tool_call_ids)
  16. from agent.turn_failure_copy import site_copy, stamp_failure
  17. from hermes_constants import FINISH_REASON_LENGTH
  18. logger = logging.getLogger("agent.conversation_loop")
  19. @dataclass
  20. class ToolValidationVerdict:
  21. """Outcome of ``validate_tool_calls``.
  22. ``action``: ``"ok"`` (dispatch the calls), ``"continue"`` (re-issue the API call —
  23. error results / retry state were recorded) or ``"return"`` (terminal partial
  24. result in ``result``). ``mixed_invalid_batch`` is True when the batch contains BOTH
  25. valid and unknown tool names: only the invalid calls get error results, the valid
  26. ones run."""
  27. action: str
  28. result: Optional[dict[str, Any]]
  29. mixed_invalid_batch: bool
  30. def _preview_name(name: str) -> str:
  31. return name[:80] + "..." if len(name) > 80 else name
  32. def _append_tool_error_results(messages, tool_calls, content_for) -> None:
  33. """One tool-role result per call so every tool_call keeps a matching result."""
  34. for tc in tool_calls:
  35. append_message(messages, {
  36. "role": "tool",
  37. "name": tc.function.name,
  38. "tool_call_id": coalesce_tool_call_id(tc),
  39. "content": content_for(tc),
  40. })
  41. def _partial_exit(agent, messages, conversation_history, api_call_count, final_response: str) -> dict[str, Any]:
  42. """Terminal partial result. Prior retries or an earlier tool batch leave a tool-result
  43. tail; close it as interrupt aborts do so the next turn is not tool→user (#48879).
  44. This path never reaches finalize_turn, so persist here."""
  45. close_interrupted_tool_sequence(messages, final_response)
  46. agent._persist_session(messages, conversation_history)
  47. return stamp_failure({
  48. "final_response": final_response,
  49. "messages": messages,
  50. "api_calls": api_call_count,
  51. "completed": False,
  52. "partial": True,
  53. "error": final_response,
  54. }, "truncated", True)
  55. def validate_tool_calls(
  56. agent: Any, assistant_message: Any, finish_reason: str, *, messages: list[dict[str, Any]],
  57. conversation_history: Any, api_call_count: int, effective_task_id: Any,
  58. ) -> ToolValidationVerdict:
  59. """Validate ``assistant_message.tool_calls`` in place (ids uniquified, names
  60. repaired, dict/empty args normalized to JSON strings). Strikes for invalid names
  61. advance only when a turn has NO valid call, so a degenerate model still halts at
  62. 3; args cut off mid-stream (routers rewrite ``length`` → ``tool_calls``) are refused
  63. outright rather than retried."""
  64. from agent.conversation_loop import _invalid_tool_name_error_content
  65. tool_calls = assistant_message.tool_calls
  66. valid_names = agent.valid_tool_names
  67. def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ToolValidationVerdict:
  68. return ToolValidationVerdict(action=action, result=result, mixed_invalid_batch=_mixed_invalid_batch)
  69. # Uniquify duplicate tool-call ids BEFORE any downstream consumer: the
  70. # pre-API sanitizer keeps only the first call/result per id, and the desktop
  71. # binds each tool card to its call id. Some providers name every call
  72. # "call_0", so an id already used earlier in the session counts as taken.
  73. taken = {
  74. coalesce_tool_call_id(tc)
  75. for msg in messages if msg.get("role") == "assistant"
  76. for tc in msg.get("tool_calls") or ()
  77. }
  78. agent._uniquify_tool_call_ids(tool_calls, taken=taken)
  79. normalize_provider_tool_call_ids(tool_calls)
  80. # Repair mismatched tool names before validating (model hallucinations).
  81. repaired_ids = set()
  82. for tc in tool_calls:
  83. if tc.function.name not in valid_names:
  84. repaired = agent._repair_tool_call(tc.function.name)
  85. if repaired:
  86. agent._vprint(f"{agent.log_prefix}🔧 Auto-repaired tool name: '{tc.function.name}' -> '{repaired}'",
  87. force=True, diagnostic=True)
  88. tc.function.name = repaired
  89. repaired_ids.add(id(tc))
  90. # Counted here, before any exit or normalization, so every emitted call is seen once as the model sent it.
  91. from hermes_cli.observability.shared_metrics_model import record_tool_call_quality
  92. record_tool_call_quality(agent, tool_calls, repaired_ids)
  93. invalid_tool_calls = [tc.function.name for tc in tool_calls if tc.function.name not in valid_names]
  94. # Mixed batch: error-result ONLY the invalid calls and run the valid
  95. # ones; voiding the turn discards real work. Strikes advance only when a
  96. # turn has NO valid call, so a degenerate model still halts at 3.
  97. _mixed_invalid_batch = bool(invalid_tool_calls) and any(
  98. tc.function.name in valid_names for tc in tool_calls
  99. )
  100. if _mixed_invalid_batch:
  101. agent._invalid_tool_retries = 0
  102. _n_valid = sum(1 for tc in tool_calls if tc.function.name in valid_names)
  103. agent._buffer_vprint(
  104. f"⚠️ Unknown tool '{_preview_name(invalid_tool_calls[0])}' in batch — erroring that call, "
  105. f"executing {_n_valid} valid call(s)"
  106. )
  107. elif invalid_tool_calls:
  108. agent._invalid_tool_retries += 1
  109. # Return helpful error to model — model can agent-correct next turn
  110. invalid_preview = _preview_name(invalid_tool_calls[0])
  111. agent._buffer_vprint(f"⚠️ Unknown tool '{invalid_preview}' — sending error to model for agent-correction ({agent._invalid_tool_retries}/3)")
  112. if agent._invalid_tool_retries >= 3:
  113. agent._flush_status_buffer()
  114. agent._vprint(f"{agent.log_prefix}❌ Max retries (3) for invalid tool calls exceeded. Stopping as partial.", force=True, diagnostic=True)
  115. agent._invalid_tool_retries = 0
  116. return _verdict("return", _partial_exit(
  117. agent, messages, conversation_history, api_call_count,
  118. f"Model generated invalid tool call: {invalid_preview}",
  119. ))
  120. append_message(messages, agent._build_assistant_message(assistant_message, finish_reason))
  121. # See _invalid_tool_name_error_content for the blank-name anti-priming rationale (#47967).
  122. _append_tool_error_results(
  123. messages, tool_calls,
  124. lambda tc: (
  125. _invalid_tool_name_error_content(tc.function.name, valid_names)
  126. if tc.function.name not in valid_names
  127. else "Skipped: another tool call in this turn used an invalid name. Please retry this tool call."
  128. ),
  129. )
  130. return _verdict("continue")
  131. # Reset retry counter on successful tool call validation
  132. agent._invalid_tool_retries = 0
  133. # Validate tool call arguments are valid JSON; empty strings become empty
  134. # objects (common model quirk).
  135. invalid_json_args = []
  136. for tc in tool_calls:
  137. args = tc.function.arguments
  138. if isinstance(args, (dict, list)):
  139. tc.function.arguments = json.dumps(args)
  140. continue
  141. if args is not None and not isinstance(args, str):
  142. tc.function.arguments = args = str(args)
  143. if not args or not args.strip():
  144. tc.function.arguments = "{}"
  145. continue
  146. try:
  147. json.loads(args)
  148. except json.JSONDecodeError as e:
  149. # A mixed-batch invalid-name call never executes (error result later);
  150. # don't let its broken args trigger the whole-turn JSON retry.
  151. if not (_mixed_invalid_batch and tc.function.name not in valid_names):
  152. invalid_json_args.append((tc.function.name, str(e)))
  153. if invalid_json_args:
  154. invalid_names = {n for n, _ in invalid_json_args}
  155. # Routers may rewrite finish_reason "length" → "tool_calls", hiding
  156. # truncation; args not ending in } or ] (stripped) were cut off
  157. # mid-stream.
  158. _truncated = any(
  159. not (tc.function.arguments or "").rstrip().endswith(("}", "]"))
  160. for tc in tool_calls if tc.function.name in invalid_names
  161. )
  162. if _truncated:
  163. agent._vprint(
  164. f"{agent.log_prefix}⚠️ Truncated tool call arguments detected "
  165. f"(finish_reason={finish_reason!r}) — refusing to execute.",
  166. force=True, diagnostic=True,
  167. )
  168. agent._invalid_json_retries = 0
  169. agent._cleanup_task_resources(effective_task_id)
  170. # Blame the output cap only when the model reported one; otherwise the args
  171. # were cut by a stream break or a router rewriting finish_reason (#91717).
  172. _copy = (
  173. site_copy("truncated") if finish_reason == FINISH_REASON_LENGTH
  174. else site_copy("truncated_unreported")
  175. )
  176. return _verdict("return", _partial_exit(
  177. agent, messages, conversation_history, api_call_count, _copy,
  178. ))
  179. agent._invalid_json_retries += 1
  180. tool_name, error_msg = invalid_json_args[0]
  181. agent._buffer_vprint(f"⚠️ Invalid JSON in tool call arguments for '{tool_name}': {error_msg}")
  182. if agent._invalid_json_retries < 3:
  183. agent._buffer_vprint(f"🔄 Retrying API call ({agent._invalid_json_retries}/3)...")
  184. # Don't add anything to messages, just retry the API call
  185. return _verdict("continue")
  186. # Instead of returning partial, inject tool error results so the model can recover.
  187. # Using tool results (not user messages) preserves role alternation.
  188. agent._buffer_vprint("⚠️ Injecting recovery tool results for invalid JSON...")
  189. agent._invalid_json_retries = 0 # Reset for next attempt
  190. # Append the assistant message with its (broken) tool_calls, then one
  191. # error result per call.
  192. append_message(messages, agent._build_assistant_message(assistant_message, finish_reason))
  193. def _json_error_result(tc) -> str:
  194. if tc.function.name not in invalid_names:
  195. return "Skipped: other tool call in this response had invalid JSON."
  196. err = next(e for n, e in invalid_json_args if n == tc.function.name)
  197. return (
  198. f"Error: Invalid JSON arguments. {err}. "
  199. f"For tools with no required parameters, use an empty object: {{}}. "
  200. f"Please retry with valid JSON."
  201. )
  202. _append_tool_error_results(messages, tool_calls, _json_error_result)
  203. return _verdict("continue")
  204. # Reset retry counter on successful JSON validation
  205. agent._invalid_json_retries = 0
  206. return _verdict("ok")