tool_executor.py 96 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966
  1. """Tool-call execution: sequential and concurrent dispatch, extracted from AIAgent.
  2. Functions take the parent ``AIAgent`` first; ``run_agent`` keeps thin wrappers and is
  3. reached lazily via ``_ra()`` so ``run_agent._set_interrupt`` patches still work. Every
  4. call's identity travels as a ``_ToolCallRef``; both executors end in the same
  5. observe → commit → project pipeline so the tool-result wire shape is produced once.
  6. """
  7. from __future__ import annotations
  8. import concurrent.futures
  9. import contextlib
  10. import json
  11. from pathlib import Path
  12. import logging
  13. import os
  14. import random
  15. import threading
  16. import time
  17. from dataclasses import dataclass
  18. from typing import Any, Callable, Optional
  19. from agent.display import (
  20. KawaiiSpinner,
  21. build_tool_preview as _build_tool_preview,
  22. build_tool_label as _build_tool_label,
  23. get_cute_tool_message as _get_cute_tool_message_impl,
  24. get_tool_emoji as _get_tool_emoji,
  25. tool_row_emoji as _tool_row_emoji,
  26. redact_tool_args_for_display as _redact_tool_args_for_display,
  27. _detect_tool_failure,
  28. )
  29. from agent.compression_marker import _COMPRESSION_MARKER_PREFIX
  30. from agent.interrupt_control import _REASON_USER_INTERRUPT, interrupt_skip_wording
  31. from agent.message_sanitization import coalesce_tool_call_id
  32. from agent.onboarding import observe_tool_result as observe_onboarding_result
  33. from agent import plugin_live_notes
  34. from agent.lsp import deferred as lsp_deferred
  35. from agent.inline_tool_executors import (
  36. INLINE_TOOL_EXECUTORS,
  37. InlineToolContext,
  38. apply_transform_tool_result,
  39. emit_terminal_post_tool_call,
  40. tool_hook_ids,
  41. )
  42. from agent.message_metadata import stamp_data_use
  43. from agent.session_persistence import sync_rewritten_tool_rows
  44. from agent.tool_dispatch_helpers import (
  45. _NEVER_PARALLEL_TOOLS,
  46. _is_destructive_command,
  47. _is_multimodal_tool_result,
  48. _multimodal_text_summary,
  49. _append_subdir_hint_to_multimodal,
  50. _context_pruned_argument_paths,
  51. _plan_tool_batch_segments,
  52. make_tool_result_message,
  53. )
  54. from tools.terminal_tool_lifecycle import get_active_env
  55. from tools.thread_context import propagate_context_to_thread
  56. from tools.tool_result_storage import (
  57. maybe_persist_tool_result,
  58. enforce_turn_budget,
  59. extract_persisted_path,
  60. )
  61. from tools.budget_config import BudgetConfig, DEFAULT_BUDGET, budget_for_context_window
  62. from hermes_cli.observability.shared_metrics_efficiency import note_tool_result, record_tool_batch
  63. # A tool result this large (raw stdout, file dumps) is the biggest allocation a turn ever drops.
  64. # The commit only flags it: the string is still referenced by the publish frames here, so the
  65. # trim runs once the whole batch has unwound (AIAgent._execute_tool_calls) (#70684).
  66. _LARGE_TOOL_RESULT_TRIM_CHARS = 1_000_000
  67. logger = logging.getLogger(__name__)
  68. _pairing_tool_call_id = coalesce_tool_call_id # canonical id used by the persisted assistant message
  69. def _tc_name(tool_call: Any) -> str:
  70. return getattr(getattr(tool_call, "function", None), "name", "") or "tool"
  71. def _record_persisted_path_for_stub(agent, tool_call_id: str, function_result) -> None:
  72. """Record the spillover file path so a later result-reference stub can't dangle (best-effort)."""
  73. try:
  74. candidates = [function_result] if isinstance(function_result, str) else [
  75. function_result.get("text_summary"),
  76. *(p.get("text") for p in function_result.get("content") or [] if isinstance(p, dict)),
  77. ] if _is_multimodal_tool_result(function_result) else []
  78. path = next((p for p in map(extract_persisted_path, candidates) if p), None)
  79. if path:
  80. agent._tool_guardrails.record_persisted_result(tool_call_id, path)
  81. except Exception as exc:
  82. logger.debug("persisted-path record for result stub failed: %s", exc)
  83. def _ensure_file_checkpoint(agent, function_name: str, function_args: dict, effective_task_id: str) -> None:
  84. """Checkpoint the same workspace path that the file tool will mutate, resolved the way
  85. file tools do (against the task's live cwd, which differs from the process cwd in Docker)."""
  86. file_path = function_args.get("path", "")
  87. if not file_path:
  88. return
  89. from agent.file_safety import is_nt_namespace_path
  90. from tools.file_tools_paths import _resolve_path_for_task, container_backend_for_task
  91. if container_backend_for_task(effective_task_id or "default") is not None:
  92. return # container paths: nothing to checkpoint on the host
  93. # Resolving an NT-namespace path is itself the NTLM-leak trigger; leave the
  94. # tool's raw-string guard to refuse it without a checkpoint stat.
  95. if is_nt_namespace_path(file_path):
  96. return
  97. resolved_path = _resolve_path_for_task(file_path, effective_task_id or "default")
  98. agent._checkpoint_mgr.ensure_checkpoint(
  99. agent._checkpoint_mgr.get_working_dir_for_path(str(resolved_path)), f"before {function_name}",
  100. )
  101. def _budget_for_agent(agent) -> BudgetConfig:
  102. """Tool-result BudgetConfig scaled to the agent's context window. Unknown length goes
  103. through ``budget_for_context_window(None)`` (not DEFAULT_BUDGET) so the MCP threshold
  104. override still applies.
  105. Large-context models keep the historical 100K/200K char defaults; small models (e.g. a 65K-token local
  106. model switched into mid-session) get a budget proportional to their window so a single large tool result
  107. can't push the request past the model's limit (#23767). Falls back to the default budget when the
  108. context length isn't resolvable.
  109. """
  110. try:
  111. ctx = getattr(getattr(agent, "context_compressor", None), "context_length", None)
  112. return budget_for_context_window(int(ctx) if ctx else None)
  113. except Exception:
  114. return DEFAULT_BUDGET
  115. _MAX_TOOL_WORKERS = 8 # concurrent worker threads per batch
  116. _DEFAULT_IMAGE_PARALLEL_REQUESTS = 4
  117. # Generous: slow-but-valid tool work must never be preempted by the batch guard.
  118. _DEFAULT_CONCURRENT_TOOL_TIMEOUT_S = 420.0
  119. # Long enough for an approval round-trip, short enough that one wedged dispatch can't starve the batch.
  120. _START_ORDER_GATE_TIMEOUT_S = 120.0
  121. # Fallback only; the effective bound derives from approvals.timeout (_authorization_gate_lock_timeout).
  122. _AUTHORIZATION_GATE_LOCK_TIMEOUT_S = 360.0
  123. def _authorization_gate_lock_timeout() -> float:
  124. """Authorization-lock bound = ``tools.approval_human_wait.human_wait_ceiling`` (approval timeout +
  125. margin, capped so it can't overflow Lock.acquire): never break serialization while a
  126. prompt is answerable, never let a wedged holder park workers forever. Deliberately NOT
  127. min()'d with the fallback so the gate never gives up early.
  128. Delegates to ``tools.approval_human_wait.human_wait_ceiling`` — the same bound that clamps a human-wait window's
  129. deadline contribution — so the two can't drift. Long enough that serialization is never broken while a
  130. legitimate approval prompt is still answerable; short enough that a wedged holder (hanging
  131. ``pre_tool_call`` plugin, dead approval client) cannot park other workers forever (#79719). Resolved
  132. once per gate (per batch), so a mid-process ``approvals.timeout`` change applies from the next batch.
  133. """
  134. try:
  135. from tools.approval_human_wait import human_wait_ceiling
  136. # human_wait_ceiling is platform-safety-capped (agent/deadline.py MAX_SAFE_TIMEOUT_S): a huge
  137. # approvals.timeout can no longer overflow Lock.acquire's time_t on macOS (#83220). Deliberately NOT
  138. # min()'d with _AUTHORIZATION_GATE_LOCK_TIMEOUT_S — the gate must never give up while a legitimate
  139. # approval prompt is still answerable (#79719), so a configured approvals.timeout above 360s must
  140. # extend the gate.
  141. return human_wait_ceiling()
  142. except Exception:
  143. return _AUTHORIZATION_GATE_LOCK_TIMEOUT_S
  144. class _BatchAbandoned(BaseException):
  145. """Raised inside a worker when the batch was abandoned before dispatch; a BaseException
  146. so ``except Exception`` handlers in the middleware chain can't swallow it."""
  147. def _parse_tool_arguments(raw_arguments: Any) -> tuple[dict, Optional[str]]:
  148. """Parse model-emitted arguments without repairing or coercing them."""
  149. try:
  150. arguments = json.loads(raw_arguments)
  151. except (json.JSONDecodeError, TypeError):
  152. arguments = None
  153. if isinstance(arguments, dict):
  154. return arguments, None
  155. return {}, json.dumps(
  156. {"error": "Invalid tool arguments", "message": "Tool arguments must be a valid JSON object; tool was not executed."},
  157. ensure_ascii=False,
  158. )
  159. def _resolve_concurrent_tool_timeout() -> float | None:
  160. """Per-batch concurrent deadline: ``timeouts.tools.concurrent_batch`` wins,
  161. ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` is the legacy bridge, ``0``/negative disables."""
  162. from agent.deadline import resolve_timeout
  163. return resolve_timeout(
  164. "tools.concurrent_batch",
  165. default=_DEFAULT_CONCURRENT_TOOL_TIMEOUT_S,
  166. env_var="HERMES_CONCURRENT_TOOL_TIMEOUT_S",
  167. )
  168. def _flush_session_db_after_tool_progress(agent, messages: list, *, stage: str) -> bool:
  169. """Flush tool-call progress to the session DB before projecting it to any UI: tool side
  170. effects can kill/restart the process before turn-end persistence runs."""
  171. from agent.conversation_loop import _maybe_inject_run_budget_wrapup
  172. from agent.turn_iteration_prep import _maybe_inject_iteration_budget_warning
  173. # Persist exactly the checkpoint text the next model call will see, before stamping
  174. # this tool result as durable. Already-written rows must never be rewritten later.
  175. _maybe_inject_run_budget_wrapup(agent, messages)
  176. _maybe_inject_iteration_budget_warning(agent, messages)
  177. try:
  178. persisted = agent._flush_messages_to_session_db(messages) is not False
  179. if not persisted:
  180. agent._incremental_persistence_failed = True
  181. # The flush recorded any classified cause; default to 'unknown' only if nothing more specific exists.
  182. if getattr(agent, "_last_persistence_error_cause", None) is None:
  183. agent._last_persistence_error_cause = "unknown"
  184. return persisted
  185. except Exception as exc:
  186. agent._incremental_persistence_failed = True
  187. from hermes_state import classify_persistence_error
  188. agent._last_persistence_error_cause = classify_persistence_error(exc)
  189. logger.warning("Incremental tool-call persistence failed after %s: %s", stage, exc)
  190. return False
  191. def _image_generate_parallel_limit() -> int:
  192. """Configured image-generation parallelism cap (conservative: backend bursts hit rate limits)."""
  193. try:
  194. from hermes_cli.config import load_config
  195. cfg = load_config() or {}
  196. image_gen = cfg.get("image_gen") if isinstance(cfg, dict) else None
  197. value = image_gen.get("max_parallel_requests") if isinstance(image_gen, dict) else None
  198. except Exception:
  199. value = None
  200. try:
  201. limit = int(value)
  202. except (TypeError, ValueError):
  203. limit = _DEFAULT_IMAGE_PARALLEL_REQUESTS
  204. return max(1, min(limit, _MAX_TOOL_WORKERS))
  205. def _max_workers_for_tool_batch(runnable_calls) -> int:
  206. """Return the worker cap for a concurrent tool batch."""
  207. if not runnable_calls:
  208. return 0
  209. max_workers = _MAX_TOOL_WORKERS
  210. if any((call[2] if len(call) >= 3 else None) == "image_generate" for call in runnable_calls):
  211. max_workers = min(max_workers, _image_generate_parallel_limit())
  212. return min(len(runnable_calls), max_workers)
  213. def _ra():
  214. """Lazy reference to ``run_agent`` so patches like ``run_agent._set_interrupt`` work."""
  215. import run_agent
  216. return run_agent
  217. def _is_interpreter_shutdown_submit_error(exc: RuntimeError) -> bool:
  218. """Shutdown-race predicate; ``tools.interpreter_shutdown`` knows both CPython message variants.
  219. Delegates so all sites (cron delivery, conversation-loop retry, tool submission) recognize both CPython
  220. shutdown-message variants instead of each matching its own substring (the bug class behind
  221. #55924/#58720).
  222. """
  223. from tools.interpreter_shutdown import interpreter_shutting_down
  224. return interpreter_shutting_down(exc)
  225. _emit_terminal_post_tool_call = emit_terminal_post_tool_call
  226. @dataclass
  227. class _ToolCallRef:
  228. """Identity of one tool call as every hook / result message sees it: the (possibly
  229. middleware-rewritten) name and args, the task, the pairing id and the request trace."""
  230. name: str
  231. args: dict
  232. task_id: str
  233. call_id: str
  234. trace: list
  235. def middleware_kwargs(self) -> dict[str, Any]:
  236. """Keyword form ``_run_agent_tool_execution_middleware`` (and tests patching it) expect."""
  237. return {
  238. "function_name": self.name, "function_args": self.args, "effective_task_id": self.task_id,
  239. "tool_call_id": self.call_id, "middleware_trace": self.trace,
  240. }
  241. def emit_post(self, agent, result, *, trace=None, **outcome) -> None:
  242. """Emit the one terminal ``post_tool_call`` for this call (``outcome`` = status /
  243. error_type / error_message / duration_ms). Resolved through the module attribute so
  244. tests patching ``_emit_terminal_post_tool_call`` still intercept."""
  245. _emit_terminal_post_tool_call(
  246. agent,
  247. function_name=self.name,
  248. function_args=self.args,
  249. result=result,
  250. effective_task_id=self.task_id,
  251. tool_call_id=self.call_id,
  252. middleware_trace=list(self.trace if trace is None else trace),
  253. **outcome,
  254. )
  255. def emit_cancelled(self, agent, start_time: float) -> str:
  256. """Synthesize the ``cancelled`` result for a KeyboardInterrupt mid-tool and emit its hook."""
  257. message = f"Tool execution cancelled. {interrupt_skip_wording(agent)}"
  258. result = json.dumps({"error": message, "status": "cancelled"}, ensure_ascii=False)
  259. self.emit_post(
  260. agent, result, duration_ms=int((time.time() - start_time) * 1000),
  261. status="cancelled", error_type="keyboard_interrupt", error_message=message,
  262. )
  263. return result
  264. def emit_invalid_arguments(self, agent, result: str) -> None:
  265. self.emit_post(
  266. agent, result, trace=[],
  267. status="error", error_type="invalid_tool_arguments", error_message="Tool arguments must be a valid JSON object",
  268. )
  269. def _append_skipped_tool_results(
  270. agent,
  271. messages: list,
  272. tool_calls,
  273. effective_task_id: str,
  274. *,
  275. content: str,
  276. hook_error_type: Optional[str] = None,
  277. hook_id: Optional[Callable[[Any], str]] = None,
  278. flush_stage: Optional[str] = None,
  279. stop_on_flush_failure: bool = True,
  280. ) -> bool:
  281. """Append one ``tool`` result per unstarted call so the assistant tool-call turn never
  282. lacks matching results (role alternation). ``{name}`` in ``content`` is substituted with
  283. ``str.replace``, never ``str.format`` (a recorded reason may contain braces);
  284. ``hook_error_type`` also emits the terminal ``post_tool_call`` (status=cancelled) per
  285. call with ``hook_id`` overriding the hook's id; ``flush_stage`` flushes after each
  286. append and returns False on the first failed flush when ``stop_on_flush_failure``."""
  287. for tc in tool_calls:
  288. name = _tc_name(tc)
  289. result = content.replace("{name}", name)
  290. messages.append(make_tool_result_message(name, result, _pairing_tool_call_id(tc), effect_disposition="none"))
  291. if hook_error_type is not None:
  292. _ToolCallRef(name, {}, effective_task_id, (hook_id or _pairing_tool_call_id)(tc), []).emit_post(
  293. agent, result,
  294. status="cancelled", error_type=hook_error_type, error_message=f"Tool execution skipped. {interrupt_skip_wording(agent)}",
  295. )
  296. if flush_stage is not None:
  297. flushed = _flush_session_db_after_tool_progress(agent, messages, stage=f"{flush_stage} {name}")
  298. if not flushed and stop_on_flush_failure:
  299. return False
  300. return True
  301. def _tool_search_scoped_names(agent) -> frozenset:
  302. """Deferrable tool names the session may invoke via ``tool_call``; the unwrap bypasses
  303. the bridge's scope check in ``model_tools.handle_function_call``, so restricted sessions
  304. validate against this set. Read through the ``get_tool_definitions`` memo on every call
  305. and never copied onto the agent: the memo expires with the check_fn verdicts it was
  306. filtered on, a per-agent copy keyed on registry generation would not, and a recovered
  307. MCP server's tools would stay uncallable for the agent's lifetime."""
  308. try:
  309. import model_tools
  310. from tools import tool_search as _ts
  311. except Exception:
  312. return frozenset()
  313. try:
  314. return _ts.scoped_deferrable_names(model_tools.get_tool_definitions(
  315. enabled_toolsets=getattr(agent, "enabled_toolsets", None),
  316. disabled_toolsets=getattr(agent, "disabled_toolsets", None),
  317. quiet_mode=True, skip_tool_search_assembly=True,
  318. ) or [])
  319. except Exception:
  320. return frozenset()
  321. def _canonical_tool_name(function_name: str) -> str:
  322. """Map legacy tool-name aliases BEFORE agent-loop dispatch."""
  323. from model_tools import _LEGACY_TOOL_ALIASES as _lta
  324. return _lta.get(function_name, function_name)
  325. def _unwrap_tool_search_call(
  326. agent, function_name: str, function_args: dict, *, flatten_probe: bool = False
  327. ) -> tuple[str, dict, Optional[str]]:
  328. """Peel the ``tool_call`` bridge so downstream hooks (checkpointing, guardrails, plugin
  329. hooks, activity feed) see the underlying tool; ``tool_call.function`` stays untouched for
  330. the transcript and tool_call_id pairing.
  331. The unwrap bypasses handle_function_call's scope check, so session toolset scope is
  332. enforced HERE. Returns ``(name, args, scope_block)``; ``scope_block`` is the block
  333. message when the underlying tool is out of scope or its args fail the deferred-schema
  334. probe (``flatten_probe`` collapses the probe's JSON payload to one plain string for
  335. callers that wrap the message in ``{"error": ...}``).
  336. """
  337. scope_block: Optional[str] = None
  338. if function_name == "tool_call":
  339. from agent.runtime_context import current as embedded_context
  340. if embedded_context() is not None:
  341. # The system connector service validates the whole envelope and
  342. # approves each remote call. Never unwrap into a local/native tool.
  343. return function_name, function_args, None
  344. try:
  345. from tools import tool_search as _ts
  346. if function_name != _ts.TOOL_CALL_NAME:
  347. return function_name, function_args, None
  348. underlying, underlying_args, err = _ts.resolve_underlying_call(function_args)
  349. if err or not underlying:
  350. return function_name, function_args, None
  351. if underlying == _ts.CONNECTOR_BATCH_SENTINEL:
  352. # Both executors retain the wrapper: scope/probe/hooks run per entry
  353. # in the batch dispatcher, not against a synthetic registry name.
  354. return function_name, function_args, None
  355. if underlying not in _tool_search_scoped_names(agent):
  356. # Session-gated GUI tools name their missing surface (#120413);
  357. # anything else keeps the generic block.
  358. return function_name, function_args, _ts.scope_block_reason(underlying)
  359. # Validate before unwrapping: the generic bridge hides the concrete
  360. # parameter schema from provider-native tool-call validation.
  361. scope_block = _ts.validate_deferred_call_args(underlying, underlying_args)
  362. if scope_block is None:
  363. return underlying, underlying_args, None
  364. if flatten_probe:
  365. probe = json.loads(scope_block)
  366. scope_block = (
  367. f"{probe.get('error', '')} Parameters schema: "
  368. f"{json.dumps(probe.get('parameters', {}), ensure_ascii=False)}. "
  369. f"{probe.get('hint', '')}"
  370. ).strip()
  371. except Exception:
  372. pass
  373. return function_name, function_args, scope_block
  374. @dataclass
  375. class _ParsedCall:
  376. """One model tool call after alias canonicalization, arg parsing and bridge unwrap."""
  377. tool_call: Any
  378. name: str
  379. args: dict
  380. middleware_trace: list
  381. parse_error: Optional[str]
  382. scope_block: Optional[str]
  383. def ref(self, task_id: str) -> _ToolCallRef:
  384. return _ToolCallRef(self.name, self.args, task_id, _pairing_tool_call_id(self.tool_call), self.middleware_trace)
  385. def _parse_tool_call(agent, tool_call, *, flatten_probe: bool = False) -> _ParsedCall:
  386. name = _canonical_tool_name(tool_call.function.name)
  387. args, parse_error = _parse_tool_arguments(tool_call.function.arguments)
  388. scope_block = None
  389. if parse_error is None:
  390. name, args, scope_block = _unwrap_tool_search_call(agent, name, args, flatten_probe=flatten_probe)
  391. return _ParsedCall(tool_call, name, args, [], parse_error, scope_block)
  392. @dataclass
  393. class _ManagedToolResult:
  394. result: Any
  395. args: dict[str, Any]
  396. middleware_trace: list[dict[str, Any]]
  397. blocked: bool
  398. dispatched: bool
  399. data_use: frozenset[str] = frozenset()
  400. class _ToolTimeoutResult(str):
  401. """Marker for a synthesized sequential-tool timeout result."""
  402. class _ToolCancelledResult(str):
  403. """Marker for a synthesized sequential-tool user-interrupt result; its terminal
  404. post_tool_call was already emitted, so a late-finishing abandoned worker must not report."""
  405. class _ConcurrentToolAuthorizationGate:
  406. """Serialize policy prompts and exclude human approval waits from batch deadlines.
  407. The acquire is BOUNDED: on expiry the worker prompts unserialized rather than starving
  408. the batch behind a wedged plugin/approval client. Exclusion is measured at the SOURCE
  409. of the human wait (``tools.approval.human_wait_seconds``), NOT as gate residency —
  410. residency-based exclusion let a wedged plugin keep the deadline from ever firing.
  411. Serialization keeps concurrent approval prompts from interleaving on the user's screen. The acquire is
  412. BOUNDED: a worker wedged inside the gate (a hanging ``pre_tool_call`` plugin, or an approval round-trip
  413. to a client that went away) must not park every other worker forever. On expiry the worker runs its
  414. prompt unserialized — worst case is interleaved prompts, strictly better than permanent starvation (same
  415. tradeoff as the start-order gate, #79705).
  416. Gate residency is arbitrary code — using it as the exclusion signal let a wedged plugin grow the
  417. exclusion 1:1 with wall clock, keeping the batch deadline's ``remaining`` constant so it never fired and
  418. the turn hung forever (#79719). A wedged plugin now contributes nothing to the exclusion and the batch
  419. times out normally, while a genuine approval wait (which can legitimately exceed any fixed bound) is
  420. still excluded in full.
  421. """
  422. def __init__(self, *, lock_timeout: float | None = None, session_key: str | None = None) -> None:
  423. self._serialization_lock = threading.Lock()
  424. self._lock_timeout = _authorization_gate_lock_timeout() if lock_timeout is None else lock_timeout
  425. self._session_key = session_key
  426. if self._session_key is None:
  427. # Snapshot on the SUBMITTING thread: excluded_seconds() is polled from the
  428. # batch wait loop, whose context may differ from the workers'.
  429. try:
  430. from tools.approval_context import get_current_session_key
  431. self._session_key = get_current_session_key()
  432. except Exception:
  433. logger.debug(
  434. "authorization gate could not snapshot the session key; "
  435. "human-wait exclusion will re-resolve it at poll time",
  436. exc_info=True,
  437. )
  438. self._baseline_wait_seconds = self._human_wait_seconds()
  439. def _human_wait_seconds(self) -> float:
  440. try:
  441. from tools.approval_human_wait import human_wait_seconds
  442. return human_wait_seconds(self._session_key)
  443. except Exception:
  444. return 0.0
  445. def run(self, callback):
  446. if not self._serialization_lock.acquire(timeout=self._lock_timeout):
  447. # Deterministic failure (bad command, non-MCP URL, 401/403): every retry hits the same wall.
  448. # Park immediately instead of burning the retry ladder and spamming N identical warnings
  449. # (#65673). Auth failures park here too rather than returning. Returning ends the run task, and
  450. # with it the only listener on ``_reconnect_event`` — so a 401 on the very first connect left
  451. # the server unrevivable for the life of the process, even after the user re-authenticated with
  452. # ``hermes mcp login``. Parking keeps the task alive so the 300s self-probe (and an explicit
  453. # /mcp refresh) can pick up fresh tokens.
  454. logger.warning(
  455. "authorization gate lock not acquired after %.1fs "
  456. "(holder wedged in a pre_tool_call plugin or approval "
  457. "round-trip?); running prompt unserialized",
  458. self._lock_timeout,
  459. )
  460. return callback()
  461. try:
  462. return callback()
  463. finally:
  464. self._serialization_lock.release()
  465. def excluded_seconds(self) -> float:
  466. """Return human-approval wait seconds accrued since the batch started."""
  467. return max(0.0, self._human_wait_seconds() - self._baseline_wait_seconds)
  468. @contextlib.contextmanager
  469. def _registered_tool_worker(agent):
  470. """Track this worker tid for interrupt fan-out (``AIAgent.interrupt()``); on ANY exit
  471. (incl. BaseException) discard it and clear its interrupt bit so a recycled tid starts clean."""
  472. tid = threading.current_thread().ident
  473. with agent._tool_worker_threads_lock:
  474. agent._tool_worker_threads.add(tid)
  475. try:
  476. yield tid
  477. finally:
  478. with agent._tool_worker_threads_lock:
  479. agent._tool_worker_threads.discard(tid)
  480. with contextlib.suppress(Exception):
  481. _ra()._set_interrupt(False, tid)
  482. _NO_REASON = object()
  483. def _interrupt_worker_tids(agent, tids, *, reason=_NO_REASON) -> None:
  484. """Raise the interrupt bit on each worker tid (best-effort, via ``run_agent``)."""
  485. kwargs = {} if reason is _NO_REASON else {"reason": reason}
  486. for tid in tids:
  487. with contextlib.suppress(Exception):
  488. _ra()._set_interrupt(True, tid, **kwargs)
  489. def _set_worker_activity_callback(agent) -> None:
  490. """The activity callback is thread-local: bind it on THIS thread so tool-layer heartbeats fire."""
  491. with contextlib.suppress(Exception):
  492. from tools.environments.base import set_activity_callback
  493. set_activity_callback(agent._touch_activity)
  494. # Must stay far below the gateway turn-inactivity timeout (default 1800s) so a silent tool never looks idle.
  495. _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S = 30.0
  496. def _run_tool_activity_heartbeat(
  497. agent,
  498. stop_event: threading.Event,
  499. label: str,
  500. interval: float = _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S,
  501. worker_tid: int | None = None,
  502. ) -> None:
  503. """Daemon thread stamping ``agent._touch_activity`` every ``interval`` seconds until
  504. ``stop_event`` is set, so the gateway inactivity watchdog never abandons a turn whose
  505. tool runs silently. Wedged tools stay bounded by the tool layer's own timeouts and by the
  506. executor deadline — but a worker the executor gave up on never reaches its ``stop_event``,
  507. so the heartbeat also exits once ``worker_tid`` carries the interrupt bit the abandoning
  508. executor raises (``_interrupt_worker_tids``). Otherwise a tool wedged in a kernel probe
  509. keeps reporting "activity" for the rest of the run and the inactivity watchdog, the second
  510. line of defense, can never fire (#111922)."""
  511. from tools.interrupt import is_thread_interrupted
  512. try:
  513. while not stop_event.wait(interval):
  514. if is_thread_interrupted(worker_tid):
  515. return
  516. agent._touch_activity(label)
  517. except Exception:
  518. pass # a heartbeat must never break the agent loop
  519. def _run_with_activity_heartbeat(agent, function_name: str, fn):
  520. """Run ``fn()`` under the activity heartbeat; covers both executor paths."""
  521. stop = threading.Event()
  522. thread = threading.Thread(
  523. # Keep the gateway turn-inactivity watchdog from abandoning a turn whose tool call runs silently for
  524. # longer than the inactivity timeout (#84491): stamp activity periodically while the tool is in
  525. # flight, not just at start/completion. Both the sequential and the concurrent paths funnel through
  526. # here, so a single heartbeat covers every tool.
  527. target=_run_tool_activity_heartbeat,
  528. args=(agent, stop, f"tool running: {function_name}"),
  529. kwargs={"interval": _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S, "worker_tid": threading.current_thread().ident},
  530. daemon=True,
  531. name=f"tool-activity-hb-{function_name[:24]}",
  532. )
  533. thread.start()
  534. try:
  535. return fn()
  536. finally:
  537. stop.set()
  538. thread.join(timeout=2.0)
  539. _PRUNED_TOOL_ARGUMENTS_ERROR = "suspected_pruned_tool_arguments"
  540. _PRUNED_TOOL_ARGUMENTS_MESSAGE = (
  541. "Tool was not executed because effect-capable arguments contain a Hermes context-compression artifact. "
  542. "Recover the exact content from its durable source or re-read it, then issue a complete new call; "
  543. "do not retry these arguments. To remove a marker that already landed in a file, match it by its "
  544. f"{_COMPRESSION_MARKER_PREFIX.strip('⟪:')} prefix (e.g. a terminal sed on that line) instead of quoting the full marker."
  545. )
  546. def _blocked_tool_result(agent, ref: _ToolCallRef, *, block_body: dict[str, Any] | None, block_error_type: str, guardrail_decision) -> str:
  547. """Synthesize the result for a call blocked by scope/plugin/pruned-args (``block_body``, the
  548. JSON the model sees) or by guardrail policy (``guardrail_decision``) and emit its terminal post_tool_call."""
  549. if block_body is not None:
  550. result = json.dumps(block_body, ensure_ascii=False)
  551. error_type, error_message = block_error_type, block_body.get("message") or block_body["error"]
  552. else:
  553. result = agent._guardrail_block_result(guardrail_decision)
  554. error_type = "guardrail_block"
  555. error_message = getattr(guardrail_decision, "message", None) or "Tool blocked by guardrail policy"
  556. ref.emit_post(agent, result, status="blocked", error_type=error_type, error_message=error_message)
  557. return result
  558. def _pre_tool_block(agent, ref: _ToolCallRef):
  559. """Run ``pre_tool_call`` plugin hooks; returns ``(block_message, final_args)`` with any
  560. hook-modified args applied. Hook failures never block."""
  561. try:
  562. from hermes_cli.plugins import _dispatch_pre_tool_call_hooks
  563. block_msg, modified_args = _dispatch_pre_tool_call_hooks(
  564. ref.name,
  565. ref.args,
  566. **tool_hook_ids(agent, ref.task_id, ref.call_id),
  567. middleware_trace=list(ref.trace),
  568. )
  569. return block_msg, (ref.args if modified_args is None else modified_args)
  570. except Exception:
  571. return None, ref.args
  572. def _dispatch_authorized_once(
  573. agent,
  574. state: _ManagedToolResult,
  575. ref: _ToolCallRef,
  576. *,
  577. execute,
  578. scope_block: str | None,
  579. display_index: int | None,
  580. begin_execution,
  581. authorization_gate: _ConcurrentToolAuthorizationGate | None,
  582. ) -> Any:
  583. """Hermes policy (scope → plugin pre-hooks → pruned-arg check → guardrails) then the one real dispatch.
  584. Plugin ``modify`` hooks may rewrite ``ref.args`` (mirrored into ``state.args``).
  585. ``begin_execution`` (concurrent start-order gate) is advanced exactly once on every
  586. path so later-ordered workers keep moving; blocked calls advance it without a callback.
  587. """
  588. def _advance_start_order(callback=None) -> None:
  589. if begin_execution is not None:
  590. begin_execution(callback)
  591. elif callback is not None:
  592. callback()
  593. block_message, block_error_type = scope_block, "tool_scope_block"
  594. if block_message is None:
  595. block_error_type = "plugin_block"
  596. resolve = lambda: _pre_tool_block(agent, ref)
  597. block_message, ref.args = resolve() if authorization_gate is None else authorization_gate.run(resolve)
  598. state.args = ref.args
  599. block_body = None if block_message is None else {"error": block_message}
  600. # Checked once, after plugin modify hooks (which may replace arguments) and
  601. # before guardrails or real dispatch: a copied compression marker in an
  602. # effect-capable argument must never reach the tool.
  603. if block_body is None:
  604. pruned_paths = _context_pruned_argument_paths(ref.name, ref.args)
  605. if pruned_paths:
  606. block_body = {
  607. "error": _PRUNED_TOOL_ARGUMENTS_ERROR,
  608. "message": _PRUNED_TOOL_ARGUMENTS_MESSAGE,
  609. "argument_paths": pruned_paths,
  610. }
  611. block_error_type = _PRUNED_TOOL_ARGUMENTS_ERROR
  612. guardrail_decision = None
  613. if block_body is None:
  614. guardrail_decision = agent._tool_guardrails.before_call(ref.name, ref.args)
  615. if guardrail_decision.allows_execution:
  616. guardrail_decision = None
  617. if block_body is not None or guardrail_decision is not None:
  618. _advance_start_order()
  619. state.blocked = True
  620. return _blocked_tool_result(
  621. agent, ref,
  622. block_body=block_body, block_error_type=block_error_type, guardrail_decision=guardrail_decision,
  623. )
  624. if ref.name == "memory":
  625. agent._turns_since_memory = 0
  626. elif ref.name == "skill_manage":
  627. agent._iters_since_skill = 0
  628. from agent.terminal_approval_batch import prepare_current_terminal
  629. prepare_current_terminal(ref)
  630. _advance_start_order(lambda: _begin_tool_execution(agent, ref, display_index))
  631. from agent.tool_progress_relay import bound_tool_progress
  632. from tools.connectors.data_use import collect_data_use
  633. with bound_tool_progress(agent.tool_progress_callback, ref.name, ref.call_id), collect_data_use() as data_use:
  634. try:
  635. return _run_with_activity_heartbeat(agent, ref.name, lambda: execute(ref.args))
  636. finally:
  637. state.data_use = frozenset(data_use)
  638. def _run_agent_tool_execution_middleware(
  639. agent,
  640. *,
  641. function_name: str,
  642. function_args: dict,
  643. effective_task_id: str,
  644. tool_call_id: str,
  645. execute,
  646. scope_block: str | None = None,
  647. display_index: int | None = None,
  648. middleware_trace: list[dict[str, Any]] | None = None,
  649. begin_execution=None,
  650. authorization_gate: _ConcurrentToolAuthorizationGate | None = None,
  651. ) -> _ManagedToolResult:
  652. """Run Relay rewrites before Hermes policy and dispatch exactly once."""
  653. from agent import relay_tools
  654. from hermes_cli.middleware import (
  655. apply_tool_request_middleware,
  656. run_tool_execution_middleware,
  657. )
  658. trace = middleware_trace if middleware_trace is not None else []
  659. state = _ManagedToolResult(result=None, args=function_args, middleware_trace=trace, blocked=False, dispatched=False)
  660. dispatch_lock = threading.Lock()
  661. def _authorized_dispatch(final_args: dict[str, Any]) -> Any:
  662. with dispatch_lock:
  663. if state.dispatched:
  664. raise RuntimeError("Hermes tool execution callback invoked more than once")
  665. state.dispatched = True
  666. state.blocked = False
  667. state.args = final_args
  668. return _dispatch_authorized_once(
  669. agent,
  670. state,
  671. _ToolCallRef(function_name, final_args, effective_task_id, tool_call_id, trace),
  672. execute=execute,
  673. scope_block=scope_block,
  674. display_index=display_index,
  675. begin_execution=begin_execution,
  676. authorization_gate=authorization_gate,
  677. )
  678. from agent.terminal_approval_batch import bind_prepared_dispatch
  679. _authorized_dispatch = bind_prepared_dispatch(_authorized_dispatch)
  680. def _hermes_pipeline(relay_args: dict[str, Any]) -> Any:
  681. request_result = apply_tool_request_middleware(
  682. function_name,
  683. relay_args,
  684. original_args=function_args,
  685. skip_relay=True,
  686. **tool_hook_ids(agent, effective_task_id, tool_call_id),
  687. )
  688. request_args = request_result.payload if isinstance(request_result.payload, dict) else relay_args
  689. trace.clear()
  690. trace.extend(request_result.trace)
  691. return run_tool_execution_middleware(
  692. function_name,
  693. request_args,
  694. lambda next_args: _authorized_dispatch(next_args if isinstance(next_args, dict) else request_args),
  695. original_args=function_args,
  696. **tool_hook_ids(agent, effective_task_id, tool_call_id),
  697. )
  698. state.result, _relay_args = relay_tools.execute(
  699. function_name,
  700. function_args,
  701. _hermes_pipeline,
  702. session_id=str(getattr(agent, "session_id", "") or ""),
  703. tool_call_id=tool_call_id or None,
  704. metadata={
  705. "task_id": effective_task_id or "",
  706. "turn_id": getattr(agent, "_current_turn_id", "") or "",
  707. "api_request_id": getattr(agent, "_current_api_request_id", "") or "",
  708. "tool_call_id": tool_call_id or "",
  709. },
  710. )
  711. return state
  712. # Sequential wait-loop poll cadence: /stop lands within ~1s even if the tool never polls is_interrupted().
  713. _SEQUENTIAL_INTERRUPT_POLL_SECONDS = 1.0
  714. def _resolve_sequential_tool_timeout() -> float | None:
  715. """Deadline for one sequential call: ``timeouts.tools.sequential_call``, else the
  716. concurrent batch deadline so the two paths can't drift; ``0``/negative disables.
  717. Deliberately NOT ``agent.deadline.run_bounded_sync``: both executors extend the
  718. deadline while an approval prompt is open, which a fixed deadline can't express."""
  719. from agent.deadline import resolve_timeout
  720. return resolve_timeout("tools.sequential_call", default=_resolve_concurrent_tool_timeout())
  721. # Tools whose call blocks on a long-running operation that supervises its own liveness: no generic
  722. # sequential deadline. ``delegate_task`` in a nested orchestrator blocks for the whole batch by design
  723. # (children carry heartbeats, the stale monitor, and ``delegation.child_timeout_seconds``); under the
  724. # 420 s deadline every real batch "timed out" while its children ran on as orphans, and the orchestrator
  725. # spent the following hours polling transcripts (measured: 332 timeouts, ~$4k of orchestrator turns in
  726. # one run).
  727. # ``manage_connections`` waits on the connection operation's own deadline; the generic deadline
  728. # would return tool_timeout while its approval card is still open.
  729. _SEQUENTIAL_DEADLINE_EXEMPT_TOOLS = frozenset({"delegate_task", "manage_connections"})
  730. def _abandoned_sequential_result(agent, ref: _ToolCallRef, message: str, result_cls, **outcome) -> _ManagedToolResult:
  731. """Emit the terminal post_tool_call for a worker the sequential runner gave up on
  732. (timeout / interrupt) and wrap ``message`` in its marker ``result_cls``."""
  733. ref.emit_post(agent, message, **outcome)
  734. return _ManagedToolResult(result=result_cls(message), args=ref.args, middleware_trace=ref.trace, blocked=False, dispatched=True)
  735. def _poll_sequential_future(agent, future, function_name: str, deadline: float | None, started: float, authorization_gate) -> tuple[str, Any]:
  736. """Wait for the worker in interrupt-poll slices, extending the deadline by human approval
  737. wait; returns ``("done", result)``, ``("timeout", None)`` or ``("interrupted", None)``.
  738. A disabled deadline still polls: this loop is what makes a non-cooperative tool
  739. interruptible, so no deadline must not mean no interrupt checks."""
  740. _last_heartbeat = 0
  741. while True:
  742. wait_slice = _SEQUENTIAL_INTERRUPT_POLL_SECONDS
  743. if deadline is not None:
  744. remaining = deadline + authorization_gate.excluded_seconds() - time.monotonic()
  745. if remaining <= 0:
  746. return "timeout", None
  747. wait_slice = min(wait_slice, remaining)
  748. try:
  749. return "done", future.result(timeout=wait_slice)
  750. except concurrent.futures.TimeoutError:
  751. # Aliases builtin TimeoutError (3.11+): also fires when the TOOL WORKER died with one (#63892).
  752. # A settled future never unsettles — re-waiting spun until the deadline (forever if None); propagate.
  753. if future.done():
  754. return "done", future.result()
  755. if agent._interrupt_requested:
  756. return "interrupted", None
  757. elapsed = int(time.monotonic() - started)
  758. if elapsed - _last_heartbeat >= 30:
  759. _last_heartbeat = elapsed
  760. agent._touch_activity(f"sequential tool running ({elapsed}s): {function_name}")
  761. def _run_sequential_tool_execution_middleware(
  762. agent,
  763. *,
  764. function_name: str,
  765. function_args: dict,
  766. effective_task_id: str,
  767. tool_call_id: str,
  768. execute,
  769. scope_block: str | None = None,
  770. display_index: int | None = None,
  771. middleware_trace: list[dict[str, Any]] | None = None,
  772. ) -> _ManagedToolResult:
  773. """Run one sequential call on a worker thread under the concurrent executor's deadline.
  774. Interactive tools (``clarify``) own their wait via ``agent.clarify_timeout``; the
  775. generic deadline would report ``tool_timeout`` while the prompt is still live. They
  776. are ``_NEVER_PARALLEL_TOOLS`` and run inline below, before any deadline is armed, so
  777. they need no ``_SEQUENTIAL_DEADLINE_EXEMPT_TOOLS`` entry."""
  778. from agent.tool_dispatch_helpers import _extend_tool_timeout
  779. timeout_s = None if function_name in _SEQUENTIAL_DEADLINE_EXEMPT_TOOLS else _resolve_sequential_tool_timeout()
  780. timeout_s = _extend_tool_timeout(function_name, timeout_s)
  781. ref = _ToolCallRef(function_name, function_args, effective_task_id, tool_call_id, middleware_trace)
  782. kwargs = dict(ref.middleware_kwargs(), execute=execute, scope_block=scope_block, display_index=display_index)
  783. from agent.terminal_approval_batch import take_prepared_call
  784. prepared = take_prepared_call(tool_call_id)
  785. if prepared is not None:
  786. authorization_gate = prepared.batch.authorization_gate
  787. executor = prepared.batch.executor
  788. worker_tid = prepared.tids
  789. future = prepared.future
  790. else:
  791. authorization_gate = None
  792. if function_name in _NEVER_PARALLEL_TOOLS:
  793. return _run_agent_tool_execution_middleware(agent, **kwargs)
  794. from tools.daemon_pool import DaemonThreadPoolExecutor
  795. if prepared is None:
  796. authorization_gate = _ConcurrentToolAuthorizationGate()
  797. worker_tid: list[int] = []
  798. def _run() -> _ManagedToolResult:
  799. with _registered_tool_worker(agent) as tid:
  800. worker_tid.append(tid)
  801. return _run_agent_tool_execution_middleware(agent, authorization_gate=authorization_gate, **kwargs)
  802. if ref.trace is None:
  803. ref.trace = []
  804. if prepared is None:
  805. executor = DaemonThreadPoolExecutor(max_workers=1)
  806. future = executor.submit(propagate_context_to_thread(_run))
  807. deadline = time.monotonic() + timeout_s if timeout_s is not None else None
  808. started = time.monotonic()
  809. abandoned = False
  810. try:
  811. state, result = _poll_sequential_future(agent, future, function_name, deadline, started, authorization_gate)
  812. if state == "done":
  813. return result
  814. if state == "interrupted":
  815. # interrupt() already fanned out to tracked tids, but this worker may have
  816. # registered after that ran; then 3s grace (mirrors the concurrent path).
  817. _interrupt_worker_tids(agent, worker_tid, reason=getattr(agent, "_tool_interrupt_reason", None))
  818. concurrent.futures.wait([future], timeout=3.0)
  819. if future.done() and not future.cancelled():
  820. return future.result()
  821. interrupt_reason = getattr(agent, "_tool_interrupt_reason", None) or "interrupt requested"
  822. message = f"[Tool execution cancelled — {function_name} was abandoned: {interrupt_reason}]"
  823. logger.info(
  824. "sequential tool %s abandoned due to %s (%.1fs elapsed)",
  825. function_name, interrupt_reason, time.monotonic() - started,
  826. )
  827. result_cls, outcome = _ToolCancelledResult, dict(
  828. duration_ms=int((time.monotonic() - started) * 1000), status="cancelled",
  829. error_type="tool_interrupted", error_message=f"Tool execution cancelled: {interrupt_reason}",
  830. )
  831. else:
  832. assert timeout_s is not None # only reachable when a deadline exists
  833. message = f"Error executing tool '{function_name}': timed out after {timeout_s:.1f}s"
  834. logger.warning("sequential tool %s timed out after %.1fs", function_name, timeout_s)
  835. result_cls, outcome = _ToolTimeoutResult, dict(
  836. duration_ms=int(timeout_s * 1000), status="timeout", error_type="tool_timeout", error_message=message,
  837. )
  838. abandoned = True
  839. if prepared is not None:
  840. # A timed-out shell may still be unwinding. Never release a later
  841. # prepared command into overlapping execution.
  842. prepared.batch.close()
  843. if state == "timeout":
  844. # Label the abort as the batch guard's own, not a user stop (#130207). No message:
  845. # ``_interrupt_message`` is what gateway/CLI re-queue as the user's next turn. On the
  846. # interrupted branch the stop is already published; re-interrupting would rebook it
  847. # and null the user's queued message and redirect.
  848. agent.interrupt(tool_reason="terminal batch timeout")
  849. future.cancel()
  850. if state == "timeout":
  851. _interrupt_worker_tids(agent, worker_tid)
  852. return _abandoned_sequential_result(agent, ref, message, result_cls, **outcome)
  853. finally:
  854. # Never join a wedged worker (daemon pool also keeps it out of the atexit join).
  855. if prepared is None:
  856. executor.shutdown(wait=not abandoned, cancel_futures=abandoned)
  857. def _safe_callback(callback, label: str, *args, **kwargs) -> None:
  858. """Invoke a UI/bridge callback if set; a failing callback is logged, never fatal."""
  859. if not callback:
  860. return
  861. try:
  862. callback(*args, **kwargs)
  863. except Exception as callback_error:
  864. logging.debug("%s callback error: %s", label, callback_error)
  865. def _begin_tool_execution(agent, ref: _ToolCallRef, display_index: int | None) -> None:
  866. """Run user-visible and checkpoint preflight on final tool arguments."""
  867. function_name, function_args, effective_task_id, tool_call_id = ref.name, ref.args, ref.task_id, ref.call_id
  868. display_args = _redact_tool_args_for_display(function_name, function_args) or function_args
  869. if _tool_progress_enabled(agent):
  870. prefix = f"Tool {display_index}" if display_index is not None else "Tool"
  871. if agent.verbose_logging:
  872. print(f" 📞 {prefix}: {function_name}({list(display_args.keys())})")
  873. print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False)))
  874. else:
  875. print(f" 📞 {prefix}: {function_name}({list(function_args.keys())}) - {_preview(json.dumps(display_args, ensure_ascii=False), agent.log_prefix_chars)}")
  876. agent._current_tool = function_name
  877. agent._touch_activity(f"executing tool: {function_name}")
  878. _set_worker_activity_callback(agent)
  879. if agent.tool_progress_callback:
  880. try:
  881. preview = _build_tool_preview(function_name, display_args)
  882. except Exception as callback_error:
  883. logging.debug("Tool progress callback error: %s", callback_error)
  884. else:
  885. _safe_callback(agent.tool_progress_callback, "Tool progress", "tool.started", function_name, preview, display_args,
  886. **({"tool_call_id": tool_call_id} if getattr(agent, "_yaovi_embedded", False) else {}))
  887. _safe_callback(agent.tool_start_callback, "Tool start", tool_call_id, function_name, display_args)
  888. if not agent._checkpoint_mgr.enabled:
  889. return
  890. with contextlib.suppress(Exception):
  891. if function_name in {"write_file", "patch"}:
  892. _ensure_file_checkpoint(agent, function_name, function_args, effective_task_id)
  893. elif function_name == "terminal":
  894. command = function_args.get("command", "")
  895. if _is_destructive_command(command):
  896. from tools.file_tools_paths import container_backend_for_task
  897. if container_backend_for_task(effective_task_id or "default") is None:
  898. from agent.runtime_cwd import scope_terminal_cwd
  899. cwd = function_args.get("workdir") or scope_terminal_cwd() or os.getcwd()
  900. agent._checkpoint_mgr.ensure_checkpoint(cwd, f"before terminal: {command[:60]}")
  901. def _emit_tool_complete_and_risk(agent, ref: _ToolCallRef, result, risk_metadata, blocked: bool) -> None:
  902. """Fire ``tool_complete_callback`` (unless blocked) then the ``tool.output_risk`` projection."""
  903. if not blocked and agent.tool_complete_callback:
  904. try:
  905. display_args = _redact_tool_args_for_display(ref.name, ref.args) or ref.args
  906. except Exception as cb_err:
  907. logging.debug("Tool complete callback error: %s", cb_err)
  908. else:
  909. _safe_callback(agent.tool_complete_callback, "Tool complete", ref.call_id, ref.name, display_args, result)
  910. if risk_metadata is not None and risk_metadata.get("risk") != "low":
  911. _safe_callback(
  912. agent.tool_progress_callback, "Tool output risk",
  913. "tool.output_risk", ref.name, None, None, tool_call_id=ref.call_id, risk_metadata=risk_metadata,
  914. )
  915. def _commit_tool_result(
  916. agent,
  917. messages: list,
  918. ref: _ToolCallRef,
  919. function_result,
  920. *,
  921. budget: BudgetConfig,
  922. tool_duration: float,
  923. is_error: bool,
  924. blocked: bool,
  925. effect_disposition,
  926. observed: bool = False,
  927. data_use: frozenset[str] = frozenset(),
  928. error_preview: Callable[[Any], Any] = lambda result: result,
  929. success_log_chars: Optional[int] = None,
  930. verbose_text: Callable[[Any], Any] = lambda result: result,
  931. ):
  932. """Observe (``observed`` results only) and log the outcome; mark the tool done; persist/
  933. spill, hint, wrap and append the result; flush the session DB; project ``tool.completed``.
  934. Blocked calls never ran, so they are neither guardrail-observed nor fed to the file-
  935. mutation verifier; ``success_log_chars`` (sequential path) also logs the completion line.
  936. Returns ``(persisted_result, display_result, risk_metadata)`` (``display_result`` =
  937. pre-persist content for UI previews) or ``None`` when the flush failed (stop the batch).
  938. """
  939. function_name, function_args, tool_call_id, effective_task_id = ref.name, ref.args, ref.call_id, ref.task_id
  940. from hermes_cli.observability.shared_metrics_harness import observe_tool_outcome
  941. observe_tool_outcome(agent, function_name, is_error)
  942. if not (blocked or is_error):
  943. function_result = observe_onboarding_result(agent, function_name, function_args, function_result)
  944. if observed:
  945. if not blocked:
  946. function_result = agent._append_guardrail_observation(
  947. function_name, function_args, function_result, failed=is_error, tool_call_id=tool_call_id,
  948. )
  949. if is_error:
  950. logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, error_preview(function_result))
  951. elif success_log_chars is not None:
  952. logger.info("tool %s completed (%.2fs, %d chars)", function_name, tool_duration, success_log_chars)
  953. if not blocked:
  954. try:
  955. agent._record_file_mutation_result(
  956. function_name, function_args, function_result, is_error, task_id=effective_task_id,
  957. )
  958. except Exception as _ver_err:
  959. logging.debug("file-mutation verifier record failed: %s", _ver_err)
  960. if agent.verbose_logging:
  961. logging.debug("Tool %s completed in %.2fs", function_name, tool_duration)
  962. _log_result = verbose_text(function_result)
  963. logging.debug("Tool result (%d chars): %s", len(_log_result), _log_result)
  964. agent._current_tool = None
  965. _status_suffix = " (error)" if is_error else ""
  966. agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}")
  967. persisted_result = function_result
  968. if _is_multimodal_tool_result(persisted_result):
  969. persisted_result = _persist_multimodal_text_parts(
  970. persisted_result, function_name, tool_call_id, get_active_env(effective_task_id), budget,
  971. )
  972. else:
  973. persisted_result = maybe_persist_tool_result(
  974. content=persisted_result,
  975. tool_name=function_name,
  976. tool_use_id=tool_call_id,
  977. env=get_active_env(effective_task_id),
  978. config=budget,
  979. )
  980. _record_persisted_path_for_stub(agent, tool_call_id, persisted_result)
  981. note_tool_result(agent, function_name, tool_call_id, function_result, persisted_result)
  982. subdir_hints = agent._subdirectory_hints.check_tool_call(function_name, function_args)
  983. if subdir_hints:
  984. if _is_multimodal_tool_result(persisted_result):
  985. # Hint goes on the text summary part so the model still sees it; image blocks untouched.
  986. _append_subdir_hint_to_multimodal(persisted_result, subdir_hints)
  987. else:
  988. persisted_result += subdir_hints
  989. # Multimodal dicts become an OpenAI-style content list; text-only servers get a
  990. # string-safe fallback so a rejected image result never poisons history.
  991. _tool_content = agent._tool_result_content_for_active_model(function_name, persisted_result)
  992. tool_message = make_tool_result_message(function_name, _tool_content, tool_call_id, effect_disposition=effect_disposition)
  993. # A plugin that went live during this turn, and late LSP verdicts on this session's earlier edits,
  994. # are told on this new row (outside any untrusted-data wrapper).
  995. lsp_deferred.forward_to_agent(agent)
  996. plugin_live_notes.attach_to_tool_message(agent, tool_message)
  997. # Prepare presentation data before the append. The emitting completion callback
  998. # stays below the durability fence; raw tool/model content remains unchanged.
  999. prepare_metadata = getattr(agent, "tool_result_metadata_callback", None)
  1000. if not blocked and prepare_metadata:
  1001. try:
  1002. display_args = _redact_tool_args_for_display(function_name, function_args) or function_args
  1003. metadata = prepare_metadata(tool_call_id, function_name, display_args, function_result)
  1004. if metadata:
  1005. tool_message["display_metadata"] = metadata
  1006. except Exception as callback_error:
  1007. logging.debug("Tool result metadata callback error: %s", callback_error)
  1008. stamp_data_use(tool_message, sorted(data_use))
  1009. messages.append(tool_message)
  1010. if not _flush_session_db_after_tool_progress(agent, messages, stage=f"tool result {function_name}"):
  1011. return None
  1012. if not blocked:
  1013. # ``tool.completed`` projects AFTER the canonical append + flush so resume can
  1014. # reconstruct the result even if the UI bridge dies mid-projection.
  1015. _safe_callback(
  1016. agent.tool_progress_callback, "Tool progress",
  1017. "tool.completed", function_name, None, None, duration=tool_duration, is_error=is_error, result=function_result,
  1018. **({"tool_call_id": tool_call_id} if getattr(agent, "_yaovi_embedded", False) else {}),
  1019. )
  1020. if isinstance(function_result, str) and len(function_result) >= _LARGE_TOOL_RESULT_TRIM_CHARS:
  1021. agent._trim_after_tool_batch = True
  1022. return persisted_result, function_result, tool_message.get("_tool_output_risk")
  1023. def _persist_multimodal_text_parts(result: dict, tool_name: str, tool_call_id: str, env, budget: BudgetConfig) -> dict:
  1024. """Spill oversized TEXT parts of a multimodal envelope through the same persistence policy as
  1025. string results (#95429). A ``browser_exec`` call that captured a screenshot bakes its full
  1026. stdout into the envelope's text part, which used to bypass ``maybe_persist_tool_result``
  1027. entirely and ride every later request inline. Image parts are left untouched (their size is
  1028. governed by the vision embed budget); a fresh dict is returned so history is never mutated."""
  1029. parts = result.get("content") or []
  1030. bounded_parts, first_replacement, spilled = [], None, 0
  1031. for part in parts:
  1032. text = part.get("text") if isinstance(part, dict) and part.get("type") == "text" else None
  1033. if isinstance(text, str):
  1034. # One spill file per part: a second oversized part under the same id would overwrite the first's file.
  1035. part_id = tool_call_id if not spilled else f"{tool_call_id}_part{spilled}"
  1036. replaced = maybe_persist_tool_result(content=text, tool_name=tool_name, tool_use_id=part_id,
  1037. env=env, config=budget)
  1038. if replaced != text:
  1039. part = {**part, "text": replaced}
  1040. first_replacement = first_replacement or replaced
  1041. spilled += 1
  1042. bounded_parts.append(part)
  1043. if first_replacement is None:
  1044. return result
  1045. bounded = {**result, "content": bounded_parts}
  1046. summary = bounded.get("text_summary")
  1047. # The summary is a subset of the (already spilled) part text: reuse that bounded reference instead
  1048. # of a second persist under the same id, which would overwrite the spill file with the summary.
  1049. if isinstance(summary, str) and len(summary) > budget.resolve_threshold(tool_name):
  1050. bounded["text_summary"] = first_replacement
  1051. return bounded
  1052. def _finalize_tool_batch(agent, messages: list, effective_task_id: str, num_tools: int, budget: BudgetConfig) -> None:
  1053. """Per-turn aggregate budget enforcement, then /steer injection — in that order, so the
  1054. steer marker is never truncated/discarded when enforcement replaces a result."""
  1055. if num_tools <= 0:
  1056. return
  1057. batch = messages[-num_tools:]
  1058. contents_before = [message.get("content") for message in batch]
  1059. enforce_turn_budget(batch, env=get_active_env(effective_task_id), config=budget)
  1060. plugin_live_notes.keep_after_turn_budget(agent, batch)
  1061. sync_rewritten_tool_rows(agent, batch, contents_before)
  1062. record_tool_batch(agent, batch, contents_before)
  1063. agent._apply_pending_steer_to_tool_results(messages, num_tools)
  1064. def _tool_progress_enabled(agent) -> bool:
  1065. return not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off"
  1066. def _preview(text: str, limit: int) -> str:
  1067. return text[:limit] + "..." if len(text) > limit else text
  1068. def _print_tool_completed(agent, index: int, tool_duration: float, result) -> None:
  1069. """Non-quiet ``✅ Tool N completed`` line (full result under verbose logging)."""
  1070. if agent.verbose_logging:
  1071. print(f" ✅ Tool {index} completed in {tool_duration:.2f}s")
  1072. print(agent._wrap_verbose("Result: ", result))
  1073. else:
  1074. print(f" ✅ Tool {index} completed in {tool_duration:.2f}s - {_preview(result if isinstance(result, str) else str(result), agent.log_prefix_chars)}")
  1075. # ── Concurrent batch machinery ──────────────────────────────────────────────
  1076. @dataclass
  1077. class _ToolOutcome:
  1078. """One finished worker slot of a concurrent batch (``ref`` holds the final name/args/trace)."""
  1079. ref: _ToolCallRef
  1080. result: Any
  1081. duration: float
  1082. is_error: bool
  1083. blocked: bool
  1084. data_use: frozenset[str] = frozenset()
  1085. def _start_order_gate_timeout(batch_timeout: float | None) -> float:
  1086. """The gate bound must sit UNDER the batch deadline, else parked workers are falsely
  1087. reported timed out without starting. A disabled deadline keeps the stock bound."""
  1088. if batch_timeout is None:
  1089. return _START_ORDER_GATE_TIMEOUT_S
  1090. return min(_START_ORDER_GATE_TIMEOUT_S, batch_timeout / 2)
  1091. class _StartOrderGate:
  1092. """Serialize worker dispatch by submit order (prompts appear in call order); ``abandon()``
  1093. releases every parked worker so none dispatches a tool the turn already gave up on."""
  1094. def __init__(self, timeout: float) -> None:
  1095. self._condition = threading.Condition()
  1096. self._next_order = 0
  1097. self._timeout = timeout
  1098. self.abandoned = threading.Event()
  1099. def abandon(self) -> None:
  1100. self.abandoned.set()
  1101. with self._condition:
  1102. self._condition.notify_all()
  1103. def begin_in_order(self, order: int, callback=None, *, tool_name: str = "") -> bool:
  1104. """Wait for ``order``, run ``callback``, advance. Returns False if abandoned."""
  1105. with self._condition:
  1106. # Bounded wait so one wedged dispatch can't starve later-ordered workers; on
  1107. # expiry proceed out of order (interleaved prompts beat starvation). ``>=`` (not
  1108. # ``==``) releases every skipped worker at once; abandoned short-circuits.
  1109. in_order = self._condition.wait_for(
  1110. lambda: self._next_order >= order or self.abandoned.is_set(), timeout=self._timeout,
  1111. )
  1112. if self.abandoned.is_set():
  1113. return False # the turn already synthesized this result; don't advance
  1114. if not in_order:
  1115. logger.warning(
  1116. "start-order gate timed out for %s (order=%d next=%d); proceeding out of order",
  1117. tool_name or "tool", order, self._next_order,
  1118. )
  1119. try:
  1120. if callback is not None:
  1121. callback()
  1122. finally:
  1123. self._next_order = max(self._next_order, order + 1)
  1124. self._condition.notify_all()
  1125. return True
  1126. class _WorkerStartOnce:
  1127. """One worker's handle on the start-order gate: advances at most once, raising
  1128. ``_BatchAbandoned`` (instead of dispatching late) when the batch was abandoned."""
  1129. def __init__(self, gate: _StartOrderGate, order: int, tool_name: str) -> None:
  1130. self._gate, self._order, self._tool_name, self._advanced = gate, order, tool_name, False
  1131. def advance(self, callback=None) -> None:
  1132. if self._advanced:
  1133. return
  1134. self._advanced = True
  1135. if not self._gate.begin_in_order(self._order, callback, tool_name=self._tool_name):
  1136. raise _BatchAbandoned(self._tool_name)
  1137. class _ConcurrentBatch:
  1138. """Shared state of one concurrent tool batch: per-slot results, the start-order and
  1139. authorization gates, and the deadline bookkeeping the wait loop needs."""
  1140. def __init__(self, agent, messages: list, effective_task_id: str, parsed_calls: list[_ParsedCall], timeout_s: float | None) -> None:
  1141. self.agent = agent
  1142. self.messages = messages
  1143. self.effective_task_id = effective_task_id
  1144. self.parsed_calls = parsed_calls
  1145. self.timeout_s = timeout_s
  1146. self.results: list[Optional[_ToolOutcome]] = [None] * len(parsed_calls)
  1147. for i, pc in enumerate(parsed_calls):
  1148. if pc.parse_error is not None:
  1149. self.results[i] = _ToolOutcome(pc.ref(effective_task_id), pc.parse_error, 0.0, True, True)
  1150. self.gate = _StartOrderGate(_start_order_gate_timeout(timeout_s))
  1151. self.authorization_gate = _ConcurrentToolAuthorizationGate()
  1152. self.timed_out_indices: set[int] = set()
  1153. def _dispatch_worker(self, index: int, ref: _ToolCallRef, scope_block, start_gate: _WorkerStartOnce) -> Optional[_ToolOutcome]:
  1154. """Run one call through the middleware and synthesize its slot outcome; ``None`` when
  1155. abandoned at the gate (the main thread already wrote this slot; emitting would
  1156. double-report the tool_call_id)."""
  1157. agent = self.agent
  1158. # Approval/sudo callbacks (thread-local) and the agent turn's ContextVars are propagated by
  1159. # propagate_context_to_thread() at the submit site below (GHSA-qg5c-hvr5-hjgr, #13617).
  1160. start = time.time()
  1161. blocked = dispatched = False
  1162. data_use: frozenset[str] = frozenset()
  1163. try:
  1164. managed = _run_agent_tool_execution_middleware(
  1165. agent,
  1166. **ref.middleware_kwargs(),
  1167. execute=lambda next_args: agent._invoke_tool(
  1168. ref.name, next_args, ref.task_id, ref.call_id,
  1169. messages=self.messages,
  1170. pre_tool_block_checked=True,
  1171. skip_tool_request_middleware=True,
  1172. skip_tool_execution_middleware=True,
  1173. tool_request_middleware_trace=list(ref.trace),
  1174. ),
  1175. scope_block=scope_block,
  1176. display_index=index + 1,
  1177. begin_execution=start_gate.advance,
  1178. authorization_gate=self.authorization_gate,
  1179. )
  1180. result, ref.args, ref.trace = managed.result, managed.args, managed.middleware_trace
  1181. blocked, dispatched, data_use = managed.blocked, managed.dispatched, managed.data_use
  1182. except _BatchAbandoned:
  1183. logger.info("tool %s abandoned at start-order gate; skipping dispatch", ref.name)
  1184. return None
  1185. except KeyboardInterrupt:
  1186. with contextlib.suppress(Exception):
  1187. agent.interrupt("keyboard interrupt", tool_reason=_REASON_USER_INTERRUPT)
  1188. result = ref.emit_cancelled(agent, start)
  1189. duration = time.time() - start
  1190. logger.info("tool %s cancelled (%.2fs)", ref.name, duration)
  1191. return _ToolOutcome(ref, result, duration, True, False)
  1192. except Exception as tool_error:
  1193. result = f"Error executing tool '{ref.name}': {tool_error}"
  1194. logger.error("_invoke_tool raised for %s: %s", ref.name, tool_error, exc_info=True)
  1195. duration = time.time() - start
  1196. if not blocked and not dispatched:
  1197. ref.emit_post(agent, result, duration_ms=int(duration * 1000))
  1198. is_error, _ = _detect_tool_failure(ref.name, result)
  1199. if is_error:
  1200. logger.info("tool %s failed (%.2fs): %s", ref.name, duration, str(result)[:200])
  1201. else:
  1202. result_chars = len(result) if isinstance(result, str) else len(str(result))
  1203. logger.info(
  1204. "tool %s completed (%.2fs, %d chars)", ref.name, duration, result_chars
  1205. )
  1206. return _ToolOutcome(ref, result, duration, is_error, blocked, data_use)
  1207. def run_worker(self, index: int, start_order: int) -> None:
  1208. """Worker function executed in a thread."""
  1209. agent, pc = self.agent, self.parsed_calls[index]
  1210. with _registered_tool_worker(agent) as _worker_tid:
  1211. # An interrupt may have fanned out before our registration; apply it to our tid.
  1212. if agent._interrupt_requested:
  1213. _interrupt_worker_tids(agent, [_worker_tid], reason=getattr(agent, "_tool_interrupt_reason", None))
  1214. _set_worker_activity_callback(agent)
  1215. start_gate = _WorkerStartOnce(self.gate, start_order, pc.name)
  1216. try:
  1217. outcome = self._dispatch_worker(index, pc.ref(self.effective_task_id), pc.scope_block, start_gate)
  1218. if outcome is not None:
  1219. self.results[index] = outcome
  1220. finally:
  1221. with contextlib.suppress(_BatchAbandoned):
  1222. start_gate.advance() # keep later-ordered workers moving
  1223. def submit_all(self, executor, runnable: list[int]) -> tuple[list, dict]:
  1224. """Submit every runnable slot; on interpreter shutdown, synthesize error results
  1225. for the unsubmitted remainder instead of raising. ``propagate_context_to_thread``
  1226. carries turn ContextVars and thread-local approval/sudo callbacks into the worker."""
  1227. futures = []
  1228. future_to_index = {}
  1229. for submit_index, i in enumerate(runnable):
  1230. try:
  1231. f = executor.submit(propagate_context_to_thread(self.run_worker), i, submit_index)
  1232. except RuntimeError as submit_error:
  1233. if not _is_interpreter_shutdown_submit_error(submit_error):
  1234. raise
  1235. skipped = runnable[submit_index:]
  1236. logger.warning(
  1237. "interpreter shutdown while scheduling concurrent tools; skipping %d unsubmitted tool(s)", len(skipped),
  1238. )
  1239. for skipped_i in skipped:
  1240. ref = self.parsed_calls[skipped_i].ref(self.effective_task_id)
  1241. if self.results[skipped_i] is None:
  1242. result = f"Error executing tool '{ref.name}': Python interpreter is shutting down; tool was not started"
  1243. self.results[skipped_i] = _ToolOutcome(ref, result, 0.0, True, False)
  1244. break
  1245. futures.append(f)
  1246. future_to_index[f] = i
  1247. return futures, future_to_index
  1248. def _running_names(self, not_done, future_to_index) -> list[str]:
  1249. return [self.parsed_calls[future_to_index[f]].name for f in not_done if f in future_to_index]
  1250. def await_completion(self, futures, future_to_index, deadline: float | None) -> bool:
  1251. """Wait with periodic heartbeats and interrupt checks; True when the batch was
  1252. abandoned (deadline or interrupt) and the executor must not join its workers."""
  1253. agent = self.agent
  1254. _conc_start = time.time()
  1255. while True:
  1256. wait_timeout = 5.0
  1257. if deadline is not None:
  1258. remaining = deadline + self.authorization_gate.excluded_seconds() - time.monotonic()
  1259. if remaining <= 0:
  1260. not_done = {f for f in futures if not f.done()}
  1261. else:
  1262. wait_timeout = min(wait_timeout, remaining)
  1263. if deadline is None or remaining > 0:
  1264. _done, not_done = concurrent.futures.wait(futures, timeout=wait_timeout)
  1265. if not not_done:
  1266. return False
  1267. timed_out = deadline is not None and time.monotonic() >= deadline + self.authorization_gate.excluded_seconds()
  1268. if timed_out:
  1269. self.timed_out_indices = {future_to_index[f] for f in not_done if f in future_to_index}
  1270. logger.warning(
  1271. "concurrent tool batch timed out after %.1fs; %d tool(s) still running: %s",
  1272. self.timeout_s,
  1273. len(self.timed_out_indices),
  1274. ", ".join(self._running_names(not_done, future_to_index)[:5]),
  1275. )
  1276. elif agent._interrupt_requested:
  1277. # Tools without interrupt checks (web_search, read_file) run to
  1278. # completion; cancel unstarted futures so we don't block on them.
  1279. agent._vprint(
  1280. f"{agent.log_prefix}⚡ Interrupt: cancelling {len(not_done)} pending concurrent tool(s)",
  1281. force=True,
  1282. )
  1283. else:
  1284. _conc_elapsed = int(time.time() - _conc_start)
  1285. # Heartbeat every ~30s (6 × 5s poll intervals)
  1286. if _conc_elapsed > 0 and _conc_elapsed % 30 < 6:
  1287. _still_running = self._running_names(not_done, future_to_index)
  1288. agent._touch_activity(
  1289. f"concurrent tools running ({_conc_elapsed}s, "
  1290. f"{len(not_done)} remaining: {', '.join(_still_running[:3])})"
  1291. )
  1292. continue
  1293. for f in not_done:
  1294. f.cancel()
  1295. # Release gate-parked workers BEFORE interrupt fan-out so none later
  1296. # dispatches a tool the turn already reported as timed out / interrupted.
  1297. self.gate.abandon()
  1298. if timed_out:
  1299. with agent._tool_worker_threads_lock:
  1300. worker_tids = list(agent._tool_worker_threads)
  1301. _interrupt_worker_tids(agent, worker_tids)
  1302. else:
  1303. # Give running tools a moment to notice the per-thread interrupt and exit gracefully.
  1304. concurrent.futures.wait(not_done, timeout=3.0)
  1305. return True
  1306. def run(self) -> None:
  1307. """Dispatch the runnable calls on a daemon pool and wait for the batch."""
  1308. runnable = [i for i, pc in enumerate(self.parsed_calls) if pc.parse_error is None]
  1309. if not runnable:
  1310. return
  1311. deadline = time.monotonic() + self.timeout_s if self.timeout_s is not None else None
  1312. max_workers = _max_workers_for_tool_batch([(i, None, self.parsed_calls[i].name) for i in runnable])
  1313. # Daemon workers: the stdlib pool's atexit join would let one wedged tool block exit.
  1314. from tools.daemon_pool import DaemonThreadPoolExecutor
  1315. executor = DaemonThreadPoolExecutor(max_workers=max_workers)
  1316. abandon_executor = False
  1317. try:
  1318. futures, future_to_index = self.submit_all(executor, runnable)
  1319. abandon_executor = self.await_completion(futures, future_to_index, deadline)
  1320. finally:
  1321. # Every abandoning exit releases gate-parked workers and leaves wedged threads
  1322. # detached rather than joining them; normal completion joins.
  1323. if abandon_executor:
  1324. self.gate.abandon()
  1325. executor.shutdown(wait=not abandon_executor, cancel_futures=abandon_executor)
  1326. def _unfinished_tool_result(agent, ref: _ToolCallRef, *, timed_out: bool, timeout_s: float | None) -> tuple[str, float, Optional[str]]:
  1327. """Synthesize the result for a slot no worker filled (deadline, interrupt, or a thread
  1328. that never returned), emit its terminal post_tool_call, and return
  1329. ``(function_result, tool_duration, effect_disposition)``."""
  1330. if timed_out:
  1331. suffix = f"{timeout_s:.1f}s" if timeout_s is not None else "the configured timeout"
  1332. function_result = f"Error executing tool '{ref.name}': timed out after {suffix}"
  1333. outcome = dict(duration_ms=int((timeout_s or 0.0) * 1000), status="timeout", error_type="tool_timeout", error_message=function_result)
  1334. tool_duration, effect_disposition = float(timeout_s or 0.0), "unknown"
  1335. elif agent._interrupt_requested:
  1336. why = interrupt_skip_wording(agent)
  1337. function_result = f"[Tool execution cancelled — {ref.name} was skipped. {why}]"
  1338. outcome = dict(status="cancelled", error_type="keyboard_interrupt", error_message=f"Tool execution cancelled. {why}")
  1339. tool_duration, effect_disposition = 0.0, None
  1340. else:
  1341. function_result = f"Error executing tool '{ref.name}': thread did not return a result"
  1342. outcome = dict(status="error", error_type="thread_missing_result", error_message=function_result)
  1343. tool_duration, effect_disposition = 0.0, None
  1344. ref.emit_post(agent, function_result, **outcome)
  1345. return function_result, tool_duration, effect_disposition
  1346. def _append_batch_results(agent, messages: list, effective_task_id: str, batch: _ConcurrentBatch, budget: BudgetConfig) -> bool:
  1347. """Append every slot's result in original call order; returns False at the first
  1348. failed flush (the caller must stop the batch)."""
  1349. for i, pc in enumerate(batch.parsed_calls):
  1350. r = batch.results[i]
  1351. # A worker may finish between the deadline snapshot and this loop;
  1352. # prefer its real result over a fabricated timeout.
  1353. if r is None:
  1354. ref, is_error, blocked = pc.ref(effective_task_id), True, False
  1355. function_result, tool_duration, effect_disposition = _unfinished_tool_result(
  1356. agent, ref, timed_out=i in batch.timed_out_indices, timeout_s=batch.timeout_s,
  1357. )
  1358. else:
  1359. ref, function_result, tool_duration, is_error, blocked = r.ref, r.result, r.duration, r.is_error, r.blocked
  1360. effect_disposition = "none" if blocked else None
  1361. if pc.parse_error is not None:
  1362. ref.emit_invalid_arguments(agent, r.result)
  1363. committed = _commit_tool_result(
  1364. agent, messages, ref, function_result,
  1365. budget=budget, tool_duration=tool_duration, is_error=is_error, blocked=blocked,
  1366. effect_disposition=effect_disposition, observed=r is not None, data_use=r.data_use if r else frozenset(),
  1367. error_preview=lambda res: _multimodal_text_summary(res)[:200],
  1368. )
  1369. if committed is None:
  1370. return False
  1371. _persisted, display_function_result, risk_metadata = committed
  1372. if agent._should_emit_quiet_tool_messages():
  1373. cute_msg = _get_cute_tool_message_impl(ref.name, ref.args, tool_duration, result=display_function_result)
  1374. agent._safe_print(f" {cute_msg}")
  1375. elif _tool_progress_enabled(agent):
  1376. _print_tool_completed(agent, i + 1, tool_duration, _multimodal_text_summary(display_function_result))
  1377. _emit_tool_complete_and_risk(agent, ref, display_function_result, risk_metadata, blocked)
  1378. return True
  1379. def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None:
  1380. """Execute tool calls concurrently; results are appended in original call order.
  1381. ``finalize=False`` skips end-of-batch budget enforcement and /steer injection (the
  1382. segmented dispatcher owns turn-end work)."""
  1383. tool_calls = assistant_message.tool_calls
  1384. num_tools = len(tool_calls)
  1385. _tool_budget = _budget_for_agent(agent) # once per turn, not per result
  1386. if agent._interrupt_requested:
  1387. print(f"{agent.log_prefix}⚡ Interrupt: skipping {num_tools} tool call(s)")
  1388. _append_skipped_tool_results(
  1389. agent, messages, tool_calls, effective_task_id,
  1390. content=f"[Tool execution cancelled — {{name}} was skipped. {interrupt_skip_wording(agent)}]",
  1391. hook_error_type="user_interrupt",
  1392. flush_stage="cancelled tool result",
  1393. stop_on_flush_failure=False,
  1394. )
  1395. return
  1396. parsed_calls = [_parse_tool_call(agent, tc) for tc in tool_calls]
  1397. tool_names_str = ", ".join(pc.name for pc in parsed_calls)
  1398. if _tool_progress_enabled(agent):
  1399. print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}")
  1400. # Resolved before the batch is built so the start-order gate can clamp under the deadline.
  1401. timeout_s = _resolve_concurrent_tool_timeout()
  1402. batch = _ConcurrentBatch(agent, messages, effective_task_id, parsed_calls, timeout_s)
  1403. agent._current_tool = tool_names_str
  1404. agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}")
  1405. spinner = _start_quiet_tool_spinner(agent, "", {}, label=f"⚡ running {num_tools} tools concurrently")
  1406. try:
  1407. batch.run()
  1408. finally:
  1409. if spinner:
  1410. finished = [r for r in batch.results if r is not None]
  1411. spinner.stop(f"⚡ {len(finished)}/{num_tools} tools completed in {sum(r.duration for r in finished):.1f}s total")
  1412. if not _append_batch_results(agent, messages, effective_task_id, batch, _tool_budget):
  1413. return
  1414. if finalize:
  1415. _finalize_tool_batch(agent, messages, effective_task_id, len(parsed_calls), _tool_budget)
  1416. # ── Sequential dispatch ─────────────────────────────────────────────────────
  1417. def _start_quiet_tool_spinner(agent, function_name: str, function_args: dict, *, gate: bool = True, label: Optional[str] = None):
  1418. """Start the quiet-mode kawaii spinner for one tool call, or return None; ``gate=False``
  1419. skips ``_should_start_quiet_spinner`` (context-engine tools always spin)."""
  1420. if not agent._should_emit_quiet_tool_messages() or (gate and not agent._should_start_quiet_spinner()):
  1421. return None
  1422. face = random.choice(KawaiiSpinner.get_waiting_faces())
  1423. if label is None:
  1424. display_args = _redact_tool_args_for_display(function_name, function_args) or function_args
  1425. label = f"{_tool_row_emoji(function_name, display_args)} {_build_tool_label(function_name, display_args) or function_name}"
  1426. spinner = KawaiiSpinner(f"{face} {label}", spinner_type='dots', print_fn=agent._print_fn)
  1427. spinner.start()
  1428. return spinner
  1429. def _finish_quiet_tool_spinner(agent, spinner, function_name: str, function_args: dict, tool_duration: float, result) -> None:
  1430. """Stop the spinner with the cute completion line, or print it when no spinner ran."""
  1431. if spinner or agent._should_emit_quiet_tool_messages():
  1432. cute = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=result)
  1433. spinner.stop(cute) if spinner else agent._vprint(f" {cute}")
  1434. def _delegate_spinner_label(function_args: dict) -> str:
  1435. action = str(function_args.get("action") or "").strip().lower()
  1436. tasks = function_args.get("tasks")
  1437. if action in ("list", "steer", "stop"):
  1438. return f"🔀 subagent {action}"
  1439. if tasks and isinstance(tasks, list):
  1440. return f"🔀 delegating {len(tasks)} tasks · (/agents to monitor)"
  1441. goal_preview = (function_args.get("goal") or "")[:30]
  1442. return f"🔀 {goal_preview} · (/agents to monitor)" if goal_preview else "🔀 delegating · (/agents to monitor)"
  1443. @dataclass
  1444. class _SequentialDispatch:
  1445. """How one sequential call executes: the callable plus its spinner/error policy."""
  1446. execute: Callable[[dict], Any]
  1447. spinner: Any = None
  1448. middleware_trace_arg: Optional[list] = None # forwarded to the middleware runner (registry closure reads it)
  1449. error_result: Optional[Callable[[Exception], str]] = None # None → exceptions propagate (inline/delegate own failures)
  1450. error_log: str = ""
  1451. handles_keyboard_interrupt: bool = False
  1452. is_delegate: bool = False
  1453. finish_spinner: bool = True
  1454. finish_in_finally: bool = True # inline tools print their completion line only on success
  1455. transform_applied: bool = False # True when execute already fired transform_tool_result
  1456. def _resolve_sequential_dispatch(agent, ref: _ToolCallRef, messages: list) -> _SequentialDispatch:
  1457. """Pick the execute callable for one sequential call and start its spinner. Precedence:
  1458. inline agent-level tools, delegate_task, context-engine tools, memory-provider tools,
  1459. then the registry."""
  1460. function_name, function_args, effective_task_id, tool_call_id, middleware_trace = (
  1461. ref.name, ref.args, ref.task_id, ref.call_id, ref.trace,
  1462. )
  1463. if function_name != "delegate_task" and function_name in INLINE_TOOL_EXECUTORS:
  1464. # Agent-level tools that need live AIAgent state; table shared with invoke_tool.
  1465. inline_executor = INLINE_TOOL_EXECUTORS[function_name]
  1466. inline_ctx = InlineToolContext(effective_task_id=effective_task_id, tool_call_id=tool_call_id, messages=messages)
  1467. return _SequentialDispatch(lambda next_args: inline_executor(agent, next_args, inline_ctx), finish_in_finally=False)
  1468. if function_name == "delegate_task":
  1469. spinner = _start_quiet_tool_spinner(agent, function_name, function_args, label=_delegate_spinner_label(function_args))
  1470. agent._delegate_spinner = spinner
  1471. return _SequentialDispatch(agent._dispatch_delegate_task, spinner=spinner, is_delegate=True)
  1472. if agent._context_engine_tool_names and function_name in agent._context_engine_tool_names:
  1473. return _SequentialDispatch(
  1474. execute=lambda next_args: agent.context_compressor.handle_tool_call(function_name, next_args, messages=messages),
  1475. spinner=_start_quiet_tool_spinner(agent, function_name, function_args, gate=False),
  1476. error_result=lambda e: json.dumps({"error": f"Context engine tool '{function_name}' failed: {e}"}),
  1477. error_log="context_engine.handle_tool_call raised for %s: %s",
  1478. )
  1479. if agent._memory_manager and agent._memory_manager.has_tool(function_name):
  1480. # Memory-provider tools (hindsight_retain, honcho_search, ...) are not in the registry.
  1481. return _SequentialDispatch(
  1482. execute=lambda next_args: agent._memory_manager.handle_tool_call(
  1483. function_name, next_args,
  1484. caller_gsk=(getattr(agent, "_gateway_session_key", "") or ""),
  1485. ),
  1486. spinner=_start_quiet_tool_spinner(agent, function_name, function_args),
  1487. error_result=lambda e: json.dumps({"error": f"Memory tool '{function_name}' failed: {e}"}),
  1488. error_log="memory_manager.handle_tool_call raised for %s: %s",
  1489. )
  1490. # Registry tools: post hook is owned by this executor (inner observer suppressed).
  1491. def _execute(next_args: dict) -> Any:
  1492. import model_tools
  1493. with model_tools.suppress_post_tool_call_hook():
  1494. return model_tools.handle_function_call(
  1495. function_name,
  1496. next_args,
  1497. effective_task_id,
  1498. tool_call_id=tool_call_id,
  1499. session_id=agent.session_id or "",
  1500. turn_id=getattr(agent, "_current_turn_id", "") or "",
  1501. api_request_id=getattr(agent, "_current_api_request_id", "") or "",
  1502. enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None,
  1503. skip_pre_tool_call_hook=True,
  1504. skip_tool_request_middleware=True,
  1505. skip_tool_execution_middleware=True,
  1506. tool_request_middleware_trace=list(middleware_trace),
  1507. enabled_toolsets=getattr(agent, "enabled_toolsets", None),
  1508. disabled_toolsets=getattr(agent, "disabled_toolsets", None),
  1509. )
  1510. return _SequentialDispatch(
  1511. execute=_execute,
  1512. spinner=_start_quiet_tool_spinner(agent, function_name, function_args) if agent.quiet_mode else None,
  1513. middleware_trace_arg=middleware_trace,
  1514. error_result=lambda e: f"Error executing tool '{function_name}': {e}",
  1515. error_log="handle_function_call raised for %s: %s",
  1516. handles_keyboard_interrupt=True,
  1517. finish_spinner=bool(agent.quiet_mode),
  1518. transform_applied=True, # handle_function_call fires transform_tool_result itself
  1519. )
  1520. def _skip_remaining_sequential(agent, messages: list, remaining, effective_task_id: str, *, notice: str, **skip_kwargs) -> bool:
  1521. """Announce an interrupt and append one skipped result per unstarted call; False when
  1522. a flush failed (the caller must stop the batch)."""
  1523. agent._vprint(f"{agent.log_prefix}⚡ Interrupt: skipping {len(remaining)} {notice}", force=True)
  1524. return _append_skipped_tool_results(agent, messages, remaining, effective_task_id, **skip_kwargs)
  1525. def _append_invalid_arguments_result(agent, messages: list, ref: _ToolCallRef, parse_error: str) -> bool:
  1526. """Emit + append the parse-error result for a call whose arguments were not a JSON object."""
  1527. ref.emit_invalid_arguments(agent, parse_error)
  1528. messages.append(make_tool_result_message(ref.name, parse_error, ref.call_id))
  1529. return _flush_session_db_after_tool_progress(agent, messages, stage=f"invalid tool arguments {ref.name}")
  1530. def _run_sequential_call(
  1531. agent,
  1532. dispatch: _SequentialDispatch,
  1533. ref: _ToolCallRef,
  1534. *,
  1535. scope_block: Optional[str],
  1536. messages: list,
  1537. remaining_calls,
  1538. display_index: int,
  1539. tool_start_time: float,
  1540. ) -> tuple[_ManagedToolResult, float]:
  1541. """Run one sequential call with its spinner/error policy; returns ``(managed, duration)``.
  1542. KeyboardInterrupt (registry tools only) emits results for THIS and every remaining call
  1543. before re-raising so the tool-call turn keeps matching results (alternation)."""
  1544. _spinner_result = None
  1545. try:
  1546. managed = _run_sequential_tool_execution_middleware(
  1547. agent,
  1548. **dict(ref.middleware_kwargs(), middleware_trace=dispatch.middleware_trace_arg),
  1549. execute=dispatch.execute,
  1550. scope_block=scope_block,
  1551. display_index=display_index,
  1552. )
  1553. ref.args = managed.args
  1554. _spinner_result = managed.result
  1555. except KeyboardInterrupt:
  1556. if not dispatch.handles_keyboard_interrupt:
  1557. raise
  1558. # Publish the user-interrupt reason BEFORE emitting, so the hook sees it (as the concurrent path does).
  1559. with contextlib.suppress(Exception):
  1560. agent.interrupt("keyboard interrupt", tool_reason=_REASON_USER_INTERRUPT)
  1561. _spinner_result = ref.emit_cancelled(agent, tool_start_time)
  1562. _append_skipped_tool_results(
  1563. agent, messages, remaining_calls, ref.task_id,
  1564. content=f"[Tool execution cancelled — {{name}} was skipped. {interrupt_skip_wording(agent)}]",
  1565. )
  1566. raise
  1567. except Exception as tool_error:
  1568. if dispatch.error_result is None:
  1569. raise
  1570. function_result = dispatch.error_result(tool_error)
  1571. logger.error(dispatch.error_log, ref.name, tool_error, exc_info=True)
  1572. managed = _ManagedToolResult(result=function_result, args=ref.args, middleware_trace=ref.trace, blocked=False, dispatched=False)
  1573. finally:
  1574. if dispatch.is_delegate:
  1575. agent._delegate_spinner = None
  1576. tool_duration = time.time() - tool_start_time
  1577. if dispatch.finish_spinner and dispatch.finish_in_finally:
  1578. _finish_quiet_tool_spinner(agent, dispatch.spinner, ref.name, ref.args, tool_duration, _spinner_result)
  1579. if dispatch.finish_spinner and not dispatch.finish_in_finally:
  1580. _finish_quiet_tool_spinner(agent, dispatch.spinner, ref.name, ref.args, tool_duration, _spinner_result)
  1581. return managed, tool_duration
  1582. def _publish_sequential_result(agent, messages: list, ref: _ToolCallRef, managed: _ManagedToolResult, *, tool_duration: float, index: int, budget: BudgetConfig, transform_applied: bool) -> bool:
  1583. """Terminal hook → observe → commit → completion callbacks/print for one sequential
  1584. result; False when the incremental flush failed (the caller must stop the batch)."""
  1585. ref.args, ref.trace, function_result = managed.args, managed.middleware_trace, managed.result
  1586. _execution_timed_out = isinstance(function_result, (_ToolTimeoutResult, _ToolCancelledResult))
  1587. # Inline-dispatched runtime tools never reach handle_function_call, so the
  1588. # executor owns the one terminal post_tool_call per tool_call_id (the inner
  1589. # observer is suppressed); also stops an abandoned timeout worker reporting late.
  1590. # transform_tool_result follows the observer, unless the dispatch already fired it.
  1591. if not managed.blocked and not _execution_timed_out:
  1592. ref.emit_post(agent, function_result, duration_ms=int(tool_duration * 1000))
  1593. if not transform_applied:
  1594. function_result = apply_transform_tool_result(
  1595. agent, function_name=ref.name, function_args=ref.args, result=function_result,
  1596. effective_task_id=ref.task_id, tool_call_id=ref.call_id,
  1597. duration_ms=int(tool_duration * 1000),
  1598. )
  1599. # Classify the result the model will actually see, i.e. after any transform; the
  1600. # registry and concurrent paths both classify post-transform.
  1601. # Multimodal dict results (_multimodal=True) are not sliceable as strings.
  1602. _result_len = len(function_result) if isinstance(function_result, str) else len(str(function_result))
  1603. _is_error_result, _ = _detect_tool_failure(ref.name, function_result)
  1604. committed = _commit_tool_result(
  1605. agent, messages, ref, function_result,
  1606. budget=budget, tool_duration=tool_duration, is_error=_is_error_result, blocked=managed.blocked,
  1607. effect_disposition="unknown" if _execution_timed_out else None, observed=True, data_use=managed.data_use,
  1608. error_preview=lambda res: res[:200] if isinstance(res, str) and not agent.verbose_logging else res,
  1609. success_log_chars=_result_len,
  1610. verbose_text=_multimodal_text_summary,
  1611. )
  1612. if committed is None:
  1613. return False
  1614. function_result, display_function_result, risk_metadata = committed
  1615. # Terminal approval batching (#113158): once this slot's failure is
  1616. # published, the informed consent collected for later slots describes a
  1617. # batch state that no longer holds — flag it so consume_prepared_guard
  1618. # drops their pre-made decisions and the live guard flow re-runs.
  1619. from agent.terminal_approval_batch import mark_batch_outcome
  1620. mark_batch_outcome(_is_error_result or bool(managed.blocked))
  1621. _emit_tool_complete_and_risk(agent, ref, display_function_result, risk_metadata, managed.blocked)
  1622. if _tool_progress_enabled(agent):
  1623. _print_tool_completed(agent, index, tool_duration, function_result)
  1624. return True
  1625. def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None:
  1626. from types import SimpleNamespace
  1627. from agent.terminal_approval_batch import terminal_approval_batch, terminal_approval_runs
  1628. for calls in terminal_approval_runs(agent, assistant_message.tool_calls):
  1629. with terminal_approval_batch(agent, calls, messages, effective_task_id):
  1630. _execute_tool_calls_sequential(agent, SimpleNamespace(tool_calls=calls), messages, effective_task_id, api_call_count, finalize=False)
  1631. if getattr(agent, "_incremental_persistence_failed", False):
  1632. return
  1633. if finalize:
  1634. _finalize_tool_batch(agent, messages, effective_task_id, len(assistant_message.tool_calls), _budget_for_agent(agent))
  1635. def _execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None:
  1636. """Execute tool calls sequentially (single calls or interactive tools). ``finalize=False``
  1637. skips end-of-batch budget enforcement and /steer injection (the segmented dispatcher
  1638. owns turn-end work)."""
  1639. _tool_budget = _budget_for_agent(agent) # once per turn, not per result
  1640. tool_calls = assistant_message.tool_calls
  1641. for i, tool_call in enumerate(tool_calls, 1):
  1642. if getattr(agent, "_incremental_persistence_failed", False):
  1643. return
  1644. # Check interrupt BEFORE each tool so a "stop" during the previous one skips the rest.
  1645. if agent._interrupt_requested:
  1646. if not _skip_remaining_sequential(
  1647. agent, messages, tool_calls[i - 1:], effective_task_id,
  1648. notice="tool call(s)",
  1649. content=f"[Tool execution cancelled — {{name}} was skipped. {interrupt_skip_wording(agent)}]",
  1650. hook_error_type="user_interrupt",
  1651. hook_id=lambda tc: getattr(tc, "id", "") or "",
  1652. flush_stage="cancelled tool result",
  1653. ):
  1654. return
  1655. break
  1656. pc = _parse_tool_call(agent, tool_call, flatten_probe=True)
  1657. ref = pc.ref(effective_task_id)
  1658. if pc.parse_error is not None:
  1659. if not _append_invalid_arguments_result(agent, messages, ref, pc.parse_error):
  1660. return
  1661. continue
  1662. tool_start_time = time.time()
  1663. dispatch = _resolve_sequential_dispatch(agent, ref, messages)
  1664. managed, tool_duration = _run_sequential_call(
  1665. agent, dispatch, ref,
  1666. scope_block=pc.scope_block,
  1667. messages=messages,
  1668. remaining_calls=tool_calls[i - 1:],
  1669. display_index=i,
  1670. tool_start_time=tool_start_time,
  1671. )
  1672. if not _publish_sequential_result(agent, messages, ref, managed, tool_duration=tool_duration, index=i,
  1673. budget=_tool_budget, transform_applied=dispatch.transform_applied):
  1674. return
  1675. if agent._interrupt_requested and i < len(tool_calls):
  1676. if not _skip_remaining_sequential(
  1677. agent, messages, tool_calls[i:], effective_task_id,
  1678. notice="remaining tool call(s)",
  1679. content=f"[Tool execution skipped — {{name}} was not started. {interrupt_skip_wording(agent)}]",
  1680. flush_stage="skipped tool result",
  1681. ):
  1682. return
  1683. break
  1684. if finalize:
  1685. _finalize_tool_batch(agent, messages, effective_task_id, len(tool_calls), _tool_budget)
  1686. def execute_tool_calls_segmented(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, segments=None) -> None:
  1687. """Execute a mixed batch as ordered parallel/sequential segments (the ``(kind, calls)``
  1688. plan from ``_plan_tool_batch_segments``), preserving per-call result order and barrier
  1689. boundaries exactly as fully-sequential execution. Turn-end work (budget + /steer) runs
  1690. once here (segments run with ``finalize=False``); each segment executor checks the
  1691. interrupt flag up front, so an interrupt drains later segments with one result per call."""
  1692. from types import SimpleNamespace
  1693. if segments is None:
  1694. _active_env = get_active_env(effective_task_id)
  1695. _exec_cwd = Path(_active_env.cwd) if _active_env is not None and _active_env.cwd else None
  1696. segments = _plan_tool_batch_segments(assistant_message.tool_calls, execution_cwd=_exec_cwd)
  1697. for kind, calls in segments:
  1698. if getattr(agent, "_incremental_persistence_failed", False):
  1699. return
  1700. segment_message = SimpleNamespace(tool_calls=list(calls))
  1701. run_segment = execute_tool_calls_concurrent if kind == "parallel" else execute_tool_calls_sequential
  1702. run_segment(agent, segment_message, messages, effective_task_id, api_call_count, finalize=False)
  1703. if getattr(agent, "_incremental_persistence_failed", False):
  1704. return
  1705. total_tools = len(assistant_message.tool_calls)
  1706. if total_tools > 0:
  1707. _finalize_tool_batch(agent, messages, effective_task_id, total_tools, _budget_for_agent(agent))
  1708. __all__ = [
  1709. "execute_tool_calls_concurrent",
  1710. "execute_tool_calls_segmented",
  1711. "execute_tool_calls_sequential",
  1712. ]