delegate_tool_dispatch.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520
  1. """Batch execution + background dispatch for delegate_task: ``delegate_task`` builds a ``_Batch``
  2. (children + origin identity) and hands it to ``_run_batch``, which runs it synchronously or as
  3. detached async units — one per task ``group`` (ungrouped tasks are a unit each), so a finished
  4. review does not wait for the slowest sibling."""
  5. from __future__ import annotations
  6. import contextvars
  7. import json
  8. import logging
  9. import time
  10. from concurrent.futures import FIRST_COMPLETED, wait as _cf_wait
  11. from dataclasses import dataclass, field, replace
  12. from functools import partial
  13. from typing import Any, Dict, List, Optional
  14. from hermes_cli.observability.shared_metrics_loop import begin_delegation_run, finish_delegation_unit
  15. from tools.async_delegation import _new_delegation_id, record_unit_child
  16. from tools.delegate_tool_child_run import _attach_child, _close_child, _detach_child, _fabricated_entry, _signal_child_stop
  17. from tools.delegate_tool_progress import (
  18. SUBAGENT_FAILURE_STATUSES, _print_completion_line, _quiet, describe_subagent_failure, format_batch_tag,
  19. )
  20. from tools.delegate_tool_registry import _capture_gateway_steer_authority
  21. from tools.delegate_tool_results import _finalize_child_results
  22. logger = logging.getLogger("tools.delegate_tool") # log-record parity with the origin module
  23. @dataclass
  24. class _Batch:
  25. """One delegate_task call's built children plus everything needed to run them
  26. and assemble the combined result (shared by the sync path and the background runner)."""
  27. task_list: list[dict[str, Any]]
  28. children: list[tuple]
  29. parent_agent: Any
  30. creds: dict[str, Any]
  31. context: Optional[str]
  32. top_role: str
  33. max_children: int
  34. live_deleg_id: Optional[str]
  35. live_writers: list
  36. live_paths: list
  37. origin_wake_sid: str
  38. origin_ui_session_id: str
  39. origin_owner_transport: Any
  40. origin_owner_session_record: Any
  41. origin_session_history_delivery: bool
  42. overall_start: float
  43. # Set on per-group units carved out by ``_dispatch_background``; None for the whole batch / ungrouped units.
  44. group: Optional[str] = None
  45. unit_id: Optional[str] = None # the async registry id this unit runs under (``<call_id>-k`` for split calls)
  46. live_home: Any = None # explicit profile home for transcripts/manifest (#91996); None = ambient resolve
  47. # Result entries of tasks their backend refused (no child): joined into a sync result, listed in a background
  48. # call's dispatch payload. Units carved for async dispatch never carry them.
  49. refused: list = field(default_factory=list)
  50. def owner_kwargs(self) -> dict[str, Any]:
  51. """Steer/stop authority of the originating session, passed to every child run."""
  52. return {
  53. "owner_session_id": self.origin_ui_session_id or None, "owner_transport": self.origin_owner_transport,
  54. "owner_session_record": self.origin_owner_session_record,
  55. }
  56. def run_child(self, i: int, task: dict[str, Any], child: Any) -> dict[str, Any]:
  57. from tools.delegate_tool import _run_single_child
  58. return _run_single_child(task_index=i, goal=task["goal"], child=child, parent_agent=self.parent_agent, **self.owner_kwargs())
  59. def _announce_batch(parent_agent, n_tasks: int, live_deleg_id: Optional[str]) -> None:
  60. """Announce the batch tag once so interleaved ``[tag n/N]`` lines are attributable."""
  61. if n_tasks > 1 and live_deleg_id:
  62. _hdr = f" 🔀 [{format_batch_tag(live_deleg_id, parent_agent)}] delegating {n_tasks} tasks"
  63. _print_completion_line(parent_agent, getattr(parent_agent, "_delegate_spinner", None), _hdr, console_line=_hdr)
  64. def _capture_origin() -> tuple[str, str, Any, Any, bool]:
  65. """``(wake_sid, ui_session_id, owner_transport, owner_session_record, session_history_delivery)`` of the
  66. ORIGINATING session, captured BEFORE building any child: AIAgent construction
  67. clobbers the HERMES_SESSION_ID ContextVar/os.environ with the subagent's id. The wake-
  68. capability flag rides the same request-scoped binding and is captured here for the same
  69. reason — and fails closed: a binding that never declared it (or a read error) leaves the
  70. session treated as non-wake-capable (#98619)."""
  71. from tools.async_delegation import _current_origin_session_id
  72. _origin_wake_sid = _current_origin_session_id()
  73. _origin_ui_session_id = ""
  74. _origin_session_history_delivery = False
  75. with _quiet(None):
  76. from gateway.session_context import get_session_env, session_history_delivery_supported
  77. _origin_ui_session_id = get_session_env("HERMES_UI_SESSION_ID", "")
  78. _origin_session_history_delivery = session_history_delivery_supported()
  79. return (_origin_wake_sid, _origin_ui_session_id, *_capture_gateway_steer_authority(_origin_ui_session_id), _origin_session_history_delivery)
  80. def _report_child_done(parent_agent, spinner_ref, entry, tag, task_labels, n_tasks, remaining) -> None:
  81. """Print one completion line for a finished child and refresh the spinner text. Failed/errored/timed-out children
  82. say WHY on the same line — a bare ✗ reads as "silently dropped"."""
  83. idx = entry["task_index"]
  84. label = task_labels[idx] if idx < len(task_labels) else f"Task {idx}"
  85. status = entry.get("status", "?")
  86. _slot = f"{tag} · {idx+1}/{n_tasks}" if tag else f"{idx+1}/{n_tasks}"
  87. schema_invalid = entry.get("schema_valid") is False and status == "completed"
  88. icon = "⚠" if schema_invalid else ("✓" if status == "completed" else "✗")
  89. completion_line = f"{icon} [{_slot}] {label} ({entry.get('duration_seconds', 0)}s)"
  90. _err_line = (
  91. describe_subagent_failure(entry.get("failure_reason"), entry.get("error"), max_chars=120)
  92. if status in SUBAGENT_FAILURE_STATUSES else "")
  93. if schema_invalid:
  94. _err_line = "output_schema not satisfied — raw text returned (schema_valid=false)"
  95. if _err_line:
  96. completion_line += f" — {_err_line}"
  97. _print_completion_line(parent_agent, spinner_ref, completion_line)
  98. if spinner_ref and remaining > 0:
  99. with _quiet("Spinner update_text failed: %s"):
  100. spinner_ref.update_text(f"🔀 {'[' + tag + '] ' if tag else ''}{remaining} task{'s' if remaining != 1 else ''} remaining")
  101. def _record_finished_child(batch: _Batch, entry: Any, honor_parent_interrupt: bool) -> None:
  102. """Detached (background) unit: durably record a child on the unit's own row the moment it finishes, so an owner
  103. death before the unit's join — or anywhere before its durable completion write, the whole remaining window for a
  104. one-child unit — loses only children still running, never finished work (#116000). Best-effort by construction:
  105. ``record_unit_child`` never raises into the join."""
  106. if not honor_parent_interrupt and batch.unit_id and isinstance(entry, dict):
  107. record_unit_child(batch.unit_id, entry)
  108. def _release_unstarted_child(batch: _Batch, child: Any) -> None:
  109. """Teardown for a child whose run never started (``_run_single_child``'s finally never runs for it)."""
  110. _detach_child(batch.parent_agent, child)
  111. _close_child(child, "Failed to close a subagent that never started")
  112. def _run_gated(batch: _Batch, gate: Any, honor_parent_interrupt: bool, i: int, task: dict[str, Any],
  113. child: Any) -> dict[str, Any]:
  114. """``batch.run_child`` behind the prompt-cache warm-up gate; a child stopped while it waited never starts.
  115. A detached (background) unit ignores the parent's turn interrupt, so only the child's own stop skips it."""
  116. parent = batch.parent_agent if honor_parent_interrupt else None
  117. entry = gate.run(child, lambda: batch.run_child(i, task, child), parent=parent)
  118. if entry is not None:
  119. return entry
  120. _release_unstarted_child(batch, child)
  121. return _fabricated_entry(i, "interrupted", "Stopped while waiting for the batch's shared prompt cache", child)
  122. def _run_children_parallel(batch: _Batch, results: list, *, honor_parent_interrupt: bool) -> None:
  123. """Run the batch's children in parallel, appending entries to ``results`` (sorted by task_index on return, one
  124. completion line printed per child). Polls futures with a short ``wait()`` timeout instead of ``as_completed()``
  125. so a wedged child cannot block the parent forever after an interrupt; on parent interrupt the still-pending
  126. children are reported ``interrupted`` and abandoned (they already got the interrupt signal)."""
  127. # Daemon workers (tools.daemon_pool): the `with` block still joins normally, but if the parent is interrupted
  128. # while a child is wedged, the abandoned worker must not block interpreter exit.
  129. from tools.daemon_pool import DaemonThreadPoolExecutor
  130. parent_agent, n_tasks = batch.parent_agent, len(batch.task_list)
  131. task_labels = [t["goal"][:40] for t in batch.task_list]
  132. spinner_ref = getattr(parent_agent, "_delegate_spinner", None)
  133. _tag = format_batch_tag(batch.live_deleg_id, parent_agent)
  134. # Fabricated entries for still-pending / raised futures carry the correct _delegate_role.
  135. _child_by_index = {i: child for (i, _, child) in batch.children}
  136. n_here = len(batch.children) # a per-group unit runs a subset; ``n_tasks`` keeps the call-wide ``i/N`` slot
  137. def _entry_of(future, idx):
  138. if not future.done():
  139. return _fabricated_entry(
  140. idx, "interrupted", "Parent agent interrupted — child did not finish in time", _child_by_index.get(idx),
  141. )
  142. try:
  143. return future.result()
  144. except Exception as exc:
  145. return _fabricated_entry(idx, "error", str(exc), _child_by_index.get(idx))
  146. executor = DaemonThreadPoolExecutor(max_workers=batch.max_children)
  147. # ``with`` would join every worker on exit, so a child wedged in an
  148. # uninterruptible call defeats the interrupt fast-path: the poll loop marks
  149. # it interrupted and returns, then the join hangs forever. Shutdown without
  150. # waiting on the interrupt path instead (same shape as moa_loop).
  151. interrupted, futures = False, {}
  152. try:
  153. from tools.delegate_tool_config import _get_prefix_stagger_seconds
  154. from tools.delegate_tool_prefix_gate import PrefixCacheGate
  155. gate = PrefixCacheGate.for_children(batch.children, _get_prefix_stagger_seconds())
  156. run = batch.run_child if gate is None else partial(_run_gated, batch, gate, honor_parent_interrupt)
  157. futures = {executor.submit(contextvars.copy_context().run, run, i, t, child): i for i, t, child in batch.children}
  158. pending = set(futures)
  159. while pending:
  160. if honor_parent_interrupt and getattr(parent_agent, "_interrupt_requested", False) is True:
  161. results.extend(_entry_of(f, futures[f]) for f in pending)
  162. interrupted = True
  163. break
  164. done, pending = _cf_wait(pending, timeout=0.5, return_when=FIRST_COMPLETED)
  165. for future in done:
  166. entry = _entry_of(future, futures[future])
  167. results.append(entry)
  168. # Detached unit: a crash before the join must not lose children that already finished.
  169. _record_finished_child(batch, entry, honor_parent_interrupt)
  170. _report_child_done(parent_agent, spinner_ref, entry, _tag, task_labels, n_tasks, n_here - len(results))
  171. if (not honor_parent_interrupt and batch.unit_id and entry.get("status") in SUBAGENT_FAILURE_STATUSES
  172. and len(results) < n_here):
  173. # Detached unit, a sibling is still running: tell the parent NOW, not when the last one finishes.
  174. # Non-durable and separate from the unit's final result (which is still delivered once).
  175. with _quiet("task failure notice failed", exc_info=True):
  176. from tools.async_delegation import push_task_failure_notice
  177. _i = entry.get("task_index", -1)
  178. _live = batch.live_paths[_i] if isinstance(_i, int) and 0 <= _i < len(batch.live_paths) else None
  179. push_task_failure_notice(
  180. batch.unit_id, {**entry, **({"live_transcript": _live} if _live else {})}, n_tasks=n_tasks)
  181. finally:
  182. # Abandoned workers unwind on their own (daemon threads, and the
  183. # child-run layer already applies the deferred-close transport drain).
  184. executor.shutdown(wait=not interrupted, cancel_futures=interrupted)
  185. for future, idx in futures.items() if interrupted else ():
  186. if future.cancelled(): # queued behind max_concurrent_children: never started, so never cleaned up
  187. _release_unstarted_child(batch, _child_by_index.get(idx))
  188. results.sort(key=lambda r: r["task_index"]) # match input order
  189. def _execute_and_aggregate(batch: _Batch, *, honor_parent_interrupt: bool = True) -> dict:
  190. """Run the batch's built children, join, finalize (hooks + cost rollup), return the combined dict. Shared by the
  191. sync path and the background runner; a background runner receives a per-group unit (a subset of the call's
  192. children) so each group JOINS only on itself. Live transcripts are finalized but retained as the full-fidelity
  193. record (retention pruning happens on future dispatches)."""
  194. from tools.delegation_live_log import update_manifest_statuses
  195. results: list = []
  196. if len(batch.children) == 1:
  197. results.append(batch.run_child(*batch.children[0]))
  198. # A one-child unit has no join to wait on, but everything after the child returns — host-owned finalize,
  199. # transcript, manifest, then the durable write — is still owner-lifetime: record before any of it (#116000).
  200. _record_finished_child(batch, results[-1], honor_parent_interrupt)
  201. else:
  202. _run_children_parallel(batch, results, honor_parent_interrupt=honor_parent_interrupt)
  203. _finalize_child_results(results, batch.task_list, batch.children, batch.parent_agent)
  204. total_duration = round(time.monotonic() - batch.overall_start, 2)
  205. for entry in results:
  206. _idx = entry.get("task_index", -1)
  207. if isinstance(_idx, int) and 0 <= _idx < len(batch.live_writers) and batch.live_writers[_idx] is not None:
  208. with _quiet("Live transcript finalize failed", exc_info=True):
  209. batch.live_writers[_idx].finalize(entry)
  210. if _idx < len(batch.live_paths):
  211. entry["live_transcript"] = batch.live_paths[_idx]
  212. if batch.refused:
  213. results.extend(batch.refused)
  214. results.sort(key=lambda r: r["task_index"])
  215. update_manifest_statuses(batch.live_deleg_id, results, home=batch.live_home)
  216. finish_delegation_unit(batch.task_list, results, background=not honor_parent_interrupt)
  217. combined: dict[str, Any] = {"results": results, "total_duration_seconds": total_duration}
  218. # Runtime truth about children's background processes, as prose the parent can't miss inside the JSON.
  219. from tools.process_registry_notifications import _process_accounting_lines
  220. process_notes = [line for entry in results for line in _process_accounting_lines(entry)]
  221. if process_notes:
  222. combined["process_notes"] = process_notes
  223. unit_paths = [batch.live_paths[i] for (i, _, _) in batch.children if i < len(batch.live_paths)]
  224. if unit_paths:
  225. combined["live_transcripts"] = unit_paths
  226. if batch.group is not None:
  227. combined["group"] = batch.group
  228. return combined
  229. _SYNC_FALLBACK_NOTES = {
  230. "no_async": (
  231. "background=true is not available in this session — it cannot "
  232. "receive a detached subagent result after the turn ends (a "
  233. "finite chat using -Q, --oneshot, or non-TTY stdio, `hermes -z`, a /bg background task, a cron "
  234. "job, a Kanban worker, or a stateless HTTP endpoint). The subagent(s) ran SYNCHRONOUSLY and the result is included above."
  235. ),
  236. "at_capacity": (
  237. "The background delegation pool was at capacity (delegation.max_concurrent_children), so the subagent(s) ran "
  238. "SYNCHRONOUSLY and the result is included above. Raise "
  239. "delegation.max_concurrent_children in config.yaml to allow more concurrent background delegations."
  240. ),
  241. }
  242. def _run_sync_with_note(batch: _Batch, reason: str) -> str:
  243. """Inline fallback: run the batch now and explain why it was not detached."""
  244. result = _execute_and_aggregate(batch)
  245. if isinstance(result, dict):
  246. result["note"] = _SYNC_FALLBACK_NOTES[reason]
  247. return json.dumps(result, ensure_ascii=False)
  248. def _resolve_async_wake_sid(
  249. origin_wake_sid: str, origin_session_history_delivery: bool = False, oneshot_session: bool = False,
  250. ) -> Optional[str]:
  251. """Detached result target: empty for push, a resumable API id, or None for inline.
  252. API completion only persists a row; this does not authorize a model wake. The
  253. continuation must read that row, not an authoritative caller-owned snapshot.
  254. *oneshot_session*: the parent's tools[] froze the one-shot ``delegate_task`` ("joined in this call", no
  255. list/steer/stop) for the whole session, so a later interactive turn (``chat -q`` then ``--resume``) joins too."""
  256. from gateway.session_context import get_session_env
  257. # Finite consumers need the result in this tool round. Owner admission scope
  258. # wins over legacy standalone launch markers, without changing terminal delivery.
  259. from gateway.session_finite import finite_turn_required
  260. finite = finite_turn_required()
  261. if oneshot_session or finite is True or (finite is None and get_session_env("HERMES_SINGLE_QUERY_SESSION") == "1"):
  262. return None
  263. try:
  264. # Finite sessions cannot route a detached subagent result back to the agent after their turn/process
  265. # ends. This includes stateless HTTP requests (#10760) and one-shot Kanban workers (#63169). Fall
  266. # back to SYNCHRONOUS execution so the result returns in this same turn instead of handing out a
  267. # handle with no durable consumer. Mirrors the pool-at-capacity inline fallback below.
  268. from gateway.session_context import async_delivery_supported
  269. if async_delivery_supported():
  270. return ""
  271. except Exception:
  272. return ""
  273. if origin_wake_sid and origin_session_history_delivery:
  274. logger.info(
  275. "delegate_task: session %s resumes server history — detached result will be persisted "
  276. "for the next client turn (no model wake).", origin_wake_sid,
  277. )
  278. return origin_wake_sid
  279. if origin_wake_sid:
  280. logger.info(
  281. "delegate_task: session %s has no declared server-history consumer — running the batch "
  282. "synchronously so the result returns in this turn.", origin_wake_sid,
  283. )
  284. return None
  285. def _resolve_async_session_key(parent_agent: Any, origin_ui_session_id: str) -> tuple[str, str]:
  286. """``(session_key, origin_ui_session_id)`` the async registry routes completions by.
  287. Desktop/TUI: the routable key is the durable AIAgent.session_id — compression can rotate it mid-turn before the
  288. TUI-side dict is re-anchored, and a stale approval-context key would orphan the completion. Gateway chats keep the
  289. platform conversation key (agent:main:...). The CLI has no bound approval contextvar and no HERMES_SESSION_KEY, so
  290. the key resolves empty; its drain is a positive-ownership filter on the durable session_id (empty would fail
  291. closed), so stamp the parent's durable id.
  292. """
  293. from tools.approval_context import get_current_session_key
  294. session_key = get_current_session_key(default="")
  295. agent_session_id = str(getattr(parent_agent, "session_id", "") or "")
  296. with _quiet(None):
  297. from gateway.session_context import get_session_env
  298. source = get_session_env("HERMES_SESSION_SOURCE", "")
  299. # Refresh from the task-local source when available, else retain the
  300. # immutable value captured before child construction.
  301. origin_ui_session_id = get_session_env("HERMES_UI_SESSION_ID", "") or origin_ui_session_id
  302. if source == "tui" and agent_session_id:
  303. session_key = agent_session_id
  304. return session_key or agent_session_id, origin_ui_session_id
  305. def _batch_progress_token(child_agents: list[Any]) -> tuple:
  306. """Progress token for the async registry's stale monitor: every child's (api_call_count, current_tool,
  307. last_activity_ts). last_activity_ts ticks on streamed chunks, tool transitions and API-call start/completion,
  308. so a child streaming a long response counts as alive; a fully frozen token past the threshold means the batch
  309. is wedged. ``in_tool`` is True while ANY child is inside a tool so slow tools get the higher ceiling (mirrors
  310. the sync heartbeat)."""
  311. # Progress token for the async registry's stale monitor: the combined (api_call_count, current_tool,
  312. # last_activity_ts) of every child. last_activity_ts is ticked by _touch_activity on every streamed
  313. # chunk ("receiving stream response"), every tool transition, and every API-call start/completion — so a
  314. # child streaming a long response is alive even though api_call_count only advances when the call
  315. # completes (same liveness signal as the compaction inactivity budget, PR #71508).
  316. parts = []
  317. in_tool = False
  318. for c in child_agents:
  319. try:
  320. summary = c.get_activity_summary()
  321. tool = summary.get("current_tool")
  322. parts.append((summary.get("api_call_count", 0), tool, summary.get("last_activity_ts")))
  323. in_tool = in_tool or bool(tool)
  324. except Exception:
  325. parts.append(None)
  326. return tuple(parts), in_tool
  327. _BACKGROUND_NOTES = {
  328. "one": (
  329. "Subagent is running in the background; its full result re-enters the conversation as a new message when it "
  330. "finishes. Results are delivered only after you END YOUR TURN: do anything that does not depend on it, then "
  331. "stop with a one-line status. Do not poll its transcript or artifacts to wait for it."
  332. ),
  333. "many": (
  334. "{n} subagents are running in parallel in the background as {k} completion unit(s); each unit's results "
  335. "re-enter the conversation as their own new message when THAT unit finishes. Results are delivered only "
  336. "after you END YOUR TURN: do anything that does not depend on them, then stop with a one-line status. Do not "
  337. "poll transcripts or artifacts to wait for them."
  338. ),
  339. "control_hint": (
  340. "While a child runs you can orchestrate it live with this same tool: delegate_task(action='list') to see live "
  341. "children, action='steer' with subagent_id + message to redirect one, action='stop' with subagent_id to end "
  342. "one early."
  343. ),
  344. "live_transcripts_hint": (
  345. "Each subagent streams a human-readable transcript of its operations to the file listed above (append-only, "
  346. "one per task). Read or `tail -f` these paths at any time to watch a child work while it runs."
  347. ),
  348. }
  349. def _dispatched_payload(batch: _Batch, units: list[tuple[_Batch, str]]) -> dict:
  350. """Model-facing handle for an accepted background call: one entry per async unit."""
  351. goals = [t["goal"] for t in batch.task_list]
  352. n = len(batch.children) # running children; refused tasks are listed under "refused"
  353. payload = {
  354. "status": "dispatched", "mode": "background", "count": n,
  355. "delegation_id": batch.live_deleg_id or units[0][1], "goals": goals,
  356. "note": _BACKGROUND_NOTES["one"] if n == 1 else _BACKGROUND_NOTES["many"].format(n=n, k=len(units)),
  357. }
  358. if len(units) > 1:
  359. payload["units"] = [
  360. {"delegation_id": uid, "group": unit.group, "task_indexes": [i for (i, _, _) in unit.children]}
  361. for unit, uid in units
  362. ]
  363. sids = [getattr(c, "_subagent_id", None) for (_, _, c) in batch.children]
  364. if any(isinstance(s, str) and s for s in sids):
  365. payload["subagent_ids"] = sids
  366. payload["control_hint"] = _BACKGROUND_NOTES["control_hint"]
  367. if batch.refused:
  368. payload["refused"] = batch.refused
  369. if batch.live_paths:
  370. payload["live_transcripts"] = list(batch.live_paths)
  371. payload["live_transcripts_hint"] = _BACKGROUND_NOTES["live_transcripts_hint"]
  372. return payload
  373. def _units_of(batch: _Batch) -> list[_Batch]:
  374. """Partition the call's children into async units: one per distinct task ``group`` (first-appearance order) and
  375. one per ungrouped task. Each unit is a ``_Batch`` sharing the call's task_list/transcripts but owning a subset of
  376. ``children``, so a unit joins only on itself and its completion re-enters the conversation on its own.
  377. Off by default (``delegation.independent_completions``): the whole call is ONE unit and returns as one message.
  378. A per-task flurry of completions (one new turn each) fragmented orchestrators that had no plan for it."""
  379. from tools.delegate_tool_config import _get_independent_completions
  380. if not _get_independent_completions():
  381. return [batch]
  382. members: dict[Any, list[tuple]] = {}
  383. for i, t, c in batch.children:
  384. g = t.get("group")
  385. key = ("g", str(g)) if g not in (None, "") else ("i", i)
  386. members.setdefault(key, []).append((i, t, c))
  387. return [replace(batch, children=ch, group=(key[1] if key[0] == "g" else None)) for key, ch in members.items()]
  388. def _dispatch_unit(unit: _Batch, unit_id: Optional[str], slot_key: Optional[str], routing: dict) -> dict:
  389. """Hand ONE unit to the async registry; the runner joins on that unit's children only."""
  390. from tools.async_delegation import dispatch_async_delegation_batch
  391. child_agents = [c for (_, _, c) in unit.children]
  392. def _interrupt():
  393. for c in child_agents:
  394. _signal_child_stop(c, "Async delegation cancelled")
  395. return dispatch_async_delegation_batch(
  396. # Call-wide goals: completion formatting indexes them by task_index.
  397. goals=[t["goal"] for t in unit.task_list], context=unit.context,
  398. toolsets=None, # metadata for the completion block only; subagents inherit the parent's toolsets
  399. # creds["model"] is None when delegation.model is blank (children inherit): report what the child runs on.
  400. role=unit.top_role, model=unit.creds["model"] or next(
  401. (getattr(c, "model", None) for c in child_agents if getattr(c, "model", None)), None),
  402. runner=lambda: _execute_and_aggregate(unit, honor_parent_interrupt=False),
  403. interrupt_fn=_interrupt, delegation_id=unit_id, slot_key=slot_key,
  404. task_indexes=[i for (i, _, _) in unit.children] if len(unit.children) < len(unit.task_list) else None,
  405. # Persist locators before starting workers; live_paths omits failed writers and can be compressed.
  406. task_transcripts={str(i): str(unit.live_writers[i].path) for i, _, _ in unit.children
  407. if i < len(unit.live_writers) and unit.live_writers[i] is not None
  408. and unit.live_writers[i].path is not None},
  409. progress_fn=lambda: _batch_progress_token(child_agents), **routing,
  410. )
  411. def _restore_parent_cancellation(unit: _Batch) -> None:
  412. """Rejected children stay owned by the parent: re-attach them (``_attach_child`` replays a stop that
  413. arrived while async admission had them detached)."""
  414. for _, _, child in unit.children:
  415. _attach_child(unit.parent_agent, child)
  416. def _dispatch_background(batch: _Batch) -> str:
  417. """Dispatch the call as independent async units (see ``_units_of``) and return the tool result JSON. Every unit
  418. of one call shares ONE pool slot (``slot_key``), so grouping never changes capacity accounting. Falls back to
  419. running synchronously (with an explanatory ``note``) when the session cannot receive detached completions or the
  420. async pool is at capacity."""
  421. from tools.delegate_tool import _get_max_async_children
  422. wake_sid = _resolve_async_wake_sid(
  423. batch.origin_wake_sid, batch.origin_session_history_delivery,
  424. oneshot_session=getattr(batch.parent_agent, "_oneshot_session", None) is True, # agent_is_oneshot's pin
  425. )
  426. if wake_sid is None:
  427. logger.info("delegate_task: async delivery unsupported on this session runtime; running the batch synchronously instead.")
  428. return _run_sync_with_note(batch, "no_async")
  429. parent_agent = batch.parent_agent
  430. session_key, origin_ui_session_id = _resolve_async_session_key(parent_agent, batch.origin_ui_session_id)
  431. routing = dict(
  432. session_key=session_key, origin_ui_session_id=origin_ui_session_id, origin_session_id=wake_sid,
  433. parent_session_id=getattr(parent_agent, "session_id", None), max_async_children=_get_max_async_children(),
  434. )
  435. units = [replace(unit, refused=[]) for unit in _units_of(batch)]
  436. dispatched: list[tuple[_Batch, str]] = []
  437. inline_results: list[dict] = []
  438. slot_key: Optional[str] = None
  439. for k, unit in enumerate(units):
  440. # One unit keeps the live-transcript directory's id so the returned delegation_id matches
  441. # cache/delegation/live/<id>/; several units suffix it (-1, -2, ...) and the call keeps the bare id.
  442. unit_id = batch.live_deleg_id if len(units) == 1 else (f"{batch.live_deleg_id}-{k + 1}" if batch.live_deleg_id else None)
  443. unit.unit_id = unit_id = unit_id or _new_delegation_id() # fixed before the runner can start
  444. # The worker can start before admission returns. Detach only this unit:
  445. # unsubmitted units must still receive parent stops while a fallback runs.
  446. for _, _, child in unit.children:
  447. _detach_child(parent_agent, child)
  448. dispatch = _dispatch_unit(unit, unit_id, slot_key, routing)
  449. if dispatch.get("status") == "dispatched":
  450. slot_key = slot_key or dispatch["delegation_id"]
  451. dispatched.append((unit, dispatch["delegation_id"]))
  452. continue
  453. _restore_parent_cancellation(unit)
  454. if not dispatched:
  455. logger.info(
  456. "delegate_task: async pool at capacity (%s); running the whole batch synchronously instead.",
  457. dispatch.get("error", "rejected"),
  458. )
  459. return _run_sync_with_note(batch, "at_capacity")
  460. # Later units of an admitted call share its slot and cannot be capacity-rejected; a scheduler failure runs
  461. # the unit inline so no task is silently dropped.
  462. logger.warning("delegate_task: unit %d/%d not accepted (%s); running it inline.", k + 1, len(units), dispatch.get("error"))
  463. inline_results.extend(_execute_and_aggregate(unit)["results"])
  464. payload = _dispatched_payload(batch, dispatched)
  465. if inline_results:
  466. payload["inline_results"] = inline_results
  467. return json.dumps(payload, ensure_ascii=False)
  468. def _run_batch(batch: _Batch, background: bool) -> str:
  469. """Tool result JSON: a dispatch handle (background) or the joined combined results."""
  470. from tools.delegate_tool import session_joins_delegation
  471. # Every unit of this call shares task_list, so its last joined unit emits the call's one row.
  472. begin_delegation_run(
  473. batch.task_list, subagents=len(batch.children), depth=getattr(batch.parent_agent, "_delegate_depth", 0) + 1,
  474. )
  475. if background and not session_joins_delegation(batch.parent_agent):
  476. return _dispatch_background(batch)
  477. return json.dumps(_execute_and_aggregate(batch), ensure_ascii=False)