delegate_tool_child_run.py 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173
  1. """Running ONE pre-built child agent: heartbeat, registry entry, workspace seeding,
  2. timeout/failure handling, result-entry assembly and cleanup (``_ChildRun``)."""
  3. from __future__ import annotations
  4. import logging
  5. import contextvars
  6. import json
  7. import os
  8. import threading
  9. import time
  10. from tools.delegation_redaction import redact_child_result, redact_child_text
  11. from concurrent.futures import TimeoutError as FuturesTimeoutError
  12. from typing import Any, Dict, List, Optional
  13. from agent.compression_marker import elide
  14. from agent.interrupt_compat import request_hard_interrupt
  15. from dataclasses import dataclass, field
  16. from tools import file_state
  17. from tools.delegate_tool_progress import _quiet, _safe_progress
  18. from tools.delegate_tool_registry import (
  19. _capture_gateway_steer_authority, _close_subagent_steering, _register_subagent, _unregister_subagent,
  20. )
  21. from tools.delegate_tool_results import (
  22. _extract_output_tail, _looks_like_error_output, _stringify_tool_content, _summarize_tool_arguments,
  23. )
  24. logger = logging.getLogger("tools.delegate_tool") # log-record parity with the origin module
  25. def _reserve_child_transcript(child: Any) -> None:
  26. """Make a queued child readable without completing its first-turn persistence."""
  27. from agent.runtime_session_store import RuntimeSessionStore
  28. db = child._session_db
  29. if db is None or child._persist_disabled or isinstance(db, RuntimeSessionStore):
  30. # Managed workers already have an owner-reserved row. Persistence-disabled
  31. # children retain their existing no-transcript behavior.
  32. return
  33. try:
  34. # Leave model_config and the prompt unset. The constructor upsert keeps
  35. # existing values, so a partial config here would freeze first-turn state.
  36. db.create_session(
  37. session_id=child.session_id, source="subagent",
  38. parent_session_id=child._parent_session_id,
  39. )
  40. child._session_db_reserved = True
  41. except BaseException:
  42. # Ownership has transferred, but the child is not attached or published yet.
  43. child.close()
  44. raise
  45. def _num(value: Any, default: int = 0) -> int:
  46. """int() for counters that may be mocks/None on test doubles."""
  47. return int(value) if isinstance(value, (int, float)) else default
  48. def _str_or_none(value: Any) -> Optional[str]:
  49. return value if isinstance(value, str) else None
  50. def _tier_in_effect(child: Any) -> Any:
  51. """The child's speed tier: its own (a route's tier), else the one its request settings carry (an inherited
  52. ``service_tier: priority`` or ``speed: fast``, top level or in ``extra_body``)."""
  53. if getattr(child, "service_tier", None):
  54. return child.service_tier
  55. overrides = getattr(child, "request_overrides", None)
  56. overrides = overrides if isinstance(overrides, dict) else {}
  57. extra = overrides.get("extra_body") if isinstance(overrides.get("extra_body"), dict) else {}
  58. for scope in (overrides, extra):
  59. if scope.get("service_tier"):
  60. return scope["service_tier"]
  61. if scope.get("speed") == "fast":
  62. return "priority"
  63. return None
  64. def _route_report(child: Any) -> Optional[dict[str, Any]]:
  65. """What a backend-routed child actually ran on: its backend and label plus the provider, model, reasoning effort
  66. (None = the provider's default) and speed tier read off the child when its result is assembled, so a fallback
  67. provider that took over shows (``fallback_activated``). None for the default backend's unlabelled route."""
  68. route = getattr(child, "_delegate_route", None)
  69. if not isinstance(route, dict):
  70. return None
  71. from agent.fast_mode import service_tier_word
  72. reasoning = getattr(child, "reasoning_config", None)
  73. effort = None
  74. if isinstance(reasoning, dict):
  75. effort = reasoning.get("effort") if reasoning.get("enabled", True) else "none"
  76. return {**route, "provider": _str_or_none(getattr(child, "provider", None)),
  77. "model": _str_or_none(getattr(child, "model", None)), "reasoning_effort": _str_or_none(effort),
  78. "service_tier": service_tier_word(_tier_in_effect(child)),
  79. **({"fallback_activated": True} if getattr(child, "_fallback_activated", False) is True else {})}
  80. def _fabricated_entry(idx: int, status: str, error: str, child: Any, duration: float = 0) -> dict[str, Any]:
  81. """Result entry for a child that raised, never finished, or was abandoned."""
  82. entry = {
  83. "task_index": idx, "status": status, "summary": None, "error": error, "api_calls": 0,
  84. "duration_seconds": duration, "_child_role": getattr(child, "_delegate_role", None),
  85. }
  86. route = _route_report(child)
  87. if route is not None:
  88. entry["route"] = route
  89. return entry
  90. def _append_missed_steer(entry: dict[str, Any], late_steer: Optional[str]) -> None:
  91. """Record steer text that won the race with the child's failure/timeout."""
  92. if late_steer:
  93. entry["missed_steer"] = late_steer
  94. entry["error"] += (" [steer did not land before the subagent stopped: " f"{late_steer}]")
  95. def _close_child(child: Any, log_message: str) -> None:
  96. """Best-effort ``child.close()`` (tool sandboxes, browser daemons, httpx clients)."""
  97. with _quiet(log_message, exc_info=True):
  98. close = getattr(child, "close", None)
  99. if callable(close):
  100. close()
  101. def _with_children_lock(parent_agent: Any, op: str, child: Any) -> None:
  102. """``parent_agent._active_children.<op>(child)`` under the parent's lock when it has one."""
  103. lock = getattr(parent_agent, "_active_children_lock", None)
  104. if lock:
  105. with lock:
  106. getattr(parent_agent._active_children, op)(child)
  107. else:
  108. getattr(parent_agent._active_children, op)(child)
  109. def _attach_child(parent_agent: Any, child: Any) -> None:
  110. """Register the child for parent interrupt propagation.
  111. ``interrupt()`` fans out to a SNAPSHOT of ``_active_children``; a child attached after the stop
  112. landed (a fan-out still building its siblings, a turn that has not reached its iteration check yet)
  113. would otherwise start with no signal and run to completion as an orphan. Mirror a pending stop here
  114. so the whole spawn tree dies with its parent."""
  115. if hasattr(parent_agent, "_active_children"):
  116. _with_children_lock(parent_agent, "append", child)
  117. if getattr(parent_agent, "_interrupt_requested", False) is not True:
  118. return
  119. # Same soft/hard split as ``interrupt()``'s own fan-out: a hard stop cancels, a soft one redirects.
  120. message = getattr(parent_agent, "_interrupt_message", None)
  121. hard = getattr(parent_agent, "_hard_interrupt_requested", None)
  122. if hard is None or hard.is_set():
  123. _signal_child_stop(child, message or "parent agent interrupted",
  124. tool_reason=getattr(parent_agent, "_tool_interrupt_reason", None) or "parent agent interrupted")
  125. else:
  126. with _quiet("Failed to propagate interrupt to late child: %s"):
  127. child.interrupt(message)
  128. def _detach_child(parent_agent: Any, child: Any) -> None:
  129. """Remove the child from parent interrupt propagation (no-op if absent)."""
  130. if not hasattr(parent_agent, "_active_children"):
  131. return
  132. try:
  133. _with_children_lock(parent_agent, "remove", child)
  134. except (ValueError, UnboundLocalError) as e:
  135. logger.debug("Could not remove child from active_children: %s", e)
  136. def _signal_child_stop(child: Any, *reason: str, tool_reason: str = "parent delegation ended") -> None:
  137. """Cooperative interrupt so the child's worker thread can exit cleanly. ``tool_reason`` is the
  138. fixed cause the child's tools see (a pending approval wait reports it instead of a user deny)."""
  139. with _quiet(None):
  140. if (child is not None and not request_hard_interrupt(child, *reason, tool_reason=tool_reason)
  141. and hasattr(child, "_interrupt_requested")):
  142. child._interrupt_requested = True
  143. # ── 0-API-call timeout diagnostic ────────────────────────────────────────────
  144. def _format_thread_stack(frame: Any, indent: str) -> list[str]:
  145. import traceback as _traceback
  146. return [f"{indent}{sub}" for frame_line in _traceback.format_stack(frame) for sub in frame_line.rstrip().split("\n")]
  147. _DIAG_CHILD_ATTRS = (
  148. "model", "provider", "api_mode", "base_url", "max_iterations", "quiet_mode", "skip_memory", "skip_context_files",
  149. "platform", "_delegate_role", "_delegate_depth",
  150. )
  151. def _diag_section(label: str, produce) -> list[str]:
  152. """Lines from ``produce()``, or one ``<label: ...exc>`` line so a broken attribute never aborts the dump."""
  153. try:
  154. return list(produce())
  155. except Exception as exc:
  156. return [f" {label}{exc}>"]
  157. def _diag_sizes(child: Any) -> list[str]:
  158. def _prompt():
  159. sys_prompt = getattr(child, "ephemeral_system_prompt", None) or getattr(child, "system_prompt", None) or ""
  160. is_str = isinstance(sys_prompt, str)
  161. return [
  162. f" system_prompt_bytes: {len(sys_prompt.encode('utf-8')) if is_str else 'n/a'}",
  163. f" system_prompt_chars: {len(sys_prompt) if is_str else 'n/a'}",
  164. ]
  165. def _tools():
  166. tools_schema = getattr(child, "tools", None)
  167. if tools_schema is None:
  168. return []
  169. return [
  170. f" tool_schema_count: {len(tools_schema)}",
  171. f" tool_schema_bytes: {len(json.dumps(tools_schema, default=str).encode('utf-8'))}",
  172. ]
  173. return ["## Prompt / schema sizes"] + _diag_section("system_prompt: <error: ", _prompt) + _diag_section("tool_schema: <error: ", _tools)
  174. def _diag_threads(worker_thread: Optional[threading.Thread]) -> list[str]:
  175. """Worker stack plus all other live threads (bounded to 40): the worker is often parked on a helper thread, so a
  176. pre-HTTP wedge is indistinguishable from a slow provider without the full picture."""
  177. import sys as _sys
  178. lines = ["## Worker thread stack at timeout"]
  179. frames = _sys._current_frames()
  180. if worker_thread is not None and worker_thread.is_alive():
  181. worker_frame = frames.get(worker_thread.ident)
  182. lines.extend(_format_thread_stack(worker_frame, " ") if worker_frame is not None else [" <worker frame not available>"])
  183. else:
  184. lines.append(" <no worker thread handle>" if worker_thread is None else " <worker thread already exited>")
  185. def _all_threads():
  186. frames = _sys._current_frames()
  187. by_ident = {th.ident: th for th in threading.enumerate() if th.ident}
  188. worker_ident = worker_thread.ident if worker_thread else None
  189. out: list[str] = []
  190. for dumped, (ident, frame) in enumerate(f for f in frames.items() if f[0] != worker_ident): # worker dumped above
  191. if dumped >= 40:
  192. out.append(f" <{len(frames) - dumped - 1} more threads omitted>")
  193. break
  194. th = by_ident.get(ident)
  195. out.append(f" --- {th.name if th else f'ident={ident}'}{' daemon' if (th and th.daemon) else ''} ---")
  196. out.extend(_format_thread_stack(frame, " "))
  197. return out
  198. return lines + ["", "## All thread stacks at timeout"] + _diag_section("<all-thread dump failed: ", _all_threads)
  199. def _dump_subagent_timeout_diagnostic(
  200. *, child: Any, task_index: int, timeout_seconds: float, duration_seconds: float,
  201. worker_thread: Optional[threading.Thread], goal: str,
  202. ) -> Optional[str]:
  203. """Structured diagnostic for a subagent that timed out before any API call (otherwise "timed out with no
  204. response", 0 API calls, nothing to inspect): ``~/.hermes/logs/subagent-timeout-<sid>-<ts>.log`` with the
  205. child's config, prompt/schema sizes, activity snapshot and worker stack. Path, or None on failure."""
  206. try:
  207. from hermes_constants import get_hermes_home
  208. import datetime as _dt
  209. logs_dir = get_hermes_home() / "logs"
  210. try:
  211. logs_dir.mkdir(parents=True, exist_ok=True)
  212. except Exception:
  213. return None
  214. subagent_id = getattr(child, "_subagent_id", None) or f"idx{task_index}"
  215. dump_path = logs_dir / f"subagent-timeout-{subagent_id}-{_dt.datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
  216. _goal_preview = elide((goal or "").strip(), 1000)
  217. def _attr_line(attr):
  218. try:
  219. return f" {attr}: {getattr(child, attr, None)!r}"
  220. except Exception:
  221. return f" {attr}: <unreadable>"
  222. tool_names = getattr(child, "valid_tool_names", None)
  223. lines: list[str] = [
  224. "# Subagent timeout diagnostic — issue #14726", f"# Generated: {_dt.datetime.now().isoformat()}", "",
  225. "## Timeout", f" task_index: {task_index}", f" subagent_id: {subagent_id}",
  226. f" configured_timeout: {timeout_seconds}s", f" actual_duration: {duration_seconds:.2f}s", "", "## Goal",
  227. _goal_preview or "(empty)", "", "## Child config", *map(_attr_line, _DIAG_CHILD_ATTRS),
  228. "", "## Toolsets", f" enabled_toolsets: {getattr(child, 'enabled_toolsets', None)!r}",
  229. ]
  230. if tool_names:
  231. lines.append(f" loaded tool count: {len(tool_names)}")
  232. with _quiet(None):
  233. lines.append(f" loaded tools: {sorted(tool_names)}")
  234. lines += ["", *_diag_sizes(child), "", "## Activity summary"]
  235. lines += _diag_section(
  236. "<get_activity_summary failed: ", lambda: [f" {k}: {v!r}" for k, v in child.get_activity_summary().items()],
  237. )
  238. lines += ["", *_diag_threads(worker_thread), "", "## Notes",
  239. " This file is written ONLY when a subagent times out with 0 API calls.",
  240. " 0-API-call timeouts mean the child never reached its first LLM request.",
  241. " Common causes: oversized prompt rejected by provider, transport hang,",
  242. " credential resolution stuck. See issue #14726 for context.",
  243. ]
  244. dump_path.write_text("\n".join(lines), encoding="utf-8")
  245. return str(dump_path)
  246. except Exception as exc:
  247. logger.warning("Subagent timeout diagnostic dump failed: %s", exc)
  248. return None
  249. # ── Per-run helpers ──────────────────────────────────────────────────────────
  250. # Granularity for re-checking a child's progress while a configured ``child_timeout_seconds`` budget runs.
  251. # Five seconds is finer than the 30s heartbeat and cheap (one activity-summary read per slice).
  252. _LIVENESS_POLL_SECONDS = 5.0
  253. # Fraction of the inactivity budget after which the child is warned once (via its steer channel) that the window is
  254. # closing, so a child that is merely slow can wrap up and return instead of losing its whole context (#116001).
  255. _BUDGET_WARNING_FRACTION = 0.8
  256. def _budget_warning_text(idle_seconds: float, child_timeout: float) -> str:
  257. return (
  258. f"[delegation budget warning] No progress signal for {idle_seconds:.0f}s of your {child_timeout:.0f}s "
  259. "inactivity window. Finish the current step and return your summary now — the work is discarded if the "
  260. "window elapses."
  261. )
  262. def _warn_child_budget(child: Any, idle_seconds: float, child_timeout: float) -> None:
  263. """Queue the one-line warning through the child's steer path (delivered at its next iteration boundary)."""
  264. steer = getattr(child, "steer", None)
  265. if not callable(steer):
  266. return
  267. try:
  268. steer(_budget_warning_text(idle_seconds, child_timeout))
  269. except Exception as exc:
  270. logger.debug("budget warning steer failed: %s", exc)
  271. def _child_activity_fingerprint(child: Any) -> tuple:
  272. """``(completed calls, current tool, activity clock)`` — the progress signals the heartbeat's stale verdict
  273. already watches. A child waiting on an in-flight LLM completion is ALIVE: its activity clock ticks while the
  274. provider works (``direct_api_call``'s 15s heartbeat) and the call itself is bounded by the per-call stale
  275. watchdog, so at the delegation layer it must not read as stuck (#116001)."""
  276. try:
  277. summary = child.get_activity_summary() or {}
  278. except Exception:
  279. return (None, None, None)
  280. return (summary.get("api_call_count"), summary.get("current_tool"), summary.get("last_activity_ts"))
  281. def _child_last_event_age(child: Any) -> Optional[float]:
  282. """Seconds since the child's activity clock last ticked, or None when the child exposes no clock. Reported on a
  283. timeout so an operator can tell a slow-but-live provider from a runaway without transcript forensics."""
  284. try:
  285. ts = (child.get_activity_summary() or {}).get("last_activity_ts")
  286. return round(max(0.0, time.time() - float(ts)), 2) if ts is not None else None
  287. except Exception:
  288. return None
  289. class _Heartbeat:
  290. """One child's parent-activity heartbeat via the shared periodic scheduler
  291. (``agent.periodic_scheduler``) — not one daemon thread per child. NOT started at construction:
  292. the caller calls ``start()`` inside its ``try`` so a failed schedule (OS thread exhaustion on
  293. first use) leaves ``handle`` None and ``stop()`` is a no-op."""
  294. def __init__(self, child: Any, parent_agent: Any, task_index: int):
  295. self.child, self.parent_agent, self.task_index = child, parent_agent, task_index
  296. # Stale detection: a cycle counts as stale when (tool, iteration,
  297. # activity_ts) all froze; thresholds differ idle vs in-tool.
  298. self.last_seen = {"iter": 0, "tool": None, "ts": None, "stale": 0}
  299. self.handle = None
  300. # Set on the stale verdict; ``await_child`` waits on it (its worker's done-callback
  301. # sets it too) so a wedged child ends the wait instead of only ending the heartbeat.
  302. self.settled = threading.Event()
  303. # Threshold (seconds of frozen activity) the stale verdict fired at; None until it does.
  304. # ``await_child`` reads it to name the real cause when a configured cap was still pending.
  305. self.stale_threshold_seconds: Optional[float] = None
  306. def start(self) -> None:
  307. from agent.periodic_scheduler import schedule
  308. from tools.delegate_tool import _HEARTBEAT_INTERVAL
  309. self.handle = schedule(self.tick, _HEARTBEAT_INTERVAL)
  310. def stop(self) -> None:
  311. """wait=5 mirrors the old thread join: an in-flight tick finishes."""
  312. if self.handle is not None:
  313. self.handle.cancel(wait=5)
  314. def tick(self):
  315. """Returning False stops the periodic callback."""
  316. from tools.delegate_tool import _HEARTBEAT_INTERVAL, _HEARTBEAT_STALE_CYCLES_IDLE, _HEARTBEAT_STALE_CYCLES_IN_TOOL
  317. child, parent_agent, task_index, last_seen = self.child, self.parent_agent, self.task_index, self.last_seen
  318. touch = getattr(parent_agent, "_touch_activity", None) if parent_agent is not None else None
  319. if not touch:
  320. return None
  321. desc = f"delegate_task: subagent {task_index} working"
  322. try:
  323. child_summary = child.get_activity_summary()
  324. child_tool = child_summary.get("current_tool")
  325. child_iter = child_summary.get("api_call_count", 0)
  326. child_max = child_summary.get("max_iterations", 0)
  327. child_activity_ts = child_summary.get("last_activity_ts")
  328. # A slow model wait refreshes last_activity_ts (direct_api_call
  329. # heartbeat), so it never looks stale at the idle threshold.
  330. activity_advanced = child_activity_ts is not None and (
  331. last_seen["ts"] is None or child_activity_ts > last_seen["ts"]
  332. )
  333. if child_iter > last_seen["iter"] or child_tool != last_seen["tool"] or activity_advanced:
  334. last_seen.update(iter=child_iter, tool=child_tool, stale=0)
  335. if child_activity_ts is not None:
  336. last_seen["ts"] = child_activity_ts
  337. else:
  338. last_seen["stale"] += 1
  339. stale_cycles = _HEARTBEAT_STALE_CYCLES_IN_TOOL if child_tool else _HEARTBEAT_STALE_CYCLES_IDLE
  340. if last_seen["stale"] >= stale_cycles:
  341. logger.warning(
  342. "Subagent %d appears stale (no progress for %d heartbeat cycles, tool=%s) — abandoning its wait",
  343. task_index, last_seen["stale"], child_tool or "<none>",
  344. )
  345. # A finite/-Q turn has no gateway watchdog behind this; the wait itself must end (#109749).
  346. self.stale_threshold_seconds = stale_cycles * _HEARTBEAT_INTERVAL
  347. self.settled.set()
  348. return False
  349. if child_tool:
  350. desc = f"delegate_task: subagent running {child_tool} (iteration {child_iter}/{child_max})"
  351. elif child_summary.get("last_activity_desc", ""):
  352. desc = f"delegate_task: subagent {child_summary.get('last_activity_desc', '')} (iteration {child_iter}/{child_max})"
  353. except Exception:
  354. pass
  355. with _quiet(None):
  356. touch(desc)
  357. return None
  358. def _start_heartbeat(child: Any, parent_agent: Any, task_index: int) -> _Heartbeat:
  359. """Build (not start) one child's heartbeat; see ``_Heartbeat``."""
  360. return _Heartbeat(child, parent_agent, task_index)
  361. def _register_child(
  362. child: Any, parent_agent: Any, goal: str, *, owner_session_id: Optional[str], owner_transport: Any,
  363. owner_session_record: Any,
  364. ) -> Optional[str]:
  365. """Register the live child in the module registry; return its subagent_id. Test doubles without a stable string
  366. ``_subagent_id`` are not registered (None) and the caller skips every registry interaction for them."""
  367. _subagent_id = getattr(child, "_subagent_id", None)
  368. if not isinstance(_subagent_id, str) or not _subagent_id:
  369. return None
  370. if owner_session_id is None:
  371. with _quiet(None):
  372. from gateway.session_context import get_session_env
  373. owner_session_id = get_session_env("HERMES_UI_SESSION_ID", "") or None
  374. if owner_session_id and (owner_transport is None or owner_session_record is None):
  375. owner_transport, owner_session_record = _capture_gateway_steer_authority(owner_session_id)
  376. _raw_depth = getattr(child, "_delegate_depth", 1)
  377. _register_subagent({
  378. "subagent_id": _subagent_id,
  379. "parent_id": _str_or_none(getattr(child, "_parent_subagent_id", None)),
  380. "depth": max(0, _raw_depth - 1) if isinstance(_raw_depth, int) else 0,
  381. "goal": goal,
  382. "delegation_id": _str_or_none(getattr(child, "_delegation_id", None)),
  383. "model": _str_or_none(getattr(child, "model", None)),
  384. "started_at": time.time(), "status": "running", "tool_count": 0, "agent": child,
  385. # Owning conversation's durable session id (same lineage completion delivery routes by), sourced from the
  386. # child's stamp so it survives a parent_agent rebuild between dispatch and run; used for list/steer/stop
  387. # ownership when the weakref chain breaks.
  388. "owner_agent_session_id": (
  389. str(getattr(child, "_parent_session_id", "") or "") or str(getattr(parent_agent, "session_id", "") or "") or None
  390. ),
  391. # Immutable live gateway/TUI session that commissioned this child.
  392. # Empty outside those hosts; RPC authority fails closed.
  393. "owner_session_id": owner_session_id,
  394. "owner_transport": owner_transport,
  395. "owner_session_record": owner_session_record,
  396. })
  397. return _subagent_id
  398. def _create_isolated_worktree(parent_agent: Any, parent_task_id: Any, subagent_id: Optional[str]):
  399. """Opt-in worktree isolation: own git worktree off the parent's HEAD (the
  400. child's terminal starts there). Git-only, local-backend-only; failures
  401. degrade silently to the shared workspace. Returns the worktree info or None."""
  402. from tools.delegate_tool import _get_worktree_isolation, _resolve_workspace_hint
  403. if not _get_worktree_isolation():
  404. return None
  405. with _quiet("worktree isolation setup failed: %s"):
  406. from tools import subagent_worktree
  407. if not subagent_worktree.local_backend_active():
  408. logger.debug("worktree isolation skipped: non-local terminal backend")
  409. return None
  410. _parent_cwd = None
  411. with _quiet(None):
  412. from tools.terminal_tool import get_session_cwd as _gsc
  413. _parent_cwd = _gsc(parent_task_id)
  414. return subagent_worktree.create_subagent_worktree(
  415. _parent_cwd or _resolve_workspace_hint(parent_agent), subagent_id=subagent_id,
  416. )
  417. return None
  418. def _defer_close_after_timeout(child: Any, child_future: Any) -> None:
  419. """Hand ``child.close()`` to a Future done-callback and drain its transports.
  420. The interrupt is cooperative: the worker still runs its finally path, so closing now could close SQLite under its
  421. final write — the done-callback is the first safe boundary. The abandoned worker is usually parked in an OpenSSL
  422. read; NEVER hard-close that transport from this thread (cross-thread FD release under a live SSL read corrupts
  423. native state) — shutdown() the pooled sockets so the read settles with EOF and the worker unwinds. One immediate
  424. sweep + one delayed re-sweep for a connection opened in between; a worker that still won't settle keeps its
  425. resources until process exit.
  426. """
  427. child_future.add_done_callback(lambda _done: _close_child(child, "Failed to close timed-out child after worker exit"))
  428. # Bounded drain (#94248 native half): the deferred close above only fires once the abandoned worker
  429. # unwinds, but that worker is typically parked inside an in-flight OpenSSL read (Codex / httpx). Never
  430. # hard-close that transport from this thread — releasing FDs under a live SSL read is the #29507/#70773
  431. # native-corruption family. Instead shutdown() the child's pooled sockets, which is FD-safe from any
  432. # thread and settles the blocked read with EOF/EPIPE so the worker can unwind and trigger the deferred
  433. # close.
  434. _drain = getattr(child, "_drain_transports_after_abandonment", None)
  435. if not callable(_drain):
  436. return
  437. def _drain_once(phase: str) -> None:
  438. with _quiet("Timed-out child transport drain (%s) failed", phase, exc_info=True):
  439. _drain(reason=f"delegate_timeout_{phase}")
  440. _drain_once("immediate")
  441. _resweep_timer = threading.Timer(5.0, lambda: None if child_future.done() else _drain_once("resweep"))
  442. _resweep_timer.daemon = True
  443. _resweep_timer.start()
  444. def _lease_child_credential(child: Any) -> tuple[Any, Optional[str]]:
  445. """Lease a credential from the child's pool (if any) and bind it; ``(pool, lease_id)``. The bound entry must
  446. serve the child's endpoint: on a mixed same-provider pool the least-leased pick may target another host, so it is
  447. released and an endpoint-matching entry is leased by id instead (#68237). Child work, so it runs in the child's
  448. profile scope: a lease may refresh that profile's OAuth entries, and ``_swap_credential`` re-reads TLS/header
  449. config for the bound endpoint."""
  450. child_pool = getattr(child, "_credential_pool", None)
  451. if child_pool is None:
  452. return None, None
  453. from agent.credential_pool import credential_pool_entry_serves_endpoint as _entry_serves_endpoint
  454. from tools.delegate_tool_routes import child_profile_scope
  455. base_url = getattr(child, "base_url", None)
  456. with child_profile_scope(child):
  457. leased_cred_id = child_pool.acquire_lease()
  458. if leased_cred_id is not None:
  459. with _quiet("Failed to bind child to leased credential: %s"):
  460. # Resolve the leased entry by id: the pool is shared with the parent/siblings, so current() is a
  461. # mutable cursor that may already point at someone else's pick.
  462. leased_entry = next((e for e in child_pool.entries() if e.id == leased_cred_id), None)
  463. if not _entry_serves_endpoint(leased_entry, base_url):
  464. child_pool.release_lease(leased_cred_id)
  465. leased_entry = next(
  466. (e for e in child_pool.entries() if e.last_status != "dead" and _entry_serves_endpoint(e, base_url)),
  467. None,
  468. )
  469. leased_cred_id = child_pool.acquire_lease(leased_entry.id) if leased_entry is not None else None
  470. if leased_entry is not None and hasattr(child, "_swap_credential"):
  471. child._swap_credential(leased_entry)
  472. return child_pool, leased_cred_id
  473. def _merge_late_steer(result: dict[str, Any], subagent_id: Optional[str], child: Any) -> None:
  474. """Linearization boundary for registry steering: from here the child cannot consume another steer. Closing under
  475. the registry lock either rejects a concurrent caller or drains every accepted exact text into the result before
  476. callbacks/result assembly run."""
  477. late = _close_subagent_steering(subagent_id, child) if subagent_id else None
  478. if late:
  479. existing = result.get("pending_steer")
  480. result["pending_steer"] = f"{existing}\n{late}" if isinstance(existing, str) and existing else late
  481. @dataclass
  482. class _SchemaOutcome:
  483. schema: Optional[dict[str, Any]]
  484. valid: Optional[bool]
  485. errors: list[str]
  486. retries: int
  487. def _validate_child_output_schema(
  488. child: Any, result: dict[str, Any], task_index: int, child_task_id: str, relay_child_text: Any
  489. ) -> _SchemaOutcome:
  490. """Validate the final answer against the attached output_schema with ONE bounded retry. Schema-less children (no
  491. dict on ``child._delegate_output_schema``) take no branch here so their result entry stays byte-identical."""
  492. _output_schema = getattr(child, "_delegate_output_schema", None)
  493. if not isinstance(_output_schema, dict):
  494. return _SchemaOutcome(_output_schema, None, [], 0)
  495. from tools.delegation_output_schema import build_retry_message, validate_output
  496. _first_text = result.get("final_response") or ""
  497. _schema_valid, _schema_errors = validate_output(_first_text, _output_schema)
  498. if _schema_valid or not _first_text.strip() or result.get("interrupted", False):
  499. return _SchemaOutcome(_output_schema, _schema_valid, _schema_errors, 0)
  500. # Exactly one retry turn, carrying the validation errors verbatim (no
  501. # schema re-paste — the child already holds the contract in its context).
  502. _retry_result = None
  503. try:
  504. # Same identity as the main child turn: this runs on the parent worker's thread, and an
  505. # unmarked turn is misread as the dispatcher-owned worker by every HERMES_KANBAN_* gate.
  506. from agent.delegation_context import delegated_child_context
  507. from tools.delegate_tool_routes import child_profile_scope
  508. with child_profile_scope(child), delegated_child_context(str(getattr(child, "session_id", "") or "")):
  509. _retry_result = child.run_conversation(
  510. user_message=build_retry_message(_schema_errors), task_id=child_task_id,
  511. stream_callback=relay_child_text,
  512. )
  513. except Exception as _retry_exc:
  514. logger.warning("Subagent %d schema-retry turn failed: %s", task_index, _retry_exc)
  515. if isinstance(_retry_result, dict):
  516. _retry_text = _retry_result.get("final_response") or ""
  517. if _retry_text.strip():
  518. result["final_response"] = _retry_text
  519. try:
  520. result["api_calls"] = int(result.get("api_calls", 0) or 0) + int(_retry_result.get("api_calls", 0) or 0)
  521. except (TypeError, ValueError):
  522. pass
  523. _retry_messages = _retry_result.get("messages")
  524. if isinstance(_retry_messages, list) and isinstance(result.get("messages"), list):
  525. result["messages"] = result["messages"] + _retry_messages
  526. _schema_valid, _schema_errors = validate_output(_retry_text, _output_schema)
  527. return _SchemaOutcome(_output_schema, _schema_valid, _schema_errors, 1)
  528. def _build_tool_trace(messages: Any) -> list[dict[str, Any]]:
  529. """Tool trace from the child's conversation messages, pairing parallel
  530. tool calls with their results by tool_call_id."""
  531. tool_trace: list[dict[str, Any]] = []
  532. trace_by_id: dict[str, dict[str, Any]] = {}
  533. if not isinstance(messages, list):
  534. return tool_trace
  535. for msg in messages:
  536. if not isinstance(msg, dict):
  537. continue
  538. if msg.get("role") == "assistant":
  539. for tc in msg.get("tool_calls") or []:
  540. fn = tc.get("function", {})
  541. arguments = fn.get("arguments", "")
  542. entry_t = {
  543. "tool": fn.get("name", "unknown"), "args_bytes": len(arguments),
  544. "input_summary": _summarize_tool_arguments(arguments),
  545. }
  546. tool_trace.append(entry_t)
  547. if tc.get("id"):
  548. trace_by_id[tc["id"]] = entry_t
  549. elif msg.get("role") == "tool":
  550. content = _stringify_tool_content(msg.get("content", ""))
  551. result_meta = {"result_bytes": len(content), "status": "error" if _looks_like_error_output(content) else "ok"}
  552. tc_id = msg.get("tool_call_id")
  553. target = trace_by_id.get(tc_id) if tc_id else None
  554. if target is not None:
  555. target.update(result_meta)
  556. elif tool_trace:
  557. tool_trace[-1].update(result_meta) # no tool_call_id: pair with the latest call
  558. return tool_trace
  559. def _build_result_entry(
  560. child: Any, result: dict[str, Any], task_index: int, duration: float, schema: _SchemaOutcome,
  561. ) -> dict[str, Any]:
  562. """Parent-visible result entry (status, exit_reason, tool trace, tokens, cost).
  563. ``status``/``exit_reason``/``truncated`` follow the ``_run_single_child`` contract; a structured failure always
  564. wins over the summary-presence heuristic (a fallback for legacy/mock results only)."""
  565. summary = result.get("final_response") or ""
  566. # "(empty)" is run_agent's give-up sentinel after repeated empty LLM
  567. # responses (usually a transport bug) — a failure, not a success.
  568. usable_summary = bool(summary) and summary.strip() != "(empty)"
  569. interrupt_note = ""
  570. if result.get("interrupted", False):
  571. status, exit_reason = "interrupted", "interrupted"
  572. # The loop's final_response is a placeholder here ("Operation interrupted…", also appended as the closing
  573. # assistant row); the completion must carry what the child actually had so far — its last real assistant
  574. # text — and keep the placeholder as the error.
  575. from agent.message_content import flatten_message_text
  576. placeholders = {"", summary.strip(), "Operation interrupted."}
  577. partial = next((t for m in reversed(result.get("messages") or []) if m.get("role") == "assistant"
  578. and (t := flatten_message_text(m.get("content")).strip()) not in placeholders), "")
  579. if partial:
  580. interrupt_note, summary = summary.strip(), partial
  581. elif result.get("failed") or result.get("error"):
  582. # The loop returns the error text as final_response, which would otherwise read as "completed". Never report a
  583. # provider rejection as "max_iterations" — that is only truthful for real budget exhaustion.
  584. status, exit_reason = "failed", "error"
  585. else:
  586. # exit_reason ("completed" vs "max_iterations") tells the parent HOW the task ended; completed=False with no
  587. # failure = budget exhaustion. A declared schema still violated after the bounded retry does NOT fail the
  588. # run: the child's raw final text is the deliverable (audits of up to 68 min were written off as "failed"
  589. # over a stray code fence or one missing field); ``schema_valid: false`` + ``schema_errors`` carry the
  590. # contract verdict, and the summary is prefixed with a notice so a status-only reader cannot mistake it
  591. # for validated output.
  592. exit_reason = "completed" if result.get("completed", False) else "max_iterations"
  593. status = "completed" if usable_summary else "failed"
  594. _cost = getattr(child, "session_estimated_cost_usd", 0.0)
  595. _cost_status = getattr(child, "session_cost_status", None)
  596. # Result entry contract: see the _run_single_child docstring.
  597. entry: dict[str, Any] = {
  598. "task_index": task_index,
  599. "status": status,
  600. "summary": summary,
  601. "api_calls": result.get("api_calls", 0),
  602. "duration_seconds": duration,
  603. "model": _str_or_none(getattr(child, "model", None)),
  604. "exit_reason": exit_reason,
  605. # A budget-exhausted child still returns a summary (status stays
  606. # "completed"), so the parent needs this explicit flag.
  607. "truncated": exit_reason == "max_iterations",
  608. "tokens": {
  609. "input": _num(getattr(child, "session_prompt_tokens", 0)),
  610. "output": _num(getattr(child, "session_completion_tokens", 0)),
  611. },
  612. "tool_trace": _build_tool_trace(result.get("messages") or []),
  613. # Captured before the finally block calls child.close() so the parent thread can fire subagent_stop with the
  614. # correct role; stripped before the dict is serialised back to the model (as is _child_cost_usd, folded into
  615. # the parent's session cost by the aggregator).
  616. "_child_role": getattr(child, "_delegate_role", None),
  617. "_child_cost_usd": float(_cost or 0.0) if isinstance(_cost, (int, float)) else 0.0,
  618. }
  619. # Model-visible per-delegation spend (unlike _child_cost_usd above).
  620. entry["cost_usd"] = round(entry["_child_cost_usd"], 6)
  621. entry["cost_status"] = _cost_status if isinstance(_cost_status, str) and _cost_status else "unknown"
  622. _route = _route_report(child)
  623. if _route is not None:
  624. entry["route"] = _route
  625. if status == "failed":
  626. entry["error"] = result.get("error", "Subagent did not produce a response.")
  627. # Classified reason from the child loop (e.g. "rate_limit", "billing")
  628. # lets the parent tell a quota wall from a task error without parsing prose.
  629. _failure_reason = result.get("failure_reason")
  630. if isinstance(_failure_reason, str) and _failure_reason:
  631. entry["failure_reason"] = _failure_reason
  632. elif interrupt_note:
  633. entry["error"] = interrupt_note
  634. # Schema-validation outcome — emitted ONLY when a schema was requested, so
  635. # legacy (schema-less) payloads keep their exact shape.
  636. if isinstance(schema.schema, dict):
  637. entry["schema_valid"] = bool(schema.valid)
  638. if schema.retries:
  639. entry["schema_retries"] = schema.retries
  640. if not schema.valid and schema.errors:
  641. entry["schema_errors"] = schema.errors
  642. if schema.valid is False and usable_summary:
  643. entry["schema_note"] = (
  644. "Final answer does not satisfy the declared output_schema"
  645. + (" (after 1 retry)" if schema.retries else "")
  646. + "; `summary` is the child's raw, UNVALIDATED final text — extract what you need from it "
  647. "yourself (see schema_errors) rather than re-running the task."
  648. )
  649. # A steer queued after the final assistant turn had no tool batch to land
  650. # in; name it so the parent sees it was MISSED rather than silently absorbed.
  651. _missed_steer = result.get("pending_steer")
  652. if isinstance(_missed_steer, str) and _missed_steer.strip():
  653. entry["missed_steer"] = _missed_steer
  654. _miss_note = ("[steer did not land — the subagent finished before it could " f"be delivered: {_missed_steer}]")
  655. entry["summary"] = f"{summary}\n\n{_miss_note}" if summary else _miss_note
  656. return entry
  657. def _is_image_url(ref: str) -> bool:
  658. return ref.startswith(("http://", "https://", "data:image/"))
  659. def _build_child_goal_message(goal: str, images: list[str], child) -> Any:
  660. """The child's first user message when a task forwards ``images``.
  661. Routing reuses the inbound-image policy (``agent.image_routing``, honouring ``agent.image_input_mode``): a
  662. vision-capable child gets an OpenAI-style content list (text part + one ``image_url`` part per image; local files
  663. as data URLs behind the read guard, http(s)/data URLs verbatim); otherwise the goal gains ``[Image attached …]``
  664. hint lines for ``vision_analyze``. Any failure degrades to the text-only goal so image plumbing never breaks a
  665. spawn — logged at warning since the caller asked for the images.
  666. """
  667. try:
  668. # data: URLs ride as image parts only — their base64 never goes into the text hint or a text-mode goal.
  669. data_urls = [s for s in images if s.startswith("data:image/")]
  670. urls = [s for s in images if _is_image_url(s) and s not in data_urls]
  671. paths = [s for s in images if not _is_image_url(s)]
  672. from agent.image_routing import build_native_content_parts, decide_image_input_mode
  673. cfg = None
  674. with _quiet(None):
  675. from hermes_cli.config import load_config_readonly
  676. cfg = load_config_readonly()
  677. mode = decide_image_input_mode(
  678. str(getattr(child, "provider", "") or ""), str(getattr(child, "model", "") or ""), cfg,
  679. requested_provider=str(getattr(child, "requested_provider", "") or ""),
  680. )
  681. if mode == "native":
  682. parts, skipped = build_native_content_parts(goal, paths, urls)
  683. if skipped:
  684. logger.warning("delegate_task: skipped %d unreadable image(s) for subagent: %s", len(skipped), ", ".join(skipped[:3]))
  685. if data_urls:
  686. parts = (parts or [{"type": "text", "text": goal}]) + [{"type": "image_url", "image_url": {"url": u}} for u in data_urls]
  687. return parts if any(p.get("type") == "image_url" for p in parts) else goal
  688. if data_urls:
  689. logger.warning("delegate_task: %d inline data-URL image(s) dropped for a non-vision subagent", len(data_urls))
  690. hints: list[str] = []
  691. for p in paths:
  692. if os.path.isfile(p):
  693. hints.append(f"[Image attached at: {p}]")
  694. else:
  695. logger.warning("delegate_task: image path not found, not forwarded: %s", p)
  696. hints.extend(f"[Image attached: {u}]" for u in urls)
  697. if not hints:
  698. return goal
  699. return goal + "\n\n" + "\n".join(hints) + "\nUse vision_analyze to inspect these images."
  700. except Exception:
  701. logger.warning("delegate_task: image forwarding failed; sending text-only goal", exc_info=True)
  702. return goal
  703. @dataclass
  704. class _ChildRun:
  705. """State of one child run, shared by every phase of ``_run_single_child``.
  706. ``worktree_info`` stays None until isolation engages (``attach_worktree`` is
  707. then a no-op on every early error path); ``seed_workspace`` sets the rest."""
  708. child: Any
  709. parent_agent: Any
  710. task_index: int
  711. goal: str
  712. subagent_id: Optional[str]
  713. child_progress_cb: Any
  714. heartbeat: Any = None
  715. child_start: float = field(default_factory=time.monotonic)
  716. worktree_info: Optional[dict[str, str]] = None
  717. child_task_id: str = ""
  718. parent_task_id: Optional[str] = None
  719. wall_start: float = 0.0
  720. parent_reads_snapshot: list = field(default_factory=list)
  721. def elapsed(self) -> float:
  722. return round(time.monotonic() - self.child_start, 2)
  723. def relay_text(self, delta: str) -> None:
  724. """Stream callback forwarding the child's reply text up the progress relay so gateway watch windows mirror it
  725. live (subagent.text → message.delta). Inert under CLI/TUI: their progress handlers ignore non-tool events."""
  726. if delta:
  727. _safe_progress(self.child_progress_cb, "subagent.text", preview=delta)
  728. def attach_worktree(self, entry_dict: dict[str, Any]) -> dict[str, Any]:
  729. """Inspect + prune the child worktree, reporting into the entry (no-op without isolation)."""
  730. info = self.worktree_info
  731. if info is None:
  732. return redact_child_result(entry_dict, getattr(self.child, "_delegate_output_schema", None), force=False)
  733. from tools import subagent_worktree
  734. try:
  735. entry_dict["worktree"] = subagent_worktree.finalize_subagent_worktree(info)
  736. except Exception as e:
  737. # State is unknown: emit the SAME flagged schema the parent expects,
  738. # via the shared factory so the two producers never drift.
  739. logger.warning("worktree finalize failed: %s", e)
  740. entry_dict["worktree"] = subagent_worktree.unproven_worktree_payload(info, f"finalize raised: {e}")
  741. return redact_child_result(entry_dict, getattr(self.child, "_delegate_output_schema", None), force=False)
  742. def seed_workspace(self) -> None:
  743. """Seed cwd/container aliases and optional worktree isolation for the child;
  744. ``goal`` is extended with the worktree contract note when isolation engaged."""
  745. import uuid as _uuid
  746. self.child_task_id = self.subagent_id or f"subagent-{self.task_index}-{_uuid.uuid4().hex[:8]}"
  747. self.parent_task_id = getattr(self.parent_agent, "_current_task_id", None)
  748. # Seed the child's cwd record from the parent's: same starting directory,
  749. # but the child's later `cd`s stay in its own record. Per-session container
  750. # isolation keys containers by task_id; the child must share the PARENT's.
  751. # The parent's record is read in the parent's scope; the child's is written in the child's (cwd records
  752. # are keyed by the active profile, and a profile-routed child looks its own up as that profile).
  753. from tools.delegate_tool_routes import child_profile_scope
  754. with _quiet("Child cwd seed failed: %s"):
  755. from tools.terminal_tool import get_session_cwd, record_session_cwd, register_container_alias
  756. parent_cwd = get_session_cwd(self.parent_task_id)
  757. with child_profile_scope(self.child):
  758. record_session_cwd(self.child_task_id, parent_cwd)
  759. register_container_alias(self.child_task_id, self.parent_task_id)
  760. self.worktree_info = _create_isolated_worktree(self.parent_agent, self.parent_task_id, self.subagent_id)
  761. if self.worktree_info is not None:
  762. with _quiet("worktree cwd seed failed: %s"), child_profile_scope(self.child):
  763. from tools.terminal_tool import record_session_cwd as _rsc
  764. _rsc(self.child_task_id, self.worktree_info["path"])
  765. # The child's context is already built; carry the isolation contract on
  766. # the goal message instead (same turn, no system-prompt mutation).
  767. from tools.subagent_worktree import build_worktree_context_note
  768. self.goal = self.goal + build_worktree_context_note(self.worktree_info)
  769. self.wall_start = time.time()
  770. self.parent_reads_snapshot = list(file_state.known_reads(self.parent_task_id)) if self.parent_task_id else []
  771. def finish_failed(
  772. self, entry: dict[str, Any], late_steer: Optional[str], *, preview: str, summary: str = "", status: Optional[str] = None,
  773. ) -> dict[str, Any]:
  774. """Shared tail of every failure path: emit ``subagent.complete`` (``status`` defaults to the entry's), note
  775. the steer text that won the race with the failure, report the worktree."""
  776. _safe_progress(
  777. self.child_progress_cb, "subagent.complete", preview=redact_child_text(preview, force=False), status=status or entry["status"],
  778. duration_seconds=entry["duration_seconds"],
  779. summary=redact_child_text(summary, force=False) if summary is not None else None,
  780. )
  781. _append_missed_steer(entry, late_steer)
  782. return self.attach_worktree(entry)
  783. def close_steering(self) -> Optional[str]:
  784. """Close steer acceptance (see ``_merge_late_steer``); returns late steer text, if any."""
  785. return _close_subagent_steering(self.subagent_id, self.child) if self.subagent_id else None
  786. def wait_liveness_aware(self, settled: threading.Event, child_future: Any, child_timeout: Optional[float]) -> None:
  787. """Wait for the worker or the stale verdict, where ``child_timeout`` bounds INACTIVITY, not total runtime.
  788. A configured cap used to be a dispatch-to-death stopwatch, so it only ever killed children the runtime had
  789. already judged healthy: all 75 reported deaths happened while waiting on an in-flight LLM completion, none
  790. mid-tool (#116001). A provider serving multi-minute completions is progress — the child's activity clock
  791. ticks during the wait and the request is bounded by the per-call stale watchdog — so the budget restarts on
  792. every sign of progress, exactly the signals the heartbeat's stale verdict reads. Genuinely frozen children
  793. keep two authorities: this budget (when configured) and the heartbeat's stale threshold.
  794. """
  795. if child_timeout is None:
  796. settled.wait()
  797. return
  798. deadline = time.monotonic() + child_timeout
  799. warn_at = deadline - child_timeout * (1.0 - _BUDGET_WARNING_FRACTION)
  800. warned = False
  801. fingerprint = _child_activity_fingerprint(self.child)
  802. while True:
  803. now = time.monotonic()
  804. remaining = deadline - now
  805. if remaining <= 0:
  806. return # no progress for the whole budget
  807. wait = min(_LIVENESS_POLL_SECONDS, remaining)
  808. if not warned:
  809. wait = min(wait, max(warn_at - now, 0.0))
  810. settled.wait(timeout=wait)
  811. if settled.is_set():
  812. return # the worker finished, or the heartbeat declared the child stale
  813. current = _child_activity_fingerprint(self.child)
  814. if current != fingerprint:
  815. fingerprint = current
  816. deadline = time.monotonic() + child_timeout
  817. warn_at = deadline - child_timeout * (1.0 - _BUDGET_WARNING_FRACTION)
  818. warned = False # a fresh window gets its own warning
  819. elif not warned and time.monotonic() >= warn_at:
  820. warned = True
  821. _warn_child_budget(self.child, child_timeout - (deadline - time.monotonic()), child_timeout)
  822. def await_child(self) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]], bool]:
  823. """Run the child's conversation on a daemon worker: ``(result, None, False)`` or ``(None, error_entry,
  824. close_deferred)`` on timeout/exception.
  825. Hard timeout is off by default (``result(timeout=None)``; stuck children are the heartbeat's job). Daemon
  826. worker: an abandoned timed-out child on a non-daemon thread would block interpreter exit at atexit join. The
  827. worker installs a non-interactive approval callback (deny/approve per delegation.subagent_auto_approve) so
  828. dangerous-command prompts never fall back to ``input()`` and deadlock the parent TUI. On failure: steer
  829. acceptance closes BEFORE the stop signal (a concurrent steer is drained into the entry or rejected, never
  830. lost); a 0-API-call timeout gets a diagnostic dump; a worker that still owns the child gets ``child.close()``
  831. via a Future done-callback (``close_deferred=True``) — closing here would race its still-unwinding finally
  832. path.
  833. """
  834. from tools.delegate_tool import (
  835. _STALE_RESULT_GRACE_SECONDS, _get_child_timeout, _get_subagent_approval_callback, _set_subagent_approval_cb,
  836. )
  837. from tools.daemon_pool import DaemonThreadPoolExecutor
  838. child, task_index = self.child, self.task_index
  839. child_timeout = _get_child_timeout()
  840. executor = DaemonThreadPoolExecutor(
  841. max_workers=1, initializer=_set_subagent_approval_cb, initargs=(_get_subagent_approval_callback(),),
  842. )
  843. # Worker thread handle so the timeout diagnostic can dump its stack.
  844. worker_thread_holder: dict[str, Optional[threading.Thread]] = {"t": None}
  845. # Resolved after seed_workspace so a multimodal goal's text part carries the worktree note too.
  846. _images = list(getattr(child, "_delegate_images", None) or [])
  847. user_message: Any = _build_child_goal_message(self.goal, _images, child) if _images else self.goal
  848. def _run_with_thread_capture():
  849. worker_thread_holder["t"] = threading.current_thread()
  850. from agent.delegation_context import delegated_child_context
  851. from tools.delegate_tool_routes import child_profile_scope
  852. # A profile-routed child converses as that profile (config, secrets, skills, terminal policy).
  853. with child_profile_scope(child), delegated_child_context(str(getattr(child, "session_id", "") or "")):
  854. return child.run_conversation(
  855. user_message=user_message, task_id=self.child_task_id, stream_callback=self.relay_text,
  856. )
  857. future = executor.submit(contextvars.copy_context().run, _run_with_thread_capture)
  858. # One wait covers both ways out: the worker finishing, or the heartbeat's stale verdict.
  859. # Without the second, a worker wedged after its final answer holds a finite (-Q / Bot Chat
  860. # one-shot) turn — and its session lease — forever, since that runtime has no gateway
  861. # inactivity watchdog (#109749). The configured cap bounds INACTIVITY (see wait_liveness_aware).
  862. settled = self.heartbeat.settled if self.heartbeat is not None else threading.Event()
  863. future.add_done_callback(lambda _f: settled.set())
  864. # Set when the stale verdict — not the configured cap — ended the wait; the entry must name that cause.
  865. stale_after: Optional[float] = None
  866. try:
  867. self.wait_liveness_aware(settled, future, child_timeout)
  868. if not future.done():
  869. stale_after = getattr(self.heartbeat, "stale_threshold_seconds", None)
  870. if stale_after is None:
  871. # A configured inactivity cap elapsing keeps its teeth: the user armed a
  872. # budget, the fingerprint never moved, abandon now (#116001 contract).
  873. raise FuturesTimeoutError()
  874. # Grace collection (#113222): the heartbeat's stale verdict ended the wait
  875. # while the worker was still pending — a child that already wrote its final
  876. # answer often lands its result moments later, and that real result must be
  877. # collected instead of a synthesized timeout that strands a finished batch
  878. # as `running`. Signal the cooperative stop (a stoppable worker can then
  879. # unwind inside the window), poll the future for the grace period, and only
  880. # fall through to the timeout path if nothing lands. Steering stays OPEN
  881. # here on purpose: the success path's `_merge_late_steer` (or the failure
  882. # path below) is the linearization boundary, so a concurrent steer is never
  883. # lost either way.
  884. _signal_child_stop(child)
  885. grace = _STALE_RESULT_GRACE_SECONDS
  886. grace_done = threading.Event()
  887. future.add_done_callback(lambda _f: grace_done.set())
  888. if not grace_done.wait(timeout=grace) or not future.done():
  889. raise FuturesTimeoutError()
  890. logger.info(
  891. "Subagent %d settled inside the %.1fs stale-result grace window — collecting its real result",
  892. task_index, grace,
  893. )
  894. return future.result(), None, False
  895. return future.result(), None, False
  896. except Exception as wait_exc:
  897. exc: BaseException = wait_exc # ``as`` targets are unbound after the except block
  898. finally:
  899. # Shut down without waiting — a child stuck on blocking I/O would hang wait=True forever.
  900. executor.shutdown(wait=False)
  901. _late_pending_steer = self.close_steering()
  902. _signal_child_stop(child)
  903. is_timeout = isinstance(exc, (FuturesTimeoutError, TimeoutError))
  904. # What actually ended the wait: the stale threshold pre-empts a longer configured cap.
  905. timeout_cause = stale_after if stale_after is not None else child_timeout
  906. duration = self.elapsed()
  907. logger.warning("Subagent %d %s after %.1fs", task_index, "timed out" if is_timeout else f"raised {type(exc).__name__}", duration)
  908. child_api_calls = 0
  909. with _quiet(None):
  910. child_api_calls = int(child.get_activity_summary().get("api_call_count", 0) or 0)
  911. # A timeout BEFORE any API call is a black box without a diagnostic dump.
  912. before_first_call = is_timeout and child_api_calls == 0
  913. diagnostic_path: Optional[str] = None
  914. if before_first_call:
  915. diagnostic_path = _dump_subagent_timeout_diagnostic(
  916. child=child, task_index=task_index,
  917. # A stale verdict or a configured cap; ``or 0.0`` guards the type checker.
  918. timeout_seconds=float(timeout_cause or 0.0), duration_seconds=float(duration),
  919. worker_thread=worker_thread_holder.get("t"), goal=self.goal,
  920. )
  921. if diagnostic_path:
  922. logger.warning("Subagent %d 0-API-call timeout — diagnostic written to %s", task_index, diagnostic_path)
  923. if not is_timeout:
  924. _err = str(exc)
  925. elif stale_after is not None:
  926. _err = (
  927. f"Subagent stopped making progress after {child_api_calls} API call(s) — no activity for "
  928. f"{stale_after:g}s (heartbeat stale threshold); the pending worker was abandoned."
  929. )
  930. elif before_first_call:
  931. _err = (
  932. f"Subagent timed out after {child_timeout}s without making any API call — the child never reached its "
  933. f"first LLM request (prompt construction, credential resolution, or transport may be stuck)."
  934. )
  935. else:
  936. _err = (
  937. f"Subagent timed out after {child_timeout}s with {child_api_calls} API call(s) completed — no "
  938. f"progress in that window (no completed call, tool change, or activity-clock tick): a stalled "
  939. f"provider request or an unresponsive network request."
  940. )
  941. if diagnostic_path:
  942. _err += f" Diagnostic: {diagnostic_path}"
  943. status = "timeout" if is_timeout else "error"
  944. _error_entry = {
  945. "task_index": task_index, "status": status, "summary": None, "error": _err, "exit_reason": status,
  946. "api_calls": child_api_calls, "duration_seconds": duration,
  947. "timeout_seconds": timeout_cause if is_timeout else None,
  948. "timed_out_after_seconds": duration if is_timeout else None,
  949. "timeout_phase": "before_first_llm_call" if before_first_call else "after_llm_calls" if is_timeout else None,
  950. # How long the child had been silent when the wait ended: distinguishes a slow provider from a runaway
  951. # without transcript forensics (#116001).
  952. "last_event_age": _child_last_event_age(child) if is_timeout else None,
  953. "_child_role": getattr(child, "_delegate_role", None),
  954. "diagnostic_path": diagnostic_path,
  955. }
  956. _error_entry = self.finish_failed(_error_entry, _late_pending_steer, preview=f"Timed out after {duration}s" if is_timeout else str(exc))
  957. close_deferred = is_timeout and not future.done()
  958. if close_deferred:
  959. _defer_close_after_timeout(child, future)
  960. return None, _error_entry, close_deferred
  961. def append_sibling_write_reminder(self, entry: dict[str, Any]) -> None:
  962. """Warn the parent when this child wrote files the parent had already read. Checks writes by ANY non-parent
  963. task_id (not just this child's) so nested orchestrator→worker chains are covered too."""
  964. if not (self.parent_task_id and self.parent_reads_snapshot):
  965. return
  966. with _quiet("file_state sibling-write check failed", exc_info=True):
  967. sibling_writes = file_state.writes_since(self.parent_task_id, self.wall_start, self.parent_reads_snapshot)
  968. mod_paths = sorted({p for paths in sibling_writes.values() for p in paths}) if sibling_writes else []
  969. if not mod_paths:
  970. return
  971. reminder = (
  972. "\n\n[NOTE: subagent modified files the parent previously read — re-read before editing: "
  973. + ", ".join(mod_paths[:8])
  974. + (f" (+{len(mod_paths) - 8} more)" if len(mod_paths) > 8 else "")
  975. + "]"
  976. )
  977. if entry.get("summary"):
  978. entry["summary"] = entry["summary"] + reminder
  979. else:
  980. entry["stale_paths"] = mod_paths
  981. def account_background_processes(self, entry: dict[str, Any]) -> None:
  982. """Name the child's background processes on the result BEFORE ``cleanup`` kills them: handed-off ones now
  983. belong to the parent (their completion lands in the parent's chat); anything else still running is about to be
  984. terminated, and the parent must hear that from the runtime rather than trust a child's "watcher running"."""
  985. handed = list(getattr(self.child, "_handed_off_processes", None) or [])
  986. if handed:
  987. entry["handed_off_processes"] = handed
  988. with _quiet(None):
  989. from tools.process_registry import process_registry, _output_tail
  990. leftover = process_registry.running_owned_by(self.child_task_id)
  991. if leftover:
  992. entry["orphaned_processes"] = [
  993. {"session_id": s.id, "command": redact_child_text(s.command, force=False)[:200], "runtime_seconds": round(time.time() - s.started_at)}
  994. for s in leftover]
  995. unread = process_registry.unread_completions_owned_by(self.child_task_id)
  996. if unread:
  997. entry["unread_completions"] = [
  998. {"session_id": s.id, "command": redact_child_text(s.command, force=False)[:200], "exit_code": s.exit_code,
  999. "output_tail": redact_child_text(_output_tail(s, len(s.output_buffer)), force=False)[-600:]} for s in unread]
  1000. def emit_complete(self, result: dict[str, Any], entry: dict[str, Any], duration: float) -> None:
  1001. """Fire ``subagent.complete`` with the per-branch observability payload (tokens, cost, files touched,
  1002. tool-output tail); every field is optional and degrades gracefully on the client."""
  1003. if not self.child_progress_cb:
  1004. return
  1005. child = self.child
  1006. # Parent copy + live progress: honour security.redact_secrets=false (persistence boundaries force).
  1007. entry.update(redact_child_result(entry, getattr(child, "_delegate_output_schema", None), force=False))
  1008. summary = entry["summary"]
  1009. _files_read: list = []
  1010. with _quiet(None):
  1011. _files_read = list(file_state.known_reads(self.child_task_id))[:40]
  1012. _files_written_map: dict = {}
  1013. with _quiet(None):
  1014. _files_written_map = file_state.writes_since("", self.wall_start, []) # all writes since wall_start
  1015. complete_kwargs: dict[str, Any] = {
  1016. "preview": summary[:160] if summary else entry.get("error", ""),
  1017. "status": entry["status"],
  1018. "duration_seconds": duration,
  1019. "summary": summary[:500] if summary else entry.get("error", ""),
  1020. "input_tokens": _num(getattr(child, "session_prompt_tokens", 0)),
  1021. "output_tokens": _num(getattr(child, "session_completion_tokens", 0)),
  1022. "reasoning_tokens": _num(getattr(child, "session_reasoning_tokens", 0)),
  1023. "api_calls": _num(entry["api_calls"]),
  1024. "files_read": _files_read,
  1025. "files_written": sorted({p for tid, paths in _files_written_map.items() if tid == self.child_task_id for p in paths})[:40],
  1026. "output_tail": _extract_output_tail(result, max_entries=8, max_chars=600),
  1027. }
  1028. from agent.runtime_context import current as embedded_context
  1029. if embedded_context() is not None:
  1030. # Yaovi archives this separately; only a bounded preview enters its snapshot.
  1031. complete_kwargs["summary_full"] = summary or entry.get("error", "")
  1032. if entry.get("failure_reason"):
  1033. # Classified verdict rides the event so every surface glosses the failure the same way.
  1034. complete_kwargs["failure_reason"] = entry["failure_reason"]
  1035. _cost_usd = getattr(child, "session_estimated_cost_usd", None)
  1036. if _cost_usd is not None:
  1037. with _quiet(None):
  1038. complete_kwargs["cost_usd"] = float(_cost_usd)
  1039. _safe_progress(self.child_progress_cb, "subagent.complete", **redact_child_result(complete_kwargs, force=False))
  1040. def cleanup(self, *, heartbeat: _Heartbeat, child_pool: Any, leased_cred_id: Any, close_deferred: bool) -> None:
  1041. """Finally-path teardown (idempotent, never raises). Order matters: stop heartbeat → drop registry entry →
  1042. release credential lease → restore the parent's process-global tool names → detach from the parent's
  1043. interrupt list → close the child (unless a timed-out worker still owns it) → pop the child's Relay scope if
  1044. no turn is active."""
  1045. child = self.child
  1046. heartbeat.stop()
  1047. # Safe even if the child was never registered (ID missing on test doubles).
  1048. if self.subagent_id:
  1049. _unregister_subagent(self.subagent_id, agent=child)
  1050. if child_pool is not None and leased_cred_id is not None:
  1051. with _quiet("Failed to release credential lease: %s"):
  1052. child_pool.release_lease(leased_cred_id)
  1053. # Restore the parent's tool names so the process-global is correct for
  1054. # any subsequent execute_code calls or other consumers.
  1055. import model_tools
  1056. saved_tool_names = getattr(child, "_delegate_saved_tool_names", None)
  1057. if isinstance(saved_tool_names, list):
  1058. model_tools._last_resolved_tool_names = list(saved_tool_names)
  1059. _detach_child(self.parent_agent, child)
  1060. # Close tool resources (terminal sandboxes, browser daemons, background
  1061. # processes, httpx clients) so subagent subprocesses don't outlive the delegation.
  1062. if not close_deferred:
  1063. _close_child(child, "Failed to close child agent after delegation")
  1064. # The child's execute_code kernels live exactly as long as the child (pinned against the LRU
  1065. # cap while it runs); dispose them here so they never squat the cap after the child is gone.
  1066. with _quiet("Failed to dispose child execute_code kernels: %s"):
  1067. from tools.code_kernel import shutdown_kernels_for_delegated_child
  1068. shutdown_kernels_for_delegated_child(str(getattr(child, "session_id", "") or ""))
  1069. # The AIAgent turn boundary normally closes the child scope itself. This fallback covers failures before that
  1070. # boundary starts, but must not pop a scope while a timed-out child worker is still unwinding.
  1071. with _quiet("Failed to close child Relay session after delegation"):
  1072. from agent import relay_runtime
  1073. runtime = relay_runtime.get_runtime(create=False)
  1074. child_session_id = str(getattr(child, "session_id", "") or "")
  1075. child_turn_is_active = relay_runtime.SESSION_COORDINATOR.has_active_turn(
  1076. profile_key=relay_runtime.current_profile_key(), session_id=child_session_id,
  1077. )
  1078. if runtime is not None and child_session_id and not child_turn_is_active:
  1079. runtime.unregister_subagent({"child_session_id": child_session_id})