delegate_tool_results.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437
  1. """Subagent result post-processing: summary budget/spill, tool-trace summaries, lifecycle hooks and cost rollup."""
  2. from __future__ import annotations
  3. import logging
  4. import json
  5. import threading
  6. from typing import Any, Dict, List, Optional
  7. from urllib.parse import urlsplit, urlunsplit
  8. from tools.delegation_redaction import (
  9. parent_copy, redact_child_result, redact_child_text, redact_child_value, redact_summary,
  10. )
  11. logger = logging.getLogger("tools.delegate_tool") # log-record parity with the origin module
  12. def _stringify_tool_content(content: Any) -> str:
  13. """Stable text for tool-result content. Some OpenAI-compatible paths return
  14. content-block lists; observability must never crash on them."""
  15. if content is None:
  16. return ""
  17. if isinstance(content, str):
  18. return content
  19. if isinstance(content, list):
  20. return "\n".join(
  21. item["text"] if isinstance(item, dict) and isinstance(item.get("text"), str)
  22. else json.dumps(item, ensure_ascii=False, default=str) if isinstance(item, dict)
  23. else str(item)
  24. for item in content
  25. )
  26. if isinstance(content, dict):
  27. return json.dumps(content, ensure_ascii=False, default=str)
  28. return str(content)
  29. def _looks_like_error_output(content: Any) -> bool:
  30. """Conservative error detector for tool-result previews: structured JSON with
  31. an ``error`` key or an error/failed ``status``, or a first line starting with a
  32. classic error marker. (Substring "error" alone painted normal output red.)"""
  33. content = _stringify_tool_content(content)
  34. if not content:
  35. return False
  36. if content.lstrip().startswith(("{", "[")):
  37. try:
  38. parsed = json.loads(content)
  39. except Exception:
  40. parsed = None
  41. if isinstance(parsed, dict) and (
  42. parsed.get("error") or str(parsed.get("status") or "").strip().lower() in {"error", "failed", "failure", "timeout"}
  43. ):
  44. return True
  45. first = content.splitlines()[0].strip().lower() if content.splitlines() else ""
  46. return first.startswith(("error:", "failed:", "traceback ", "exception:"))
  47. def _extract_output_tail(result: dict[str, Any], *, max_entries: int = 12, max_chars: int = 8000) -> list[dict[str, Any]]:
  48. """Last N tool-call results ``{tool, preview, is_error}`` from a child's conversation (the overlay's "Output"
  49. section), chronological order. Content blocks are flattened first so a block-wrapped "Error: ..." is still
  50. flagged; line structure is preserved (capped at ``max_chars``) so the overlay shows real output rather than a
  51. whitespace-collapsed blob."""
  52. messages = result.get("messages") if isinstance(result, dict) else None
  53. if not isinstance(messages, list):
  54. return []
  55. name_by_call_id = {
  56. tc["id"]: str((tc.get("function") or {}).get("name") or "tool")
  57. for msg in messages if isinstance(msg, dict) and msg.get("role") == "assistant"
  58. for tc in msg.get("tool_calls") or [] if tc.get("id")
  59. }
  60. tail: list[dict[str, Any]] = []
  61. for msg in reversed(messages): # newest first, then restore order below
  62. if len(tail) >= max_entries:
  63. break
  64. if not isinstance(msg, dict) or msg.get("role") != "tool":
  65. continue
  66. # Live progress (subagent.complete overlay): honours security.redact_secrets=false.
  67. content = redact_child_text(_stringify_tool_content(msg.get("content") or ""), force=False)
  68. tool_name = name_by_call_id.get(msg.get("tool_call_id") or "", "tool")
  69. tail.append({"tool": tool_name, "preview": content[:max_chars], "is_error": _looks_like_error_output(content)})
  70. tail.reverse()
  71. return tail
  72. _TOOL_INPUT_TARGET_KEYS = frozenset({
  73. "cwd", "destination_path", "directory", "dst", "endpoint", "file_path", "new_path", "old_path", "path",
  74. "source_path", "src", "target_path", "url", "urls",
  75. })
  76. _TOOL_INPUT_URL_KEYS = frozenset({"endpoint", "url", "urls"})
  77. def _sanitize_tool_target(key: str, value: Any, force: bool = True) -> Any:
  78. """Keep bounded side-effect targets while dropping URL secrets."""
  79. if isinstance(value, list):
  80. cleaned = [item for item in (_sanitize_tool_target(key, item, force) for item in value[:16]) if item is not None]
  81. return cleaned or None
  82. if not isinstance(value, str) or not value:
  83. return None
  84. bounded = redact_child_text(value, force=force)[:1024]
  85. if key in _TOOL_INPUT_URL_KEYS:
  86. try:
  87. parsed = urlsplit(bounded)
  88. if parsed.scheme and parsed.netloc:
  89. hostname = parsed.hostname
  90. if not hostname:
  91. return None
  92. # ``SplitResult.netloc`` includes ``user:password@``. Rebuild the authority from parsed host/port so
  93. # hook-visible history cannot carry URL credentials. Bracket IPv6 literals before appending a
  94. # validated port.
  95. host = f"[{hostname}]" if ":" in hostname else hostname
  96. netloc = f"{host}:{parsed.port}" if parsed.port is not None else host
  97. return urlunsplit((parsed.scheme, netloc, parsed.path, "", ""))
  98. except ValueError:
  99. return None
  100. return bounded
  101. def _sanitize_targets(mapping: dict[str, Any], force: bool = True) -> dict[str, Any]:
  102. """Keep only known side-effect target keys, each sanitized (URL secrets dropped)."""
  103. targets: dict[str, Any] = {}
  104. for raw_key, value in mapping.items():
  105. key = str(raw_key).lower()
  106. if key in _TOOL_INPUT_TARGET_KEYS:
  107. cleaned = _sanitize_tool_target(key, value, force)
  108. if cleaned is not None:
  109. targets[key] = cleaned
  110. return targets
  111. def _input_summary(keys: Any, targets: Any, force: bool = True) -> dict[str, Any]:
  112. """``{argument_keys, targets}`` with bounded, sanitized contents (empty on bad shapes). ``force=False`` for the
  113. parent's own tool_trace copy (honours the opt-out); lifecycle hooks re-sanitize with ``force=True``."""
  114. return {
  115. "argument_keys": [redact_child_text(str(key), force=force)[:128] for key in keys[:64]] if isinstance(keys, list) else [],
  116. "targets": _sanitize_targets(targets, force) if isinstance(targets, dict) else {},
  117. }
  118. def _summarize_tool_arguments(arguments: Any) -> dict[str, Any]:
  119. """Summarize argument names and side-effect targets without raw payloads."""
  120. try:
  121. parsed = json.loads(arguments) if isinstance(arguments, str) else None
  122. except (TypeError, ValueError):
  123. parsed = None
  124. if not isinstance(parsed, dict):
  125. return _input_summary([], {})
  126. return _input_summary(sorted(str(key) for key in parsed), parsed, force=False)
  127. def _subagent_stop_tool_call_history(tool_trace: Any) -> list[dict[str, Any]]:
  128. """Detached, metadata-only tool history for lifecycle hooks (input summaries re-sanitized)."""
  129. if not isinstance(tool_trace, list):
  130. return []
  131. def _byte_count(item, key: str) -> int:
  132. value = item.get(key, 0)
  133. return max(0, int(value)) if isinstance(value, (int, float)) and not isinstance(value, bool) else 0
  134. history: list[dict[str, Any]] = []
  135. for item in tool_trace:
  136. if not isinstance(item, dict):
  137. continue
  138. status = str(item.get("status") or "unknown").lower()
  139. summary = item.get("input_summary")
  140. summary = summary if isinstance(summary, dict) else {}
  141. history.append({
  142. "tool_name": str(item.get("tool") or "unknown")[:256],
  143. "tool_input": _input_summary(summary.get("argument_keys"), summary.get("targets")),
  144. "input_bytes": _byte_count(item, "args_bytes"), "output_bytes": _byte_count(item, "result_bytes"),
  145. "status": status if status in {"ok", "error"} else "unknown",
  146. })
  147. return history
  148. # Hard per-summary character ceiling layered on top of the dynamic headroom budget (see _apply_summary_budget):
  149. # belt-and-suspenders for models that ignore "be concise". 0 disables the ceiling.
  150. DEFAULT_MAX_SUMMARY_CHARS = 24000
  151. # Fraction of the parent's *remaining* context headroom the whole batch of summaries may consume, split per summary,
  152. # so N children can't collectively blow the parent's window (the compression/429 death spiral).
  153. _SUMMARY_HEADROOM_FRACTION = 0.5
  154. # Floor so a single summary always gets a usable slice even when the parent is
  155. # already nearly full — below this we'd be truncating to noise.
  156. _MIN_SUMMARY_CHARS = 2000
  157. def _spill_summary_to_file(task_index: int, summary: str) -> Optional[str]:
  158. """Write the full summary under ``cache/delegation`` (mounted read-only into remote backends via
  159. ``credential_files._CACHE_DIRS``, so the parent's terminal/``read_file`` can page it on any backend). Absolute
  160. path, or None on failure — the trimmed head+tail is still returned regardless."""
  161. try:
  162. from hermes_constants import get_hermes_dir
  163. import datetime as _dt
  164. cache_dir = get_hermes_dir("cache/delegation", "delegation_cache")
  165. cache_dir.mkdir(parents=True, exist_ok=True)
  166. path = cache_dir / f"subagent-summary-{task_index}-{_dt.datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.txt"
  167. from tools.spill_safety import write_text_exclusive
  168. # Exclusive symlink-refusing create; not private because cache/delegation is bind-mounted read-only into
  169. # remote backends whose container UID must be able to read it.
  170. # ``summary`` is already the forced, structure-preserving projection (_trim_summary_with_footer); a
  171. # second whole-text pass here would eat JSON delimiters and corrupt a valid structured answer.
  172. write_text_exclusive(path, getattr(summary, "forced", summary), private=False)
  173. return str(path)
  174. except Exception as exc:
  175. logger.debug("Failed to spill subagent summary to file: %s", exc)
  176. return None
  177. def _trim_summary_with_footer(summary: str, cap: int, task_index: int) -> tuple[str, Optional[str]]:
  178. """``(model_text, spill_path)`` for one over-budget summary: a ~75% head / ~25% tail window snapped to line
  179. boundaries (so the opening AND the closing outcomes/files-changed/issues both survive), the full text spilled
  180. to disk, and a footer giving the exact ``read_file offset=`` for the omitted middle."""
  181. # ``summary`` is the parent's copy (scrubbed per the user's setting in _finalize_child_results), carrying the
  182. # forced projection of the COMPLETE output. The spill file and the forced (memory/hook/ledger) excerpt are cut
  183. # from that projection: trimming the opt-out copy first can drop a PEM BEGIN marker and leave an unrecognizable
  184. # key body behind. Direct callers pass plain text: project it here (idempotent on already-forced text).
  185. forced = redact_child_text(summary) if hasattr(summary, "forced") else redact_summary(summary, True, True)
  186. spill_path = _spill_summary_to_file(task_index, forced)
  187. if spill_path:
  188. # Agent-visible path: the parent's read_file runs inside the active backend (#81984, #77015).
  189. from tools.credential_files import to_agent_visible_cache_path
  190. spill_path = to_agent_visible_cache_path(spill_path)
  191. model_text = _head_tail_with_footer(str(summary), cap, spill_path)
  192. return parent_copy(model_text, _head_tail_with_footer(forced, cap, spill_path)), spill_path
  193. def _head_tail_with_footer(summary: str, cap: int, spill_path: Optional[str]) -> str:
  194. original_len = len(summary)
  195. head_budget = int(cap * 0.75)
  196. tail_budget = cap - head_budget
  197. head, tail = summary[:head_budget], summary[-tail_budget:]
  198. nl = head.rfind("\n")
  199. if nl > head_budget * 0.5:
  200. head = head[:nl]
  201. nl = tail.find("\n")
  202. if 0 <= nl < tail_budget * 0.5:
  203. tail = tail[nl + 1:]
  204. footer_lines = [
  205. "", "─" * 8 + " [SUMMARY TRUNCATED] " + "─" * 8,
  206. f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) "
  207. f"of {original_len:,} total — trimmed to protect the parent's context window.",
  208. ]
  209. if spill_path:
  210. # read_file is 1-indexed; +2 moves past the last head line shown.
  211. footer_lines.append(f"Full subagent output saved to: {spill_path}")
  212. footer_lines.append(
  213. f'To read the omitted middle: read_file path="{spill_path}" '
  214. f"offset={head.count(chr(10)) + 2} limit=200 (the file is the complete "
  215. f"summary; raise/lower offset to page through it)."
  216. )
  217. else:
  218. footer_lines.append("Full output could not be stored to disk; the head+tail above is all that was preserved.")
  219. footer_lines.append("─" * 37)
  220. return head + "\n\n[... middle omitted — see footer ...]\n\n" + tail + "\n".join(footer_lines)
  221. def _parent_prompt_size_tokens(parent_agent) -> Optional[int]:
  222. """The parent's current prompt size: the aggregator's own last ``prompt_tokens`` (pre-MoA-fold), else
  223. the last provider usage. ``None`` when no request has completed yet: the caller then applies the
  224. static ceiling only. Treating "no usage yet" as zero handed a 190K/200K parent a 384K-char budget."""
  225. size = getattr(parent_agent, "_last_prompt_size_tokens", None)
  226. if isinstance(size, (int, float)) and size > 0:
  227. return int(size)
  228. last_usage = getattr(parent_agent, "_last_turn_usage", None) or {}
  229. used = last_usage.get("prompt_tokens") if isinstance(last_usage, dict) else None
  230. if isinstance(used, (int, float)) and used > 0:
  231. return int(used)
  232. return None
  233. def _parent_summary_char_budget(parent_agent, n_summaries: int) -> Optional[int]:
  234. """Per-summary char budget from the parent's *remaining* context headroom (context length − the parent's
  235. current prompt size − the compressor's output reserve), a fraction of it split across the batch at ~4
  236. chars/token. None when the parent's context state is unknown — caller then uses the static ceiling only.
  237. "Current prompt size" is the last API call's ``prompt_tokens`` (``_last_turn_usage``), never
  238. ``session_prompt_tokens``: that field is the running SUM over every call in the session, so after a
  239. few hundred calls it exceeds any context window and every summary collapsed to the 2,000-char floor
  240. (measured: all 1,393 child summaries in one run, each spilled to disk, the orchestrator working from
  241. the stub)."""
  242. try:
  243. compressor = getattr(parent_agent, "context_compressor", None)
  244. context_length = getattr(compressor, "context_length", None)
  245. if not isinstance(context_length, int) or context_length <= 0:
  246. return None
  247. used_tokens = _parent_prompt_size_tokens(parent_agent)
  248. if used_tokens is None:
  249. return None # no usage yet and nothing to estimate from: static ceiling only, never "zero context"
  250. headroom_tokens = context_length - int(used_tokens) - int(getattr(compressor, "max_tokens", 0) or 0)
  251. if headroom_tokens <= 0:
  252. return _MIN_SUMMARY_CHARS # parent already over budget: floor only
  253. per_summary_tokens = int(headroom_tokens * _SUMMARY_HEADROOM_FRACTION) // max(1, n_summaries)
  254. return max(_MIN_SUMMARY_CHARS, per_summary_tokens * 4)
  255. except Exception:
  256. logger.debug("Summary budget computation failed", exc_info=True)
  257. return None
  258. def _apply_summary_budget(results: list[dict[str, Any]], parent_agent) -> None:
  259. """Trim subagent summaries in-place so a batch can't overflow the parent's context window (full text spilled to
  260. disk). Per-summary cap = MIN(dynamic headroom budget, static ``delegation.max_summary_chars`` ceiling; 0 =
  261. disabled); over-cap summaries become head+tail plus a pointer to the spill file."""
  262. from tools.delegate_tool import _load_config
  263. summaries = [r for r in results if isinstance(r, dict) and isinstance(r.get("summary"), str) and r["summary"]]
  264. if not summaries:
  265. return
  266. try:
  267. static_ceiling = int(_load_config().get("max_summary_chars", DEFAULT_MAX_SUMMARY_CHARS))
  268. except (TypeError, ValueError):
  269. static_ceiling = DEFAULT_MAX_SUMMARY_CHARS
  270. candidates = [c for c in (static_ceiling, _parent_summary_char_budget(parent_agent, len(summaries))) if c and c > 0]
  271. if not candidates:
  272. return # both disabled / unknown → leave summaries untouched
  273. cap = min(candidates)
  274. for entry in summaries:
  275. summary = entry["summary"]
  276. if len(summary) <= cap:
  277. continue
  278. model_text, spill_path = _trim_summary_with_footer(summary, cap, entry.get("task_index", -1))
  279. entry["summary"] = model_text
  280. entry["summary_truncated"] = True
  281. if spill_path:
  282. entry["summary_full_path"] = spill_path
  283. logger.debug(
  284. "[subagent-%s] summary trimmed %d → ~%d chars (spill=%s)", entry.get("task_index", "?"), len(summary), cap,
  285. spill_path or "none",
  286. )
  287. _PARENT_FINALIZATION_LOCK_GUARD = threading.Lock()
  288. _PARENT_FINALIZATION_FALLBACK_LOCK = threading.RLock()
  289. _CHILD_CONSTRUCTION_LOCK = threading.RLock()
  290. def _build_child_preserving_parent_tools(**kwargs):
  291. """Build a child without leaking its resolved toolset into the parent."""
  292. from tools.delegate_tool import _build_child_agent
  293. import model_tools
  294. with _CHILD_CONSTRUCTION_LOCK:
  295. parent_tool_names = list(model_tools._last_resolved_tool_names)
  296. try:
  297. child = _build_child_agent(**kwargs)
  298. finally:
  299. model_tools._last_resolved_tool_names = parent_tool_names
  300. child._delegate_saved_tool_names = parent_tool_names
  301. return child
  302. def _parent_finalization_lock(parent_agent) -> threading.RLock:
  303. """Per-parent lock serializing lifecycle side effects (created once under the guard)."""
  304. if parent_agent is None:
  305. return _PARENT_FINALIZATION_FALLBACK_LOCK
  306. lock = getattr(parent_agent, "_subagent_finalization_lock", None)
  307. if lock is not None:
  308. return lock
  309. with _PARENT_FINALIZATION_LOCK_GUARD:
  310. lock = getattr(parent_agent, "_subagent_finalization_lock", None)
  311. if lock is None:
  312. lock = threading.RLock()
  313. try:
  314. parent_agent._subagent_finalization_lock = lock
  315. except Exception:
  316. return _PARENT_FINALIZATION_FALLBACK_LOCK
  317. return lock
  318. def _notify_memory_manager(results, task_list, child_by_index, parent_agent) -> None:
  319. memory = getattr(parent_agent, "_memory_manager", None) if parent_agent else None
  320. if not memory:
  321. return
  322. for entry in results:
  323. try:
  324. task_index = entry.get("task_index", -1)
  325. in_range = isinstance(task_index, int) and 0 <= task_index < len(task_list)
  326. memory.on_delegation(
  327. task=task_list[task_index]["goal"] if in_range else "",
  328. result=redact_child_text(entry.get("summary", "") or "", force=True), # memory providers persist
  329. child_session_id=getattr(child_by_index.get(task_index), "session_id", ""),
  330. )
  331. except Exception:
  332. pass
  333. def _fire_subagent_stop_hooks(results, child_by_index, parent_agent) -> float:
  334. """Pop the model-hidden ``_child_role`` / ``_child_cost_usd`` fields from every
  335. entry, fire ``subagent_stop`` per child, and return the summed child cost."""
  336. try:
  337. from hermes_cli.plugins import invoke_hook
  338. except Exception:
  339. invoke_hook = None
  340. children_cost_total = 0.0
  341. for entry in results:
  342. child_role = entry.pop("_child_role", None)
  343. child_cost = entry.pop("_child_cost_usd", 0.0)
  344. try:
  345. if child_cost:
  346. children_cost_total += float(child_cost)
  347. except (TypeError, ValueError):
  348. pass
  349. if invoke_hook is None:
  350. continue
  351. try:
  352. child = child_by_index.get(entry.get("task_index", -1))
  353. invoke_hook(
  354. "subagent_stop", parent_session_id=getattr(parent_agent, "session_id", None),
  355. parent_turn_id=getattr(parent_agent, "_current_turn_id", "") or "",
  356. child_session_id=getattr(child, "session_id", None), child_role=child_role,
  357. child_summary=redact_child_value(entry.get("summary")), # forced: plugins leave the process
  358. child_status=entry.get("status"),
  359. tool_call_history=_subagent_stop_tool_call_history(entry.get("tool_trace")),
  360. duration_ms=int((entry.get("duration_seconds") or 0) * 1000),
  361. )
  362. except Exception:
  363. logger.debug("subagent_stop hook invocation failed", exc_info=True)
  364. return children_cost_total
  365. def _rollup_children_cost(parent_agent, children_cost_total: float) -> None:
  366. """Fold the children's spend into the parent's session cost (source/status
  367. only set when the parent had none of its own)."""
  368. if children_cost_total <= 0.0:
  369. return
  370. try:
  371. current = float(getattr(parent_agent, "session_estimated_cost_usd", 0.0) or 0.0)
  372. parent_agent.session_estimated_cost_usd = current + children_cost_total
  373. if getattr(parent_agent, "session_cost_source", "none") in {None, "", "none"}:
  374. parent_agent.session_cost_source = "subagent"
  375. if getattr(parent_agent, "session_cost_status", "unknown") in {None, "", "unknown"}:
  376. parent_agent.session_cost_status = "estimated"
  377. except Exception:
  378. logger.debug("Subagent cost rollup failed", exc_info=True)
  379. def _finalize_child_results(
  380. results: list[dict[str, Any]], task_list: list[dict[str, Any]], children: list[tuple[int, dict[str, Any], Any]],
  381. parent_agent,
  382. ) -> None:
  383. """Apply host-owned summary, memory, hook, and cost contracts once."""
  384. with _parent_finalization_lock(parent_agent):
  385. child_by_index = {index: child for index, _task, child in children}
  386. for entry in results:
  387. schema = getattr(child_by_index.get(entry.get("task_index")), "_delegate_output_schema", None)
  388. # Fork the forced, structure-preserving projection from the COMPLETE output before the parent budget
  389. # trims anything; spill/memory/hooks/the durable row read it. The parent's own copy honours
  390. # security.redact_secrets=false.
  391. summary = entry.get("summary")
  392. forced = redact_summary(summary, isinstance(schema, dict) or "schema_valid" in entry, True)
  393. entry.update(redact_child_result(entry, schema, force=False))
  394. if isinstance(summary, str) and isinstance(entry.get("summary"), str):
  395. entry["summary"] = parent_copy(entry["summary"], forced)
  396. _apply_summary_budget(results, parent_agent)
  397. _notify_memory_manager(results, task_list, child_by_index, parent_agent)
  398. _rollup_children_cost(parent_agent, _fire_subagent_stop_hooks(results, child_by_index, parent_agent))
  399. def _run_child_lifecycle(task_index: int, goal: str, child=None, parent_agent=None) -> dict[str, Any]:
  400. """Run one child and apply the same host lifecycle used by delegate_task."""
  401. from tools.delegate_tool import _run_single_child
  402. result = _run_single_child(task_index, goal, child, parent_agent)
  403. result.setdefault("task_index", task_index)
  404. task = {"goal": goal}
  405. _finalize_child_results([result], [{"goal": ""} for _ in range(task_index)] + [task], [(task_index, task, child)], parent_agent)
  406. return result