delegation_live_log.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Live, tail-able transcripts for delegated subagents.
  2. One append-only log per child under ``<hermes_home>/cache/delegation/live/
  3. <delegation_id>/task-<n>.log``, pre-created with a header at dispatch (so
  4. ``tail -f`` attaches immediately); paths are returned from ``delegate_task``.
  5. ``cache/delegation`` is mounted read-only into remote terminal backends, so
  6. every line written here must be credential-redacted. Never raises into the
  7. agent loop; append mode per write (close() is the flush); 7-day retention.
  8. """
  9. from __future__ import annotations
  10. import json
  11. import logging
  12. import shutil
  13. import threading
  14. import time
  15. import uuid
  16. from contextlib import contextmanager
  17. from pathlib import Path
  18. from typing import Any, Dict, List, Optional
  19. logger = logging.getLogger(__name__)
  20. LIVE_RETENTION_DAYS = 7
  21. # Per-line truncation budgets (chars): the .log is a compact operational view;
  22. # the child's SessionDB transcript and summary spill files carry full text.
  23. _ASSISTANT_MAX = 600
  24. _THINKING_MAX = 300
  25. _ARGS_MAX = 220
  26. _RESULT_MAX = 400
  27. _KICKOFF_MAX = 500
  28. # Stream deltas are buffered and flushed as one assistant line when another
  29. # event type arrives (or on completion); capped so a huge reply can't hold memory.
  30. _STREAM_BUFFER_FLUSH_CHARS = 4000
  31. _TIME_FMT = "%Y-%m-%d %H:%M:%S"
  32. def live_transcript_root(home: Optional[Path] = None) -> Path:
  33. """Root directory for live transcripts (profile-safe, never ~/.hermes).
  34. Pass ``home`` when the caller holds stable parent-owned profile state
  35. (e.g. the parent agent's SessionDB path). Ambient ``get_hermes_home()``
  36. consults a ContextVar that raw ``threading.Thread`` boundaries drop, so
  37. in a multi-profile process an ambient resolve can land transcripts under
  38. whatever profile the process-wide ``HERMES_HOME`` names at that moment
  39. (#91996).
  40. """
  41. from hermes_constants import get_hermes_dir
  42. return get_hermes_dir("cache/delegation", "delegation_cache", home=home) / "live"
  43. @contextmanager
  44. def _best_effort(what: str):
  45. """Swallow and debug-log any failure: nothing here may reach the agent loop."""
  46. try:
  47. yield
  48. except Exception as exc:
  49. logger.debug("Live transcript %s failed: %s", what, exc)
  50. def _one_line(text: Any, limit: int) -> str:
  51. """Collapse to a single line and truncate with an elided-chars note."""
  52. s = " ".join(str(text or "").split())
  53. if len(s) > limit:
  54. s = s[:limit] + f" …(+{len(s) - limit} chars)"
  55. return s
  56. def _redact(text: str) -> str:
  57. """Mask credentials (``force=True``: safety boundary, even when the global
  58. toggle is off); if the redactor is unavailable, withhold rather than leak."""
  59. if not text:
  60. return text
  61. try:
  62. from agent.redact import redact_sensitive_text
  63. return redact_sensitive_text(text, force=True) or ""
  64. except Exception: # pragma: no cover - core module; never leak on failure
  65. return "[line withheld: redaction unavailable]"
  66. def _joined(*parts: str) -> str:
  67. return " ".join(filter(None, parts))
  68. def _dump_json(path: Path, payload: dict[str, Any]) -> None:
  69. path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
  70. class LiveTranscriptWriter:
  71. """Append-only event log for ONE subagent task. Best-effort: the first write
  72. failure flips ``_ok`` off and later calls become debug-logged no-ops."""
  73. def __init__(self, delegation_id: str, task_index: int, goal: str,
  74. context: Optional[str] = None, root: Optional[Path] = None):
  75. self.delegation_id = delegation_id
  76. self.task_index = task_index
  77. self._ok = False
  78. self._lock = threading.Lock()
  79. self._stream_buf: list[str] = []
  80. self._stream_len = 0
  81. self.path: Optional[Path] = None
  82. with _best_effort(f"init ({delegation_id} task {task_index})"):
  83. goal_line = _one_line(goal, _KICKOFF_MAX)
  84. d = (root if root is not None else live_transcript_root()) / delegation_id
  85. d.mkdir(parents=True, exist_ok=True)
  86. path = d / f"task-{task_index}.log"
  87. path.write_text(
  88. "=== Hermes subagent live transcript ===\n"
  89. f"delegation: {delegation_id} task: {task_index}\n"
  90. f"goal: {_redact(goal_line)}\n" # header bypasses event(), so redact here too
  91. f"started: {time.strftime(_TIME_FMT)}\n"
  92. "(append-only; streams while the subagent runs — tail -f me)\n"
  93. + "=" * 40 + "\n", encoding="utf-8")
  94. self.path, self._ok = path, True
  95. self.event("user", "kickoff: " + goal_line
  96. + (f" | context: {_one_line(context, _KICKOFF_MAX)}" if context else ""))
  97. def event(self, role: str, text: str) -> None:
  98. """Append one ``HH:MM:SS role | text`` line. Single choke point: every typed
  99. helper funnels through here so one redaction covers everything."""
  100. if not self._ok or self.path is None:
  101. return
  102. line = f"{time.strftime('%H:%M:%S')} {role:<9}| {_redact(text)}\n"
  103. try:
  104. with self._lock, open(self.path, "a", encoding="utf-8") as fh:
  105. fh.write(line)
  106. except Exception as exc:
  107. self._ok = False
  108. logger.debug("Live transcript write failed (%s): %s", self.path, exc)
  109. def _line(self, role: str, text: str, limit: int) -> None:
  110. if t := _one_line(text, limit):
  111. self.event(role, t)
  112. def assistant_text(self, text: str) -> None:
  113. self._line("assistant", text, _ASSISTANT_MAX)
  114. def thinking(self, text: str) -> None:
  115. self._line("think", text, _THINKING_MAX)
  116. def tool_start(self, name: str, args_preview: Any = None) -> None:
  117. self.flush_stream()
  118. self.event("tool", f"-> {name or '?'}({_one_line(args_preview, _ARGS_MAX)})")
  119. def tool_result(self, name: str, result: Any = None,
  120. duration: Any = None, is_error: bool = False) -> None:
  121. status = "ERROR" if is_error else "ok"
  122. try:
  123. dur = "" if duration is None else f" {float(duration):.1f}s"
  124. except (TypeError, ValueError):
  125. dur = ""
  126. self.event("result", f"{name or '?'} {status}{dur}: {_one_line(result, _RESULT_MAX)}")
  127. def marker(self, text: str) -> None:
  128. """Lifecycle marker: start / final / error / interrupt / budget."""
  129. self.flush_stream()
  130. self.event("final", _one_line(text, _ASSISTANT_MAX))
  131. def add_stream_delta(self, delta: str) -> None:
  132. """Buffer streamed assistant reply text; flushed as one line."""
  133. if not delta or not self._ok:
  134. return
  135. self._stream_buf.append(delta)
  136. self._stream_len += len(delta)
  137. if self._stream_len >= _STREAM_BUFFER_FLUSH_CHARS:
  138. self.flush_stream()
  139. def flush_stream(self) -> None:
  140. if self._stream_buf:
  141. text, self._stream_buf, self._stream_len = "".join(self._stream_buf), [], 0
  142. self.assistant_text(text)
  143. def _on_complete(self, tool_name, preview, args, kwargs):
  144. dur = kwargs.get("duration_seconds")
  145. summary = kwargs.get("summary") or preview
  146. self.marker(_joined(
  147. f"status={kwargs.get('status', '?')}",
  148. f"duration={dur}s" if dur is not None else "",
  149. f"summary: {_one_line(summary, _RESULT_MAX)}" if summary else ""))
  150. # Event demux (the tool_progress_callback surface): handler(self, tool_name, preview, args, kwargs).
  151. _OBSERVERS = {
  152. "tool.started": lambda s, n, p, a, kw: s.tool_start(str(n or ""), p if p else a),
  153. "tool.completed": lambda s, n, p, a, kw: s.tool_result(
  154. str(n or ""), result=kw.get("result"), duration=kw.get("duration"),
  155. is_error=bool(kw.get("is_error"))),
  156. # Fired as cb("_thinking", <text>) — text rides in the tool_name slot.
  157. "_thinking": lambda s, n, p, a, kw: s.thinking(str(n or p or "")),
  158. # cb("reasoning.available", "_thinking", <text>, None)
  159. "reasoning.available": lambda s, n, p, a, kw: s.thinking(str(p or "")),
  160. "subagent.text": lambda s, n, p, a, kw: s.add_stream_delta(str(p or "")),
  161. "subagent.start": lambda s, n, p, a, kw: s.event("start", _one_line(p, _KICKOFF_MAX)),
  162. "subagent.complete": _on_complete}
  163. def observe(self, event_type: Any, tool_name: Any = None, preview: Any = None,
  164. args: Any = None, **kwargs: Any) -> None:
  165. """Map a child tool_progress_callback event onto transcript lines.
  166. Unknown events are ignored. Never raises (event() swallows I/O)."""
  167. handler = self._OBSERVERS.get(str(event_type or ""))
  168. if handler is not None:
  169. handler(self, tool_name, preview, args, kwargs)
  170. def finalize(self, entry: dict[str, Any]) -> None:
  171. """Terminal marker with exit-reason detail subagent.complete lacks."""
  172. exit_reason = entry.get("exit_reason")
  173. self.marker(_joined(
  174. f"end status={entry.get('status', '?')}",
  175. f"exit_reason={exit_reason}" if exit_reason else "",
  176. "(iteration budget exhausted)" if exit_reason == "max_iterations" else "",
  177. f"error: {_one_line(entry['error'], _RESULT_MAX)}" if entry.get("error") else ""))
  178. def wrap_progress_callback(inner_cb, writer: LiveTranscriptWriter):
  179. """Wrap a child's tool_progress_callback (may be None) so events also land in
  180. the log; writer failures never propagate. Preserves the ``_flush`` contract."""
  181. def _cb(event_type, tool_name=None, preview=None, args=None, **kwargs):
  182. with _best_effort("observe"):
  183. writer.observe(event_type, tool_name, preview, args, **kwargs)
  184. if inner_cb is not None:
  185. inner_cb(event_type, tool_name, preview, args, **kwargs)
  186. def _flush():
  187. with _best_effort("flush"):
  188. writer.flush_stream()
  189. if callable(getattr(inner_cb, "_flush", None)):
  190. inner_cb._flush()
  191. _cb._flush = _flush
  192. return _cb
  193. def create_live_transcripts(
  194. task_list: list[dict[str, Any]], context: Optional[str] = None,
  195. delegation_id: Optional[str] = None, model: Optional[str] = None,
  196. provider: Optional[str] = None,
  197. home: Optional[Path] = None,
  198. ) -> tuple[Optional[str], list[Optional[LiveTranscriptWriter]], list[str]]:
  199. """One pre-headered writer per task + a manifest.json; prunes stale dirs.
  200. Returns ``(delegation_id, writers, paths)``; on any top-level failure
  201. ``(None, [None]*n, [])`` so delegation proceeds untouched.
  202. ``home`` pins every transcript and the manifest to one explicit profile
  203. home instead of an ambient resolve that raw thread boundaries can strip
  204. of its ContextVar override (#91996). Retention pruning runs against the
  205. same resolved root, so pinned homes clean their own stale dirs.
  206. """
  207. n = len(task_list)
  208. prune_stale_live_dirs(root=live_transcript_root(home)) # best-effort; never raises
  209. with _best_effort("creation"):
  210. # Same id shape as async_delegation's so the dir name matches the handle.
  211. deleg_id = delegation_id or f"deleg_{uuid.uuid4().hex[:8]}"
  212. root = live_transcript_root(home)
  213. made = [LiveTranscriptWriter(deleg_id, i, str(t.get("goal", "")),
  214. context=t.get("context") or context, root=root)
  215. for i, t in enumerate(task_list)]
  216. writers: list[Optional[LiveTranscriptWriter]] = [w if w.path is not None else None for w in made]
  217. paths: list[str] = [str(w.path) for w in made if w.path is not None]
  218. if not paths:
  219. return None, [None] * n, []
  220. _write_manifest(deleg_id, task_list, paths, model=model, provider=provider, home=home)
  221. return deleg_id, writers, paths
  222. return None, [None] * n, []
  223. def _manifest_path(delegation_id: str, home: Optional[Path] = None) -> Path:
  224. return live_transcript_root(home) / delegation_id / "manifest.json"
  225. def _write_manifest(delegation_id: str, task_list: list[dict[str, Any]],
  226. paths: list[str], model: Optional[str] = None,
  227. provider: Optional[str] = None, home: Optional[Path] = None) -> None:
  228. with _best_effort("manifest write"):
  229. _dump_json(_manifest_path(delegation_id, home), {
  230. "delegation_id": delegation_id, "started": time.strftime(_TIME_FMT),
  231. "task_count": len(task_list), "model": model, "provider": provider,
  232. "tasks": [{
  233. "index": i,
  234. # Same mounted dir as the .log files, so the goal needs the same redaction.
  235. "goal": _redact(str(t.get("goal", ""))[:500]),
  236. "log": paths[i] if i < len(paths) else None,
  237. "status": "running"} for i, t in enumerate(task_list)]})
  238. def update_manifest_statuses(delegation_id: Optional[str],
  239. results: list[dict[str, Any]],
  240. home: Optional[Path] = None, *, completed: bool = True) -> None:
  241. """Best-effort per-task status update once the batch has aggregated (``completed=False``: tasks settled
  242. before the batch ran, e.g. ones their backend refused, while the rest still run)."""
  243. if not delegation_id:
  244. return
  245. with _best_effort("manifest update"):
  246. mp = _manifest_path(delegation_id, home)
  247. manifest = json.loads(mp.read_text(encoding="utf-8-sig"))
  248. by_index = {r.get("task_index"): r for r in results if isinstance(r, dict)}
  249. for task in manifest.get("tasks", []):
  250. r = by_index.get(task.get("index"))
  251. if r is not None:
  252. task["status"] = r.get("status", task.get("status"))
  253. if r.get("exit_reason"):
  254. task["exit_reason"] = r["exit_reason"]
  255. if r.get("model"):
  256. task["model"] = r["model"]
  257. # The dispatch-time model is only the delegation.model override (None when children inherit the parent's).
  258. if not manifest.get("model"):
  259. manifest["model"] = next((t["model"] for t in manifest.get("tasks", []) if t.get("model")), None)
  260. if completed:
  261. manifest["completed"] = time.strftime(_TIME_FMT)
  262. _dump_json(mp, manifest)
  263. def prune_stale_live_dirs(max_age_days: int = LIVE_RETENTION_DAYS, root: Optional[Path] = None) -> int:
  264. """Remove live/<delegation_id> dirs older than the retention window. Best-effort.
  265. ``root`` defaults to the ambient resolve; callers that pin transcripts to an
  266. explicit home pass the same root so the prune sweeps where the writes actually
  267. land (stale dirs under other profiles' roots stay those profiles' business).
  268. """
  269. removed = 0
  270. with _best_effort("pruning"):
  271. root_dir = root if root is not None else live_transcript_root()
  272. if not root_dir.is_dir():
  273. return 0
  274. cutoff = time.time() - max_age_days * 86400
  275. for child in root_dir.iterdir():
  276. try:
  277. if child.is_dir() and child.stat().st_mtime < cutoff:
  278. shutil.rmtree(child, ignore_errors=True)
  279. removed += 1
  280. except OSError:
  281. continue
  282. return removed