stream_delivery.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. """Streaming / interim-message delivery for ``AIAgent``.
  2. Single-writer stream ownership, delta/reasoning hook fan-out, and interim assistant text dedup.
  3. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
  4. """
  5. import logging
  6. import re
  7. import threading
  8. from typing import Any, Dict, List
  9. from agent.memory_context import sanitize_context
  10. from agent.message_content import flatten_message_text
  11. from agent.history_commentary import COMMENTARY_WITHHELD, commentary_items, visible_commentary
  12. # Same logger name as the origin module so log records / caplog filters are unchanged.
  13. logger = logging.getLogger("run_agent")
  14. def withhold_held_row(agent: Any, assistant_msg: dict[str, Any]) -> dict[str, Any]:
  15. """Type a held turn's model row ``display_kind="hidden"`` when it has visible text; returns it.
  16. Every row ``_build_assistant_message`` makes passes here before its first flush, so commentary,
  17. continuation fragments, codex incomplete partials and an answer a stop gate sent back never reach
  18. a history load; ``finish_text_response`` publishes the one row the user gets
  19. (``publish_held_final_row``). The key never reaches the wire: the model's context and its cached
  20. prefix are unchanged."""
  21. if (getattr(agent, "_hold_stream", False) is True and not assistant_msg.get("display_kind")
  22. and agent._interim_assistant_visible_text(assistant_msg)):
  23. assistant_msg["display_kind"] = "hidden"
  24. return assistant_msg
  25. def publish_held_final_row(agent: Any, final_msg: dict[str, Any], final_response: Any, joined: bool) -> None:
  26. """The held turn's final row shows what the user got: visible again, without the Codex
  27. commentary the transform never saw, and carrying the whole stitched reply when
  28. length-continuation fragments (hidden above) were joined into it."""
  29. if getattr(agent, "_hold_stream", False) is not True:
  30. return
  31. if final_msg.get("display_kind") == "hidden":
  32. del final_msg["display_kind"]
  33. if commentary_items(final_msg): # Codex commentary rides the row's items, for replay only
  34. final_msg["display_metadata"] = {**(final_msg.get("display_metadata") or {}), COMMENTARY_WITHHELD: True}
  35. if joined and final_msg.get("content") and final_msg["content"] != final_response:
  36. final_msg.setdefault("api_content", final_msg["content"]) # the model keeps the piece it wrote
  37. final_msg["content"] = final_response
  38. class StreamDeliveryMixin:
  39. """Stream ownership, delta/reasoning hook fan-out and interim-text dedup (see module docstring)."""
  40. @staticmethod
  41. def _call_quietly(cb, *args) -> bool:
  42. """Call ``cb(*args)`` if set, swallowing errors; True when it ran without raising."""
  43. if cb is None:
  44. return False
  45. try:
  46. cb(*args)
  47. return True
  48. except Exception:
  49. return False
  50. def _deliver_to_stream_callbacks(self, text: str) -> bool:
  51. """Send ``text`` to the display + TTS delta callbacks; True if at least one accepted it.
  52. Nothing leaves while a ``hold_stream`` plugin holds this turn's reply (``_hold_stream``)."""
  53. if getattr(self, "_hold_stream", False):
  54. return False
  55. results = [self._call_quietly(cb, text) for cb in (self.stream_delta_callback, self._stream_callback)]
  56. return any(results)
  57. def _enqueue_stream_hook(self, event: str, *, label: str | None = None, **fields: Any) -> None:
  58. """Best-effort plugin stream hook enqueue; never raises into the stream path."""
  59. try:
  60. from agent.plugin_stream_hooks import enqueue_plugin_stream_hook
  61. enqueue_plugin_stream_hook(event, **self._stream_hook_base_payload(), **fields)
  62. except Exception:
  63. logger.debug("%s plugin hook enqueue failed", label or event, exc_info=True)
  64. def _reset_stream_delivery_tracking(self) -> None:
  65. """Reset tracking for text delivered during the current model response.
  66. Flushes the think scrubber's benign tail first, routed through the context scrubber (a span
  67. straddling the boundary must still be caught), then the context scrubber's own tail.
  68. """
  69. think_scrubber = getattr(self, "_stream_think_scrubber", None)
  70. ctx_scrubber = getattr(self, "_stream_context_scrubber", None)
  71. # Inline <think> text is forwarded to the reasoning pane only while no native reasoning delta
  72. # arrived for this model response (#89647).
  73. self._native_reasoning_streamed = False
  74. # Next stream re-reads plugins.stream_reasoning_deltas (config edits land per request).
  75. self._stream_reasoning_hooks_enabled = None
  76. def deliver(tail: str) -> None:
  77. # Recorded only once shown (as ``_fire_stream_delta``): a held tail must not become an
  78. # interrupted turn's untransformed partial final.
  79. if tail and self._deliver_to_stream_callbacks(tail):
  80. self._record_streamed_assistant_text(tail)
  81. # Flush any benign partial-tag tail held by the think scrubber first (#17924): an innocent '<' at
  82. # the end of the stream that turned out not to be a tag prefix should reach the UI. Then flush the
  83. # context scrubber. Order matters — the think scrubber's output feeds into the context scrubber's
  84. # state.
  85. # Suppress reasoning/thinking blocks via the stateful scrubber (#17924). Earlier versions ran
  86. # _strip_think_blocks per-delta here, which destroyed downstream state machines when a tag was split
  87. # across deltas (e.g. MiniMax-M2.7 sends '<think>' and its content as separate deltas — regex case 2
  88. # erased the first delta, so the CLI/gateway state machine never saw the open tag and leaked the
  89. # reasoning content as regular response text).
  90. if think_scrubber is not None:
  91. think_tail = think_scrubber.flush()
  92. deliver(ctx_scrubber.feed(think_tail) if think_tail and ctx_scrubber is not None else think_tail)
  93. if ctx_scrubber is not None:
  94. deliver(ctx_scrubber.flush())
  95. self._current_streamed_assistant_text = ""
  96. @property
  97. def _current_streamed_assistant_text(self) -> str:
  98. """Visible assistant text streamed so far this turn. Backed by a list of pieces: ``+=`` on a string
  99. attribute copies the whole reply on every delta (quadratic). Hot-path emptiness checks look at
  100. ``_streamed_assistant_text_parts`` so they do not join per token."""
  101. parts = getattr(self, "_streamed_assistant_text_parts", None)
  102. return "".join(parts) if parts else ""
  103. @_current_streamed_assistant_text.setter
  104. def _current_streamed_assistant_text(self, value: str) -> None:
  105. self._streamed_assistant_text_parts = [value] if value else []
  106. def _record_streamed_assistant_text(self, text: str) -> None:
  107. """Accumulate visible assistant text emitted through stream callbacks (superseded writers excluded)."""
  108. if isinstance(text, str) and text and not self._stream_writer_superseded():
  109. parts = getattr(self, "_streamed_assistant_text_parts", None)
  110. if parts is None:
  111. parts = self._streamed_assistant_text_parts = []
  112. parts.append(text)
  113. @staticmethod
  114. def _normalize_interim_visible_text(text: str) -> str:
  115. return re.sub(r"\s+", " ", text).strip() if isinstance(text, str) else ""
  116. def _interim_visible_and_streamed(self, content: str) -> tuple[str, str]:
  117. """(visible content, streamed text), both think-stripped and whitespace-normalized."""
  118. normalize = self._normalize_interim_visible_text
  119. streamed = getattr(self, "_current_streamed_assistant_text", "") or ""
  120. return normalize(self._strip_think_blocks(content or "")), normalize(self._strip_think_blocks(streamed))
  121. def _interim_content_was_streamed(self, content: str) -> bool:
  122. # Prefix match, not equality: the final may be streamed text plus a trailing delta. The
  123. # reverse (streamed longer) is NOT matched — it could suppress a needed resend.
  124. visible, streamed = self._interim_visible_and_streamed(content)
  125. return bool(visible and streamed) and visible.startswith(streamed)
  126. def _interim_content_fully_streamed(self, content: str) -> bool:
  127. """Exact-match variant for the gateway interim path: a True verdict finalizes the bubble
  128. as-is, so a truncated prefix must fall through to a full-text resend (#88954)."""
  129. visible, streamed = self._interim_visible_and_streamed(content)
  130. return bool(visible) and visible == streamed
  131. def _extract_codex_interim_visible_parts(self, assistant_msg: dict[str, Any]) -> list[str]:
  132. """Visible Codex commentary (``phase=commentary`` items), one string per message item.
  133. ``phase=analysis`` stays hidden (scratchpad); with ``display.show_commentary=false``
  134. commentary stays on the reasoning channel.
  135. """
  136. items = assistant_msg.get("codex_message_items") if getattr(self, "show_commentary", True) else None
  137. messages: list[str] = []
  138. for item in items if isinstance(items, list) else ():
  139. if not isinstance(item, dict) or item.get("type") != "message":
  140. continue
  141. phase, content_parts = item.get("phase"), item.get("content")
  142. if not isinstance(phase, str) or phase.strip().lower() != "commentary" or not isinstance(content_parts, list):
  143. continue
  144. visible = "".join(
  145. part["text"] for part in content_parts
  146. if isinstance(part, dict) and part.get("type") == "output_text"
  147. and isinstance(part.get("text"), str) and part["text"].strip()
  148. ).strip()
  149. visible = self._visible_commentary(visible)
  150. if visible:
  151. messages.append(visible)
  152. return messages
  153. def _visible_commentary(self, text: str) -> str:
  154. """Think-stripped, redacted commentary text ("" when nothing visible remains)."""
  155. return visible_commentary(text, strip_thinking=self._strip_think_blocks)
  156. def _extract_codex_interim_visible_text(self, assistant_msg: dict[str, Any]) -> str:
  157. """All visible Codex commentary joined, for comparison/fallback."""
  158. return "\n\n".join(self._extract_codex_interim_visible_parts(assistant_msg)).strip()
  159. def _interim_assistant_visible_text(self, assistant_msg: dict[str, Any]) -> str:
  160. """Assistant text eligible for interim delivery: structured Codex commentary first — a response can
  161. hold commentary AND a partial final answer while tools are pending, and treating content as
  162. progress leaks the answer early — else top-level content (may be a parts list)."""
  163. return (
  164. self._extract_codex_interim_visible_text(assistant_msg)
  165. or self._strip_think_blocks(flatten_message_text(assistant_msg.get("content"))).strip()
  166. )
  167. def _interim_text_was_delivered(self, text: str) -> bool:
  168. normalized = self._normalize_interim_visible_text(text)
  169. return bool(normalized) and normalized in getattr(self, "_delivered_interim_texts", set())
  170. def _record_delivered_interim_text(self, text: str) -> None:
  171. normalized = self._normalize_interim_visible_text(text)
  172. if normalized:
  173. if not isinstance(getattr(self, "_delivered_interim_texts", None), set):
  174. self._delivered_interim_texts = set()
  175. self._delivered_interim_texts.add(normalized)
  176. def _deliver_interim(self, visible: str, *, already_streamed: bool, record: list[str]) -> None:
  177. """Hand ``visible`` to ``interim_assistant_callback`` and mark ``record`` delivered; swallows callback errors.
  178. Commentary is model text ``transform_llm_output`` never sees, so a held turn drops it."""
  179. cb = None if getattr(self, "_hold_stream", False) else getattr(self, "interim_assistant_callback", None)
  180. if cb is None:
  181. return
  182. try:
  183. cb(visible, already_streamed=already_streamed)
  184. for part in record:
  185. self._record_delivered_interim_text(part)
  186. except Exception:
  187. logger.debug("interim_assistant_callback error", exc_info=True)
  188. def _fire_streamed_codex_commentary(self, text: str) -> None:
  189. """Deliver a completed live Codex commentary message immediately."""
  190. if getattr(self, "interim_assistant_callback", None) is None or not isinstance(text, str):
  191. return
  192. visible = self._visible_commentary(text)
  193. if not visible or visible == "(empty)" or self._interim_text_was_delivered(visible):
  194. return
  195. self._deliver_interim(visible, already_streamed=False, record=[visible])
  196. def _emit_interim_assistant_message(self, assistant_msg: dict[str, Any]) -> None:
  197. """Surface a real mid-turn assistant commentary message to the UI layer. Does NOT set
  198. ``_response_was_previewed`` ("the final response was shown") — the CLI would then suppress a
  199. different final summary."""
  200. if not isinstance(assistant_msg, dict):
  201. return
  202. commentary_parts = self._extract_codex_interim_visible_parts(assistant_msg)
  203. # Dedup within this message and against earlier deliveries, first occurrence wins.
  204. pending: dict[str, str] = {}
  205. for part in commentary_parts:
  206. key = self._normalize_interim_visible_text(part)
  207. if key and key not in pending and not self._interim_text_was_delivered(part):
  208. pending[key] = part
  209. undelivered_parts = list(pending.values())
  210. visible = "\n\n".join(undelivered_parts).strip() if commentary_parts else self._interim_assistant_visible_text(assistant_msg)
  211. if not visible or visible == "(empty)" or self._interim_text_was_delivered(visible):
  212. return
  213. already_streamed = self._interim_content_fully_streamed(visible)
  214. self._enqueue_stream_hook("on_interim_message", text=visible, already_streamed=already_streamed)
  215. self._deliver_interim(visible, already_streamed=already_streamed, record=undelivered_parts or [visible])
  216. def _ensure_stream_writer_state(self) -> None:
  217. """Lazily create the single-writer guard fields (#65991).
  218. The fields are normally set unconditionally in ``agent_init`` (``_STREAM_STATE``), so every
  219. ``__init__``-built agent shares ONE lock from birth and the lazy path below is never taken by
  220. two threads. Only agents constructed via ``AIAgent.__new__`` (test doubles, legacy/partially-
  221. initialized instances) skip that path; claiming/checking the writer must not crash those, so
  222. initialize the fields on first use.
  223. """
  224. if getattr(self, "_stream_writer_lock", None) is None:
  225. self._stream_writer_lock = threading.Lock()
  226. if getattr(self, "_stream_writer_tls", None) is None:
  227. self._stream_writer_tls = threading.local()
  228. for attr in ("_stream_writer_token", "_stream_writer_dropped"):
  229. if not hasattr(self, attr):
  230. setattr(self, attr, 0)
  231. def _claim_stream_writer(self) -> int:
  232. """Claim exclusive ownership of the delta sink for this stream attempt; returns its writer token.
  233. Every attempt (each provider path, each retry) claims right before consuming; claiming bumps
  234. the shared token, so an earlier attempt still alive on another thread is superseded and its
  235. late chunks fenced out. Stored per-thread: a thread that never claimed can never be fenced.
  236. See #65991.
  237. """
  238. self._ensure_stream_writer_state()
  239. with self._stream_writer_lock:
  240. self._stream_writer_token = token = self._stream_writer_token + 1
  241. self._stream_writer_tls.token = token
  242. return token
  243. def _stream_writer_is_current(self, token: int) -> bool:
  244. """True when ``token`` is still the active writer, so a stream loop can bail the instant it is superseded.
  245. active writer — i.e. no newer stream attempt has claimed the sink since (#65991).
  246. """
  247. return token == getattr(self, "_stream_writer_token", token)
  248. def _stream_writer_superseded(self) -> bool:
  249. """True when this thread claimed the sink but a newer attempt has since claimed it (never for a non-claimer).
  250. stream attempt has since claimed it — i.e. this thread is a stale writer whose chunks must be
  251. dropped (#65991).
  252. """
  253. token = getattr(getattr(self, "_stream_writer_tls", None), "token", None)
  254. return token is not None and token != getattr(self, "_stream_writer_token", token)
  255. def _note_dropped_stream_writer(self, where: str) -> None:
  256. """Record + log that a superseded stream's delta was discarded."""
  257. try:
  258. self._stream_writer_dropped = int(getattr(self, "_stream_writer_dropped", 0)) + 1
  259. except Exception:
  260. self._stream_writer_dropped = 1
  261. # Log sparsely (first drop, then powers of two) so a chatty superseded stream can't flood the log.
  262. _n = self._stream_writer_dropped
  263. if _n == 1 or (_n & (_n - 1)) == 0:
  264. logger.warning(
  265. "Dropped delta from a superseded stream writer at %s "
  266. "(discarded=%d this turn) — a stale stream tried to write into "
  267. "the turn after a retry superseded it.",
  268. where, _n,
  269. )
  270. def _stream_hook_base_payload(self) -> dict[str, Any]:
  271. return {
  272. "turn_id": getattr(self, "_current_turn_id", "") or "",
  273. "iteration": int(getattr(self, "_api_call_count", 0) or 0),
  274. "session_id": self.session_id or "",
  275. "model": self.model or "",
  276. "provider": self.provider or "",
  277. "surface": self.platform or "cli",
  278. }
  279. def _emit_stream_start(self) -> None:
  280. self._enqueue_stream_hook("on_stream_start")
  281. def _emit_stream_end(self, *, final_text: str, finished: bool, error: str | None) -> None:
  282. self._enqueue_stream_hook("on_stream_end", final_text=final_text, finished=finished, error=error)
  283. def _fire_stream_delta(self, text: str) -> None:
  284. """Fire all registered stream delta callbacks (display + TTS)."""
  285. # A superseded stream must not interleave its tokens alongside the retry that replaced it.
  286. if self._stream_writer_superseded():
  287. # See #65991.
  288. self._note_dropped_stream_writer("_fire_stream_delta")
  289. return
  290. # One paragraph break before the first text delta after a tool iteration, without
  291. # stacking blank lines across back-to-back tool iterations.
  292. prepended_break = bool(getattr(self, "_stream_needs_break", False) and text and text.strip())
  293. if prepended_break:
  294. self._stream_needs_break = False
  295. text = "\n\n" + text
  296. if isinstance(text, str):
  297. # Stateful scrubbers: per-delta regex stripping destroyed downstream state machines when a
  298. # tag was split across deltas; memory-context spans split across chunks must not leak to
  299. # the UI. Legacy callers lack the scrubber attributes and get the whole-string fallbacks.
  300. think_scrubber = getattr(self, "_stream_think_scrubber", None)
  301. # See #5719.
  302. scrubber = getattr(self, "_stream_context_scrubber", None)
  303. text = think_scrubber.feed(text) if think_scrubber is not None else self._strip_think_blocks(text)
  304. # Providers that inline reasoning (MiniMax-M3 <think>…</think>) send no reasoning delta, so the
  305. # live reasoning pane would stay empty; forward what the scrubber stripped instead (#89647).
  306. hidden = think_scrubber.last_hidden if think_scrubber is not None else ""
  307. if hidden and not getattr(self, "_native_reasoning_streamed", False):
  308. self._fire_reasoning_delta(hidden, inline=True)
  309. text = scrubber.feed(text) if scrubber is not None else sanitize_context(text)
  310. # Only strip leading newlines on the first delta — mid-stream "\n" is legitimate markdown.
  311. # Check the parts list, not the joined property (joining per token copies the whole reply).
  312. if not prepended_break and not getattr(self, "_streamed_assistant_text_parts", None):
  313. text = text.lstrip("\n")
  314. if not text:
  315. return
  316. delivered = self._deliver_to_stream_callbacks(text)
  317. self._enqueue_stream_hook("on_stream_delta", delta=text, kind="text")
  318. if delivered:
  319. self._record_streamed_assistant_text(text)
  320. def _fire_reasoning_delta(self, text: str, *, inline: bool = False) -> None:
  321. """Fire reasoning callback if registered; superseded writers are fenced like content deltas.
  322. ``inline`` marks text recovered from ``<think>`` blocks in content; any other call is a native
  323. provider reasoning delta and stops inline forwarding for the rest of this model response."""
  324. if not inline:
  325. self._native_reasoning_streamed = True
  326. if self._stream_writer_superseded():
  327. # Single-writer guard (#65991): fence out a superseded stream's reasoning deltas the same way as
  328. # content deltas.
  329. self._note_dropped_stream_writer("_fire_reasoning_delta")
  330. return
  331. self._call_quietly(self.reasoning_callback, text)
  332. # Resolve the opt-in once per stream, not per token: each lookup took _CONFIG_LOCK and
  333. # serialized every streaming thread in the process behind a config cache hit.
  334. enabled = getattr(self, "_stream_reasoning_hooks_enabled", None)
  335. if enabled is None:
  336. try:
  337. from agent.plugin_stream_hooks import stream_reasoning_deltas_enabled
  338. enabled = stream_reasoning_deltas_enabled()
  339. except Exception:
  340. logger.debug("reasoning on_stream_delta plugin hook enqueue failed", exc_info=True)
  341. return
  342. self._stream_reasoning_hooks_enabled = enabled
  343. if enabled:
  344. self._enqueue_stream_hook("on_stream_delta", label="reasoning on_stream_delta", delta=text, kind="reasoning")
  345. def _fire_tool_gen_started(self, tool_name: str) -> None:
  346. """Notify the display layer that the model is generating tool call arguments (spinner for large payloads)."""
  347. self._call_quietly(self.tool_gen_callback, tool_name)
  348. def _has_stream_consumers(self) -> bool:
  349. """Return True if any streaming consumer is registered."""
  350. try:
  351. from agent.plugin_stream_hooks import has_stream_observer_hooks
  352. if has_stream_observer_hooks():
  353. return True
  354. except Exception:
  355. logger.debug("plugin stream hook consumer check failed", exc_info=True)
  356. return self.stream_delta_callback is not None or getattr(self, "_stream_callback", None) is not None