chat_completion_stream_monitor.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """Display and heartbeat phase of the request-local streaming monitor.
  2. Also holds the stale-kill follow-up (re-arm, abandon a wedged worker, stop claiming liveness
  3. for a wedged inline call) and the shared "no visible text was delivered" check
  4. (``_forget_invisible_deltas``) used by the error and abandon paths.
  5. """
  6. import contextlib
  7. import logging
  8. import time
  9. from types import SimpleNamespace
  10. from urllib.parse import urlsplit
  11. from agent import chat_completion_wait_notice as wn
  12. from agent.model_metadata import is_local_endpoint
  13. from agent.proxy_bypass import is_loopback_host
  14. from agent.sleep_clock import SleepWatch
  15. from agent.stream_max_lifetime import StreamLifetime, StreamLifetimeExceeded, resolve_stream_max_lifetime
  16. logger = logging.getLogger(__name__)
  17. # Kills of one dispatched attempt whose worker never unwinds before the monitor gives up on it.
  18. _STALE_ABANDON_KILLS = 3
  19. class StreamingWaitMonitor:
  20. # Stale-kill bookkeeping: consecutive kills of the SAME dispatched attempt, and the error
  21. # that ends a call whose worker those kills could not unwind.
  22. _last_killed_attempt = 0
  23. _same_attempt_kills = 0
  24. _abandon_error = None
  25. # Inline call wedged past the limit: the stale-timer re-arm time after which activity touches
  26. # are withheld (None = claim liveness as usual). A real chunk after it lifts the hold.
  27. _liveness_withheld_at = None
  28. _lifetime = StreamLifetime(0.0) # opt-in agent.stream_max_lifetime; real clock set when the monitor starts
  29. _lifetime_kill = None # (attempt id, elapsed s) of the latest cap kill
  30. _released_attempt = 0 # the attempt the worker's except path last caught (it let go of it)
  31. def _claims_liveness(self) -> bool:
  32. since = self._liveness_withheld_at
  33. return since is None or self.last_chunk_time["t"] > since
  34. def _rearm_or_abandon_after_kill(self, elapsed: float) -> None:
  35. """After a stale kill: re-arm the stale timer and announce the reconnect, unless the
  36. kill bookkeeping gave up on a wedged attempt (see ``_note_attempt_kill``)."""
  37. # Reset the timer so we don't kill repeatedly while the worker unwinds; the cap clock
  38. # too, so a cap at the stale timeout does not cut (and count) the same window again.
  39. self.last_chunk_time["t"] = self._lifetime.started = time.time()
  40. if self._note_attempt_kill(elapsed, "stale"):
  41. return
  42. if self._claims_liveness(): # the reconnect notice touches activity too (_emit_wait_notice)
  43. self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")
  44. self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")
  45. def _note_attempt_kill(self, elapsed: float, kind: str) -> bool:
  46. """Count a stale or ``stream_max_lifetime`` kill of the current attempt. Once the same
  47. dispatched attempt has been killed ``_STALE_ABANDON_KILLS`` times without unwinding,
  48. the request is parked where a socket shutdown cannot reach it, and re-killing it every
  49. window would stall the turn forever: a worker thread is abandoned (returns True); an
  50. inline call (cron, delegated children) runs on the caller's thread and cannot be, so
  51. the monitor stops touching activity and the owner's inactivity watchdog ends it."""
  52. with self.stream_attempt_lock:
  53. attempt = int(self.stream_attempt_state["current"])
  54. if attempt == self._released_attempt: # retry backoff: the worker is not stuck in it
  55. return False
  56. rekill = bool(attempt) and attempt == self._last_killed_attempt
  57. self._last_killed_attempt = attempt
  58. self._same_attempt_kills = self._same_attempt_kills + 1 if rekill else 1
  59. if not rekill:
  60. self._liveness_withheld_at = None
  61. if not attempt or self._same_attempt_kills < _STALE_ABANDON_KILLS:
  62. return False
  63. if self.worker is not None:
  64. self._abandon_wedged_worker(elapsed, attempt, kind)
  65. return True
  66. if self._liveness_withheld_at is None:
  67. logger.error(
  68. "Inline streaming call did not unwind after %d %s kills of attempt %s; no longer "
  69. "reporting it as active, so the inactivity watchdog can end it. model=%s",
  70. self._same_attempt_kills, kind, attempt, self.api_kwargs.get("model", "unknown"))
  71. self._liveness_withheld_at = self.last_chunk_time["t"]
  72. return False
  73. def _forget_invisible_deltas(self, exc) -> bool:
  74. """Deltas fired but nothing visible reached a consumer (whitespace/think-only deltas,
  75. or no display consumer at all) and no tool call is in flight: from the user's and the
  76. model's point of view NOTHING was delivered. The "partial delivery" stub would be
  77. EMPTY and the loop would ask the model to continue from nowhere, so it repeats the
  78. lost step (#112419). Reset delivery so the failure is handled as undelivered (retry
  79. the unchanged request) instead. Returns whether a partial tool call is in flight."""
  80. tool_in_flight = bool(self.result.get("partial_tool_names")) or self.provider_tool_in_flight["yes"]
  81. if self.deltas_were_sent["yes"] and not tool_in_flight and not self._visible_text_delivered():
  82. logging.getLogger("agent.chat_completion_helpers").warning(
  83. "Stream died after deltas but before any visible text was delivered (0 chars, "
  84. "no tool call in flight); treating as an undelivered stream failure: %s", exc)
  85. self._quiet(self.agent._reset_stream_delivery_tracking)
  86. self.deltas_were_sent["yes"] = False
  87. self.first_delta_fired["done"] = False
  88. return tool_in_flight
  89. def _abandon_wedged_worker(self, elapsed: float, attempt: int, kind: str) -> None:
  90. """Like the /stop path, leave the (daemon) worker behind and end this call with a
  91. retryable timeout; ``run()`` discards anything the worker writes later."""
  92. logger.error(
  93. "Streaming worker did not unwind after %d %s kills of attempt %s (%.0fs); "
  94. "abandoning the request. model=%s", self._same_attempt_kills, kind, attempt, elapsed,
  95. self.api_kwargs.get("model", "unknown"))
  96. self._request_cancelled["value"] = True
  97. if kind == "stale":
  98. self._abandon_error = TimeoutError(
  99. f"Provider stream stalled: no chunks and the request worker did not unwind after "
  100. f"{self._same_attempt_kills} stale-stream kills; request abandoned.")
  101. else: # the copy names the setting, like a cap kill that did unwind
  102. self._abandon_error = StreamLifetimeExceeded(elapsed, self._lifetime.cap)
  103. self._call_done.set()
  104. def _poll_local_load_notice(self, now: float) -> bool:
  105. """Managed local server: surface a cold model's weight-load progress
  106. instead of the 60s neutral "waiting on <model>" notice. Polled ~1s only while no
  107. REAL chunk arrived for 2s+ (never during healthy token flow); in-memory,
  108. no network. True while loading = heartbeat liveness, skip the rest of
  109. this iteration (the stale detector's local floor dwarfs any load)."""
  110. from agent.chat_completion_local import _managed_local_load_notice
  111. m = self._mon
  112. if self._show_prompt_progress(now):
  113. return True
  114. if now - self.last_chunk_time["t"] < 2.0 or now - m.last_load_poll < 1.0:
  115. return False
  116. m.last_load_poll = now
  117. _load_notice = _managed_local_load_notice(self.agent, self.api_kwargs)
  118. if _load_notice is not None:
  119. m.wait_notice_started_ts = None # The local loader now owns the display.
  120. m.wait_notice.reset()
  121. self.agent._emit_wait_notice(_load_notice)
  122. self.agent._touch_activity("local model loading")
  123. m.load_notice_shown, m.load_notice_misses, m.last_heartbeat = True, 0, now # loading IS liveness
  124. return True
  125. if m.load_notice_shown:
  126. # One missed sample is routine (probe timeout under load); clearing on it strobed the line.
  127. m.load_notice_misses += 1
  128. if m.load_notice_misses >= 3:
  129. m.load_notice_shown, m.load_notice_misses = False, 0
  130. self.agent._emit_wait_notice("")
  131. return False
  132. def _show_prompt_progress(self, now: float) -> bool:
  133. """Prompt progress straight from the stream's own progress chunks (llama-server with
  134. ``return_progress``), at most once a second until the first token. Clears the line
  135. when the first token arrives. True when it showed progress."""
  136. from agent.local_prompt_progress import prompt_progress_percent
  137. m = self._mon
  138. state = getattr(self, "prompt_progress", None)
  139. diag = getattr(getattr(self, "clients", None), "diag", None)
  140. first_token = isinstance(diag, dict) and bool(diag.get("first_chunk_at"))
  141. if not state or state["progress"] is None:
  142. return False
  143. if first_token:
  144. if m.load_notice_shown:
  145. m.load_notice_shown, m.load_notice_misses = False, 0
  146. self.agent._emit_wait_notice("")
  147. return False
  148. if now - state["t"] > 60.0 or now - m.last_load_poll < 1.0:
  149. return False
  150. m.last_load_poll = now
  151. percent = prompt_progress_percent(state["progress"])
  152. m.wait_notice_started_ts = None # The prompt-progress line owns the display.
  153. m.wait_notice.reset()
  154. self.agent._emit_wait_notice(
  155. f"⚙ processing prompt — {percent}%" if percent is not None else "⚙ processing prompt")
  156. m.load_notice_shown, m.load_notice_misses, m.last_heartbeat = True, 0, now
  157. return True
  158. def _skip_sleep(self, slept: float) -> None:
  159. """The computer slept and froze this request with a server on the same machine: move the
  160. stall clock past the sleep, so the request gets its full patience again after wake. Never
  161. past now: a chunk can land between the sleep reading and this write."""
  162. if slept <= 0:
  163. return
  164. now = time.time()
  165. self.last_chunk_time["t"] = min(self.last_chunk_time["t"] + slept, now)
  166. self._mon.last_heartbeat = min(self._mon.last_heartbeat + slept, now)
  167. logger.info("Computer slept for %.0fs during a model request; the stall timer skips that time.", slept)
  168. self.agent._touch_activity(f"resumed after the computer slept for {int(slept)}s")
  169. def _heartbeat(self, waiting_secs: int) -> None:
  170. """Gateway inactivity heartbeat: the start-to-first-chunk gap (thinking,
  171. local prefill) can exceed the gateway timeout. Withheld for a wedged inline call."""
  172. if not self._claims_liveness():
  173. return
  174. if waiting_secs >= 60.0:
  175. # No chunks for 60s+: say WHAT the wait is and WHEN recovery kicks in —
  176. # once per silence, not every heartbeat (#92550).
  177. stale = self._stream_stale_timeout
  178. watchdog = ("stream stale", stale - waiting_secs) if stale is not None and stale != float("inf") else None
  179. diag = getattr(getattr(self, "clients", None), "diag", None)
  180. phase = "post_chunk" if isinstance(diag, dict) and diag.get("first_chunk_at") else "first_chunk"
  181. if not self._mon.wait_notice.should_emit(phase, watchdog):
  182. self.agent._touch_activity(f"waiting for stream response ({waiting_secs}s, {phase})")
  183. return
  184. self._mon.wait_notice_started_ts = self._mon.last_heartbeat
  185. self.agent._emit_wait_notice(wn.wait_notice_text(
  186. self.api_kwargs.get('model', 'the provider'), waiting_secs, phase, watchdog))
  187. else:
  188. # Chunks are flowing — keep the tracker fresh, leave the display alone.
  189. self.agent._touch_activity(f"waiting for stream response ({waiting_secs}s, no chunks yet)")
  190. def _check_stream_lifetime(self) -> None:
  191. """Opt-in ``agent.stream_max_lifetime``: a stream that keeps sending chunks but never
  192. finishes never trips the stale check. Kill it like a stale kill (cancel the attempt,
  193. abort the request-local client) so the retry loop or the partial-delivery continuation
  194. takes over. Never counted in the cross-turn stale breaker: the provider WAS responding,
  195. and a streak of cap kills would refuse the next healthy request with nothing sent."""
  196. if not self._lifetime.cap or self._call_done.is_set(): # off (the default) or call over
  197. return
  198. with self.stream_attempt_lock:
  199. attempt = int(self.stream_attempt_state["current"])
  200. # A released attempt (retry backoff) has no stream left to cut.
  201. elapsed = self._lifetime.exceeded(attempt) if attempt and attempt != self._released_attempt else None
  202. if elapsed is None:
  203. return
  204. model = self.api_kwargs.get("model", "unknown")
  205. logger.warning("Stream still running after %.0fs (stream_max_lifetime %.0fs) — killing connection. "
  206. "model=%s", elapsed, self._lifetime.cap, model)
  207. self.agent._buffer_diagnostic_status(
  208. f"⚠️ Response still streaming after {int(elapsed)}s (stream_max_lifetime "
  209. f"{int(self._lifetime.cap)}s, model: {model}). Aborting...")
  210. killed = self._attempt_stream_response
  211. self._lifetime_kill = (attempt, elapsed) # before the abort: the woken worker reads it at once
  212. with contextlib.suppress(Exception):
  213. self._cancel_current_stream_attempt("stream_max_lifetime_kill")
  214. self.clients.close_once("stream_max_lifetime_kill")
  215. self._shutdown_stale_attempt_socket(killed)
  216. # Accounted, never bumped: a stale re-kill or /stop while it unwinds must not strike it either.
  217. self._stale_counted_attempts.add(attempt)
  218. self.last_chunk_time["t"] = time.time() # the stale timer restarts with the kill
  219. # A worker the abort cannot unwind would be re-cut every window forever (the stale
  220. # detector may be off): same give-up as a stale kill, without its reconnect notice.
  221. self._note_attempt_kill(elapsed, "stream_max_lifetime")
  222. def _as_lifetime_error(self, e: Exception) -> Exception:
  223. """The worker's transport error for an attempt the cap killed, replaced by one that names
  224. ``agent.stream_max_lifetime`` (the retry, exhausted and partial-delivery lines show it)."""
  225. kill = self._lifetime_kill
  226. if kill is None or isinstance(e, StreamLifetimeExceeded): # no cap kill: touch nothing else
  227. return e
  228. with self.stream_attempt_lock:
  229. if kill[0] != int(self.stream_attempt_state["current"]):
  230. return e
  231. err = StreamLifetimeExceeded(kill[1], self._lifetime.cap)
  232. err.__cause__ = e
  233. return err
  234. def _monitor_loop(self) -> None:
  235. try:
  236. self._poll_until_done()
  237. except Exception:
  238. # A dying watchdog otherwise ends stale/cap/interrupt monitoring with no trace.
  239. logger.error("stream watchdog monitor loop aborted abnormally", exc_info=True)
  240. raise
  241. def _poll_until_done(self) -> None:
  242. _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches
  243. self._mon = SimpleNamespace(
  244. last_heartbeat=time.time(), last_load_poll=0.0,
  245. load_notice_shown=False, load_notice_misses=0, wait_notice_started_ts=None,
  246. wait_notice=wn.WaitNoticeState(),
  247. )
  248. _is_local_base = bool(self.agent.base_url) and is_local_endpoint(self.agent.base_url)
  249. # Only a server on this machine sleeps with it; a remote one kept working (or died).
  250. sleep_watch = (SleepWatch(interval=0.3) if self.agent.base_url
  251. and is_loopback_host(urlsplit(self.agent.base_url).hostname) else None)
  252. self._lifetime = StreamLifetime(resolve_stream_max_lifetime(self._stream_stale_timeout))
  253. logger.debug("stream watchdog armed: stale=%.0fs cap=%.0fs model=%s", self._stream_stale_timeout,
  254. self._lifetime.cap, self.api_kwargs.get("model", "unknown"))
  255. while not self._call_done.is_set():
  256. self._call_done.wait(timeout=0.3)
  257. if sleep_watch is not None:
  258. self._skip_sleep(sleep_watch.slept())
  259. _hb_now = time.time()
  260. if _is_local_base and self._poll_local_load_notice(_hb_now):
  261. continue
  262. # Reasoning callbacks do not clear the classic CLI spinner. The empty
  263. # protocol payload resets status without adding synthetic reasoning.
  264. if (self._mon.wait_notice_started_ts is not None
  265. and self.last_chunk_time["t"] > self._mon.wait_notice_started_ts):
  266. self.agent._emit_wait_notice("")
  267. self._mon.wait_notice_started_ts = None
  268. self._mon.wait_notice.reset()
  269. if _hb_now - self._mon.last_heartbeat >= _HEARTBEAT_INTERVAL:
  270. self._mon.last_heartbeat = _hb_now
  271. self._heartbeat(int(_hb_now - self.last_chunk_time["t"]))
  272. _stale_elapsed = time.time() - self.last_chunk_time["t"]
  273. if _stale_elapsed > self._stream_stale_timeout:
  274. self._mon.wait_notice_started_ts = None # Reconnect status has its own owner.
  275. self._mon.wait_notice.reset()
  276. self._kill_stale_stream(_stale_elapsed)
  277. self._check_stream_lifetime()
  278. if self.agent._interrupt_requested:
  279. self._abort_for_interrupt(_stale_elapsed)
  280. return