chat_completion_nonstream.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. """Request-local worker lifecycle, watchdog polling, and wait status."""
  2. from agent import chat_completion_helpers as h
  3. from agent import chat_completion_wait_notice as wn
  4. class _NonStreamRequest:
  5. """One non-streaming request on a worker thread, polled by the caller.
  6. State shared between the worker (``_call``) and the poll loop lives on the
  7. instance; ``_abort_request`` may run from the poll (stranger) thread.
  8. """
  9. def __init__(self, agent, api_kwargs: dict):
  10. self.agent = agent
  11. self.api_kwargs = api_kwargs
  12. self.result = {"response": None, "error": None}
  13. self.clients = h._RequestClientRegistry(agent)
  14. # Request-local cancel flag: agent._interrupt_requested is cleared at turn
  15. # boundaries but this daemon worker can outlive the turn, so it must know THIS
  16. # request was force-closed and not surface the transport error as a bug (#6600).
  17. self.cancelled = False
  18. # Codex retirement token: the worker checks ``agent._active_codex_stream_request_token``
  19. # to know it still owns the turn; a watchdog kill clears it so a worker still
  20. # draining SSE raises instead of returning partial output as "completed"
  21. # (run_codex_stream._request_is_current). ``codex_retired`` mirrors it locally.
  22. self.codex_token = object() if agent.api_mode == "codex_responses" else None
  23. self.codex_retired = False
  24. self.wd = h._resolve_nonstream_watchdogs(agent, api_kwargs)
  25. self.codex_watchdog_state = (
  26. h.SimpleNamespace(
  27. token=self.codex_token,
  28. lock=h.threading.Lock(),
  29. last_event_ts=None,
  30. last_progress_ts=None,
  31. retry_started_ts=None,
  32. phase_aware=self.wd.idle_requires_progress,
  33. )
  34. if self.codex_token is not None
  35. else None
  36. )
  37. self.call_start = h.time.time()
  38. self.wait_notice_started_ts = None
  39. self.wait_notice = wn.WaitNoticeState()
  40. self.thread = None
  41. def _install_codex_request_token(self) -> None:
  42. if self.codex_token is not None and not self.codex_retired: # retired before start: don't re-publish
  43. self.agent._active_codex_stream_request_token = self.codex_token
  44. def _retire_codex_request_token(self) -> None:
  45. if self.codex_token is None:
  46. return
  47. self.codex_retired = True
  48. if getattr(self.agent, "_active_codex_stream_request_token", None) is self.codex_token:
  49. self.agent._active_codex_stream_request_token = None
  50. def _make_client(self, reason: str, kind: str = "openai"):
  51. # Per-request clients are registered with the abort machinery so the watchdogs
  52. # force-close the worker's connection, never the shared client (#67142).
  53. if kind == "anthropic_messages":
  54. client = self.agent._create_request_anthropic_client(reason=reason)
  55. else:
  56. client = self.agent._create_request_openai_client(reason=reason, api_kwargs=self.api_kwargs)
  57. return self.clients.set_client(client, kind=kind)
  58. def _call(self):
  59. watchdog_state_var = watchdog_context_token = None
  60. try:
  61. self._install_codex_request_token()
  62. if self.codex_watchdog_state is not None:
  63. from agent.codex_runtime import _codex_watchdog_state_var
  64. watchdog_state_var = _codex_watchdog_state_var
  65. watchdog_context_token = watchdog_state_var.set(self.codex_watchdog_state)
  66. self.result["response"] = h._dispatch_nonstreaming_api_request(
  67. self.agent, self.api_kwargs, make_client=self._make_client)
  68. except Exception as e:
  69. # Our own force-close caused this error: swallow it, the main
  70. # thread raises InterruptedError (#6600). Retirement logs at info
  71. # (a watchdog discarded output the provider already sent — what an
  72. # operator debugging a truncated reply needs); cancellation at debug.
  73. if self.codex_retired:
  74. h.logger.info("Codex worker caught %s after request retirement — "
  75. "discarding the stale partial instead of surfacing it as a completed response. %s",
  76. type(e).__name__, self.agent._client_log_context())
  77. return
  78. if self.cancelled:
  79. h.logger.debug("Non-streaming worker caught %s after request "
  80. "cancellation — exiting without surfacing a network error.", type(e).__name__)
  81. return
  82. self.result["error"] = e
  83. finally:
  84. if watchdog_state_var is not None:
  85. watchdog_state_var.reset(watchdog_context_token)
  86. # Retire first: close_once can raise, and a leaked token would let
  87. # a later worker mistake itself for the owning attempt.
  88. self._retire_codex_request_token()
  89. # Reuse reason only on a clean response; error or cancel-swallow
  90. # really closes so the next attempt builds a fresh pool.
  91. self.clients.close_once(
  92. "request_complete" if self.result["response"] is not None else "request_error_cleanup")
  93. def _abort_request(self, reason: str) -> None:
  94. """Watchdog/interrupt kill: abort the request client (kind-aware, #67142)
  95. and retire the codex token; the worker sees its own forced close via
  96. the cancel flags."""
  97. with h.contextlib.suppress(Exception):
  98. self.clients.close_once(reason)
  99. self._retire_codex_request_token()
  100. def _await_worker_after_kill(self, timeout_message: str) -> None:
  101. # Wait briefly for the worker to notice the closed connection.
  102. self.thread.join(timeout=2.0)
  103. if self.result["error"] is None and self.result["response"] is None:
  104. self.result["error"] = TimeoutError(timeout_message)
  105. def _model(self) -> str:
  106. return self.api_kwargs.get("model", "unknown")
  107. def _codex_watchdog_snapshot(self):
  108. """``(last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts)`` under one lock.
  109. ``attempt_started_ts`` is the physical attempt's origin (the reconnect marker, else
  110. ``call_start``); the notice and the kill loop must anchor to the same value."""
  111. state = self.codex_watchdog_state
  112. if state is None: # non-codex request: no watchdog reads these
  113. return (None, None, None, self.call_start)
  114. with state.lock:
  115. retry_started_ts = state.retry_started_ts
  116. return (state.last_event_ts, state.last_progress_ts, retry_started_ts,
  117. retry_started_ts if retry_started_ts is not None else self.call_start)
  118. def _pre_progress(self, last_event_ts, last_progress_ts) -> bool:
  119. """Stream open on this attempt, but no substantive model progress yet."""
  120. return self.wd.progress_timeout > 0 and last_event_ts is not None and last_progress_ts is None
  121. def _emit_wait_notice(self, elapsed: float, *, heartbeat: bool = True) -> None:
  122. wd = self.wd
  123. try:
  124. last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot()
  125. pre_progress = self._pre_progress(last_event_ts, last_progress_ts)
  126. activity_ts = attempt_started_ts if pre_progress else (
  127. retry_started_ts if retry_started_ts is not None else last_event_ts)
  128. # Lifecycle chatter is not progress: once pre-progress begins it must not
  129. # clear/reset this notice. Real progress changes phase and clears it.
  130. if (self.wait_notice_started_ts is not None and activity_ts is not None
  131. and activity_ts > self.wait_notice_started_ts):
  132. self.agent._emit_wait_notice("")
  133. self.wait_notice_started_ts = None
  134. self.wait_notice.reset()
  135. if not heartbeat:
  136. return
  137. silence = self.call_start + elapsed - (
  138. activity_ts if activity_ts is not None else self.call_start)
  139. if silence < 60.0:
  140. self.agent._touch_activity(
  141. "waiting for first stream event after reconnect"
  142. if retry_started_ts is not None else "waiting for provider response")
  143. return
  144. phase = "first_event"
  145. if pre_progress:
  146. phase = "pre_progress"
  147. elif retry_started_ts is not None:
  148. phase = "reconnect"
  149. elif last_event_ts is not None:
  150. phase = "post_event"
  151. watchdog = wn.codex_watchdog_deadline(stale_timeout=wd.stale_timeout,
  152. ttfb_enabled=wd.ttfb_enabled, ttfb_timeout=wd.ttfb_timeout,
  153. last_event_ts=last_event_ts, last_progress_ts=last_progress_ts,
  154. retry_started_ts=retry_started_ts,
  155. call_start=self.call_start, idle_enabled=wd.idle_enabled, idle_timeout=wd.idle_timeout,
  156. idle_requires_progress=wd.idle_requires_progress, progress_timeout=wd.progress_timeout,
  157. elapsed=elapsed)
  158. # One neutral notice per silence; repeating it every heartbeat made
  159. # healthy long calls read as provider trouble (#92550).
  160. if not self.wait_notice.should_emit(phase, watchdog):
  161. self.agent._touch_activity(f"waiting for provider response ({int(silence)}s, {phase})")
  162. return
  163. self.agent._emit_wait_notice(wn.wait_notice_text(
  164. self.api_kwargs.get('model', 'the provider'), silence, phase, watchdog))
  165. self.wait_notice_started_ts = self.call_start + elapsed
  166. except Exception:
  167. h.logger.debug("wait-notice construction failed", exc_info=True)
  168. def _ttfb_kill(self, elapsed: float) -> None:
  169. """No parsed Codex event past the first-event cutoff — kill so the retry loop
  170. reconnects instead of waiting out the stale timeout."""
  171. agent, wd = self.agent, self.wd
  172. silent_hint = h._codex_silent_hang_hint(agent, self.api_kwargs)
  173. h.logger.warning("Codex stream produced no parsed stream event within TTFB cutoff "
  174. "(%.0fs > %.0fs, model=%s). Backend accepted the connection "
  175. "but sent no stream events. Killing connection so the retry loop can reconnect.", elapsed,
  176. wd.ttfb_timeout, self._model())
  177. agent._buffer_diagnostic_status(
  178. f"⚠️ No first stream event from provider in {int(elapsed)}s (codex stream, model: {self._model()}). "
  179. f"Reconnecting." + (f" {silent_hint}" if silent_hint else ""))
  180. self._abort_request("codex_ttfb_kill")
  181. agent._emit_diagnostic_wait(f"⚠ no response from provider in {int(elapsed)}s — reconnecting...")
  182. agent._touch_activity(f"codex stream killed after {int(elapsed)}s with no first stream event")
  183. self._await_worker_after_kill(
  184. f"Codex stream produced no parsed stream event within {int(elapsed)}s "
  185. f"(TTFB threshold: {int(wd.ttfb_timeout)}s)"
  186. + (f". {silent_hint}" if silent_hint else ""))
  187. def _progress_kill(self, elapsed: float) -> None:
  188. """The stream opened, but this physical attempt never made model progress."""
  189. agent, wd = self.agent, self.wd
  190. h.logger.warning("Codex stream produced lifecycle events but no substantive model progress "
  191. "for %.0fs (threshold %.0fs, model=%s, context=~%s tokens). Reconnecting.",
  192. elapsed, wd.progress_timeout, self._model(), f"{wd.est_tokens:,}")
  193. agent._buffer_diagnostic_status(
  194. f"⚠️ Codex stream opened but made no model progress for {int(elapsed)}s "
  195. f"(model: {self._model()}). Reconnecting.")
  196. self._abort_request("codex_progress_kill")
  197. agent._touch_activity(f"codex stream killed after {int(elapsed)}s without model progress")
  198. self._await_worker_after_kill(
  199. f"Codex stream produced no substantive model progress for {int(elapsed)}s "
  200. f"(progress threshold: {int(wd.progress_timeout)}s)")
  201. def _idle_kill(self, event_stale_elapsed: float) -> None:
  202. """SSE events stopped after the phase-specific idle arm point.
  203. Only the implicit official OpenAI Codex policy arms on substantive model
  204. progress; compatible providers and explicit operator timeouts arm on first
  205. parsed event. Once armed, any parsed SSE event refreshes transport activity.
  206. """
  207. agent, wd = self.agent, self.wd
  208. arm_point = "model progress began" if wd.idle_requires_progress else "the first parsed event"
  209. h.logger.warning("Codex stream produced no SSE events for %.0fs after %s "
  210. "(threshold %.0fs, model=%s, context=~%s tokens). Killing "
  211. "connection so the retry loop can reconnect.", event_stale_elapsed, arm_point, wd.idle_timeout,
  212. self._model(), f"{wd.est_tokens:,}")
  213. agent._buffer_diagnostic_status(
  214. f"⚠️ Codex stream sent no events for {int(event_stale_elapsed)}s after {arm_point} "
  215. f"(model: {self._model()}). Reconnecting.")
  216. self._abort_request("codex_stream_idle_kill")
  217. agent._touch_activity(f"codex stream killed after {int(event_stale_elapsed)}s with no SSE events")
  218. self._await_worker_after_kill(
  219. f"Codex stream produced no SSE events for {int(event_stale_elapsed)}s "
  220. f"after {arm_point} (threshold: {int(wd.idle_timeout)}s)")
  221. def _stale_kill(self, elapsed: float) -> None:
  222. """No response within the stale timeout: kill and count toward the
  223. circuit breaker (#58962, see ``_stale_streak``)."""
  224. agent, wd = self.agent, self.wd
  225. silent_hint = h._codex_silent_hang_hint(agent, self.api_kwargs)
  226. h._report_stale_nonstream_kill(agent, self.api_kwargs, elapsed, wd.stale_timeout, hint=silent_hint)
  227. self._abort_request("stale_call_kill")
  228. h._bump_stale_streak(agent)
  229. h._touch_stale_kill_activity(agent, elapsed)
  230. self._await_worker_after_kill(
  231. f"Non-streaming API call timed out after {int(elapsed)}s with no response (threshold: {int(wd.stale_timeout)}s)"
  232. + (f". {silent_hint}" if silent_hint else ""))
  233. def _interrupt(self, elapsed: float) -> None:
  234. agent = self.agent
  235. last_event_ts, _, _, _ = self._codex_watchdog_snapshot()
  236. h._record_interrupted_provider_wait(agent, elapsed,
  237. response_started=self.wd.codex and last_event_ts is not None
  238. )
  239. # Mark cancelled BEFORE force-closing so the worker treats the transport
  240. # error as a cancel (#6600). Never close the shared client (releasing a
  241. # TLS FD mid-SSL-BIO corrupted an unrelated SQLite DB, #67142). Then let
  242. # the worker unwind Relay scopes before raising (#81521).
  243. self.cancelled = True
  244. h.logger.debug("Force-closing httpx client due to interrupt (not a network error).")
  245. self._abort_request("interrupt_abort")
  246. h._join_worker_for_relay_teardown(self.thread, label="Non-streaming")
  247. raise InterruptedError("Agent interrupted during API call")
  248. def run(self):
  249. agent, wd = self.agent, self.wd
  250. if wd.codex:
  251. # Reset before the worker starts so a marker left over from a previous
  252. # call on this agent can't be misread as the first event for this one.
  253. with self.codex_watchdog_state.lock:
  254. self.codex_watchdog_state.last_event_ts = None
  255. self.codex_watchdog_state.last_progress_ts = None
  256. self.codex_watchdog_state.retry_started_ts = None
  257. agent._touch_activity("waiting for non-streaming API response")
  258. self.thread = t = h.threading.Thread(target=h._context_thread_target(self._call), daemon=True)
  259. t.start()
  260. poll_count = 0
  261. while t.is_alive():
  262. t.join(timeout=0.3)
  263. poll_count += 1
  264. # Keep the quiet gateway heartbeat; only silence warrants a notice.
  265. # Resumed events clear our notice on the next poll, not 30s later.
  266. now = h.time.time()
  267. elapsed = now - self.call_start
  268. self._emit_wait_notice(elapsed, heartbeat=poll_count % 100 == 0)
  269. last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot()
  270. attempt_elapsed = now - attempt_started_ts
  271. if (wd.ttfb_enabled and last_event_ts is None and attempt_elapsed > wd.ttfb_timeout):
  272. self._ttfb_kill(attempt_elapsed)
  273. break
  274. if (self._pre_progress(last_event_ts, last_progress_ts)
  275. and attempt_elapsed > wd.progress_timeout):
  276. self._progress_kill(attempt_elapsed)
  277. break
  278. idle_elapsed = now - last_event_ts if last_event_ts is not None else None
  279. if (retry_started_ts is None and wd.idle_enabled and idle_elapsed is not None
  280. and (not wd.idle_requires_progress or last_progress_ts is not None)
  281. and idle_elapsed > wd.idle_timeout):
  282. self._idle_kill(idle_elapsed)
  283. break
  284. if elapsed > wd.stale_timeout:
  285. self._stale_kill(elapsed)
  286. break
  287. if agent._interrupt_requested:
  288. self._interrupt(elapsed)
  289. if self.result["error"] is not None:
  290. raise self.result["error"]
  291. # Success — the provider proved responsive: clear the breaker (#58962).
  292. if self.result["response"] is not None:
  293. h._reset_stale_streak(agent)
  294. return self.result["response"]