auxiliary_stream_watchdog.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. """No-progress watchdog for streamed chat-completions auxiliary calls (#100501).
  2. The consumer blocks inside the SDK's chunk iterator, so a silent stream is only noticed when the
  3. httpx read timeout (the full auxiliary request budget, >= 300s for compression) expires; the host
  4. compression wait gives up first and the attempt dies with no fallback. The window is enforced from a
  5. Timer that only ``shutdown()``s this attempt's socket: that is FD-safe from a stranger thread
  6. (#70773), wakes the owner's blocked read, and the owner still closes the stream in its ``finally``
  7. so the pool drops the dead connection instead of recycling it.
  8. """
  9. from __future__ import annotations
  10. import inspect
  11. import threading
  12. from types import SimpleNamespace
  13. from typing import Any, Optional
  14. from agent.sleep_clock import awake_seconds
  15. def _stream_socket(stream: Any) -> Any:
  16. """The raw socket under an SDK chunk stream (``stream.response`` is the ``httpx.Response``)."""
  17. from agent.agent_runtime_helpers import _socket_from_response
  18. response = getattr(stream, "response", None)
  19. return _socket_from_response(response) if response is not None else None
  20. def _close_chunk_stream(chunks: Any, *, allow_aclose: bool = False) -> Any:
  21. """Best-effort ``close()`` (or ``aclose()``); returns a pending awaitable or None."""
  22. close_fn = getattr(chunks, "close", None) or (
  23. getattr(chunks, "aclose", None) if allow_aclose else None)
  24. if not callable(close_fn):
  25. return None
  26. try:
  27. result = close_fn()
  28. except Exception:
  29. return None
  30. return result if inspect.isawaitable(result) else None
  31. class ChatStreamWatchdog:
  32. """Re-armed by substantive chunks only. Silence before the first token is bounded by
  33. *first_token_window* (the main loop's stale patience: reasoning models think silently for
  34. minutes); ``None`` leaves it to the request timeout (a local server's silent prefill).
  35. Windows run on the awake clock, so time the computer spends asleep doesn't count."""
  36. def __init__(self, stream: Any, window: float, *, first_token_window: Optional[float] = None):
  37. self.window = window
  38. self.first_token_window = first_token_window
  39. self._started = awake_seconds()
  40. self._lock = threading.Lock()
  41. self._deadline: Optional[float] = None
  42. self._timer: Optional[threading.Timer] = None
  43. self._generation = 0
  44. self._done = False
  45. self.saw_progress = False
  46. self.fired = False
  47. # Pin THIS attempt's socket now and retire the watchdog before the response hands its
  48. # connection back to the pool: a timer callback that already woke must never shut down a
  49. # pooled connection a later request has acquired (keepalive reuse).
  50. self._sock = _stream_socket(stream)
  51. self._fence_release(getattr(getattr(stream, "response", None), "stream", None))
  52. if first_token_window is not None:
  53. self._rearm(first_token_window)
  54. def _fence_release(self, body: Any) -> None:
  55. release = getattr(body, "close", None)
  56. if self._sock is None or not callable(release):
  57. return
  58. def fenced_release() -> None:
  59. with self._lock:
  60. self._done = True
  61. release()
  62. try:
  63. body.close = fenced_release
  64. except (AttributeError, TypeError):
  65. self._sock = None # cannot fence the release: never shut down a socket we may not own
  66. def _rearm(self, window: float) -> None:
  67. # A pending timer re-checks the deadline when it fires; replace it only when the new
  68. # deadline is earlier (the first token swaps the long first-token window for the short one).
  69. previous, self._deadline = self._deadline, awake_seconds() + window
  70. if self._timer is None or (previous is not None and self._deadline < previous):
  71. if self._timer is not None:
  72. self._timer.cancel()
  73. self._schedule(window)
  74. def _schedule(self, delay: float) -> None:
  75. self._generation += 1
  76. self._timer = threading.Timer(max(delay, 0.0), self._fire, args=(self._generation,))
  77. self._timer.daemon = True
  78. self._timer.start()
  79. def progress(self) -> None:
  80. with self._lock:
  81. self.saw_progress = True
  82. if not self._done:
  83. self._rearm(self.window)
  84. def _fire(self, generation: int) -> None:
  85. # Decision and shutdown happen under the lock the fenced release takes, so a retired
  86. # attempt (or a timer superseded by ``_rearm`` after it already woke) never acts.
  87. with self._lock:
  88. if self._done or self._deadline is None or generation != self._generation:
  89. return
  90. remaining = self._deadline - awake_seconds()
  91. if remaining > 0:
  92. self._schedule(remaining)
  93. return
  94. self.fired = True
  95. if self._sock is not None:
  96. from agent.agent_runtime_helpers import _shutdown_socket
  97. _shutdown_socket(self._sock)
  98. def finish(self) -> None:
  99. with self._lock:
  100. self._done = True
  101. timer = self._timer
  102. if timer is not None:
  103. timer.cancel()
  104. def timeout_error(self) -> TimeoutError:
  105. """Zero-output stalls say "no-progress timeout" (same-provider retry stays allowed, see
  106. ``_should_skip_same_provider_retry``); a mid-stream stall goes straight to fallback."""
  107. elapsed = awake_seconds() - self._started
  108. if not self.saw_progress:
  109. return TimeoutError(
  110. f"Auxiliary chat stream produced no output within {self.first_token_window or 0:.1f}s "
  111. f"(no-progress timeout, {elapsed:.1f}s elapsed)")
  112. return TimeoutError(
  113. f"Auxiliary chat stream stalled: no new output for {self.window:.1f}s "
  114. f"({elapsed:.1f}s elapsed, timed out)")
  115. def chat_stream_windows(client: Any, kwargs: dict, task: Optional[str]) -> tuple[float, Optional[float]]:
  116. """(inter-chunk window, first-token window) for a streamed chat-completions attempt. Between
  117. chunks: the Codex guard's window (``auxiliary.<task>.no_progress_timeout``, 60s default). Before
  118. the first token: the main loop's cloud stale patience for the routed provider (its explicit
  119. ``providers.<id>.stale_timeout_seconds``, else context-scaled with the reasoning-model floor), so
  120. a model thinking silently on a large prompt is not cut; local servers keep their silent prefill
  121. on the request timeout (None). Both are capped at the request timeout."""
  122. from agent.auxiliary_client import (
  123. _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS, _RELAY_AUX_CALL_CONTEXT, _get_task_no_progress_timeout)
  124. from agent.chat_completion_helpers import _cloud_stale_timeout_for
  125. from agent.model_metadata import is_local_endpoint
  126. window = _get_task_no_progress_timeout(task or "") or _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS
  127. first = None
  128. if not is_local_endpoint(str(getattr(client, "base_url", "") or "")):
  129. route = SimpleNamespace(provider=(_RELAY_AUX_CALL_CONTEXT.get() or {}).get("stream_provider") or "",
  130. model=kwargs.get("model"))
  131. first = max(window, _cloud_stale_timeout_for(route, kwargs))
  132. timeout = kwargs.get("timeout")
  133. if isinstance(timeout, (int, float)) and timeout > 0:
  134. window = min(window, float(timeout))
  135. first = min(first, float(timeout)) if first is not None else None
  136. return window, first
  137. def consume_chat_stream(chunks: Any, acc: Any, no_progress: Optional[tuple[float, Optional[float]]]) -> None:
  138. """Feed *chunks* into the accumulator *acc*. Under a ``(window, first_token_window)`` watchdog a
  139. silent stream raises TimeoutError, except after the terminal chunk: with finish_reason (and any
  140. usage) in hand a stall is teardown, so the completed response and its billed usage are kept."""
  141. if not no_progress:
  142. for chunk in chunks:
  143. acc.feed(chunk)
  144. return
  145. watchdog = ChatStreamWatchdog(chunks, no_progress[0], first_token_window=no_progress[1])
  146. try:
  147. for chunk in chunks:
  148. if acc.feed(chunk):
  149. watchdog.progress()
  150. except Exception as exc:
  151. if not watchdog.fired:
  152. raise
  153. if not acc.finish_reason:
  154. raise watchdog.timeout_error() from exc
  155. finally:
  156. watchdog.finish()
  157. if watchdog.fired and not acc.finish_reason:
  158. raise watchdog.timeout_error()