stream_diag.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. """Stream diagnostics — per-attempt counters, exception chains, retry logging.
  2. When a streaming request dies mid-response these helpers record WHY (which CF
  3. edge / OpenRouter downstream served it, bytes+chunks before the drop, HTTP
  4. status, underlying httpx error class) to ``agent.log`` in full and to the
  5. user-facing status line compactly. ``run_agent`` keeps thin forwarders so
  6. existing call sites and tests patching ``run_agent.<helper>`` keep working.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import time
  11. from typing import Any, Dict, List, Optional
  12. logger = logging.getLogger(__name__)
  13. # Lowercased upstream headers captured per attempt for post-hoc analysis.
  14. STREAM_DIAG_HEADERS = (
  15. "cf-ray", "cf-cache-status", "x-openrouter-provider", "x-openrouter-model", "x-openrouter-id",
  16. "x-request-id", "x-vercel-id", "via", "server", "x-forwarded-for",
  17. )
  18. def stream_diag_init() -> dict[str, Any]:
  19. """Fresh per-attempt diagnostic dict; mutated in place by the streaming functions and read by the retry block."""
  20. return {
  21. "started_at": time.time(), "first_chunk_at": None, "chunks": 0, "bytes": 0,
  22. "headers": {}, "http_status": None, "serving_provider": None,
  23. # True once a terminal finish_reason is seen on this attempt: tells a mid-flight
  24. # transport failure (False) from an error raised after completion (#102766).
  25. "finish_reason_seen": False,
  26. }
  27. def stream_diag_capture_response(agent: Any, diag: dict[str, Any], http_response: Any) -> None:
  28. """Snapshot headers + HTTP status at stream open (so they survive a drop before the first chunk). Best-effort."""
  29. if http_response is None or not isinstance(diag, dict):
  30. return
  31. try:
  32. diag["http_status"] = getattr(http_response, "status_code", None)
  33. except Exception:
  34. pass
  35. try:
  36. headers = getattr(http_response, "headers", None) or {}
  37. captured: dict[str, str] = {}
  38. for name in STREAM_DIAG_HEADERS:
  39. try:
  40. if val := headers.get(name):
  41. captured[name] = str(val)[:120] # keep log lines bounded
  42. except Exception:
  43. continue
  44. diag["headers"] = captured
  45. except Exception:
  46. pass
  47. def flatten_exception_chain(error: BaseException) -> str:
  48. """Compact ``Outer(msg) <- Inner(msg) <- ...`` rendering, walking ``__cause__`` then ``__context__``
  49. (deduped, max 4 deep): the OpenAI SDK wraps httpx errors so only the wrapper class is visible at
  50. the catch site; the inner RemoteProtocolError/ConnectError/ReadError says WHY the stream died."""
  51. seen: list[BaseException] = []
  52. link: Optional[BaseException] = error
  53. while link is not None and len(seen) < 4 and link not in seen:
  54. seen.append(link)
  55. nxt = getattr(link, "__cause__", None) or getattr(link, "__context__", None)
  56. if nxt is None or nxt is link:
  57. break
  58. link = nxt
  59. def render(e: BaseException) -> str:
  60. msg = str(e).strip().replace("\n", " ")
  61. msg = msg[:140] + "…" if len(msg) > 140 else msg
  62. return f"{type(e).__name__}({msg})" if msg else type(e).__name__
  63. return " <- ".join(render(e) for e in seen) if seen else type(error).__name__
  64. def _diag_fields(diag: Optional[dict[str, Any]]) -> tuple:
  65. """(http_status, bytes, chunks, elapsed, ttfb, serving_provider, finish_reason_seen, upstream)
  66. for the retry log line; ``-`` when unknown."""
  67. _bytes = _chunks = 0
  68. _elapsed = 0.0
  69. _ttfb = _headers_repr = _http_status = _serving_provider = "-"
  70. _finish_reason_seen = False
  71. if isinstance(diag, dict):
  72. try:
  73. _now = time.time()
  74. _bytes = int(diag.get("bytes") or 0)
  75. _chunks = int(diag.get("chunks") or 0)
  76. _started = float(diag.get("started_at") or _now)
  77. _elapsed = max(0.0, _now - _started)
  78. _first = diag.get("first_chunk_at")
  79. if _first is not None:
  80. _ttfb = f"{max(0.0, float(_first) - _started):.2f}s"
  81. headers = diag.get("headers") or {}
  82. if isinstance(headers, dict) and headers:
  83. _headers_repr = " ".join(f"{k}={v}" for k, v in headers.items())
  84. if diag.get("http_status") is not None:
  85. _http_status = str(diag.get("http_status"))
  86. if diag.get("serving_provider"):
  87. _serving_provider = str(diag["serving_provider"])
  88. _finish_reason_seen = bool(diag.get("finish_reason_seen"))
  89. except Exception:
  90. pass
  91. return _http_status, _bytes, _chunks, _elapsed, _ttfb, _serving_provider, _finish_reason_seen, _headers_repr
  92. def log_stream_retry(
  93. agent: Any, *, kind: str, error: BaseException, attempt: int, max_attempts: int,
  94. mid_tool_call: bool, diag: Optional[dict[str, Any]] = None,
  95. ) -> None:
  96. """Structured WARNING to ``agent.log`` for a transient stream drop + retry, always logged regardless of
  97. UI verbosity. With *diag*, also records upstream headers, HTTP status, bytes/chunks, elapsed and TTFB on
  98. the dying attempt — enough to tell "one CF edge / downstream provider" from "random across runs".
  99. The ``serving_provider`` field names the downstream that actually served the attempt (relays re-roll it
  100. per request and report it only in the chunk body), so a drop can be attributed to a provider even when
  101. the response carried no ``x-openrouter-provider`` header."""
  102. try:
  103. try:
  104. _summary = agent._summarize_api_error(error)
  105. except Exception:
  106. _summary = str(error)
  107. if _summary and len(_summary) > 240:
  108. _summary = _summary[:240] + "…"
  109. try:
  110. _chain = flatten_exception_chain(error)
  111. except Exception:
  112. _chain = type(error).__name__
  113. logger.warning(
  114. "Stream %s on attempt %s/%s — retrying. subagent_id=%s depth=%s provider=%s base_url=%s "
  115. "error_type=%s error=%s chain=%s http_status=%s bytes=%d chunks=%d elapsed=%.2fs ttfb=%s "
  116. "serving_provider=%s finish_reason_seen=%s upstream=[%s]",
  117. kind, attempt, max_attempts,
  118. getattr(agent, "_subagent_id", None) or "-", getattr(agent, "_delegate_depth", 0),
  119. agent.provider or "-", agent.base_url or "-",
  120. type(error).__name__, _summary, _chain, *_diag_fields(diag),
  121. extra={"mid_tool_call": mid_tool_call},
  122. )
  123. except Exception:
  124. logger.debug("stream-retry log emit failed", exc_info=True)
  125. def emit_stream_drop(
  126. agent: Any, *, error: BaseException, attempt: int, max_attempts: int,
  127. mid_tool_call: bool, diag: Optional[dict[str, Any]] = None,
  128. ) -> None:
  129. """One compact user-visible status line for a stream drop+retry, plus the full WARNING via log_stream_retry.
  130. ``after Xs`` distinguishes "couldn't connect" (0s) from "died mid-stream" (idle-kill / proxy timeout)."""
  131. kind = "drop mid tool-call" if mid_tool_call else "drop"
  132. log_stream_retry(
  133. agent, kind=kind, error=error, attempt=attempt, max_attempts=max_attempts, mid_tool_call=mid_tool_call, diag=diag
  134. )
  135. provider = agent.provider or "provider"
  136. _suffix = ""
  137. try:
  138. started = diag.get("started_at") if isinstance(diag, dict) else None
  139. if started is not None:
  140. _suffix = f" after {max(0.0, time.time() - float(started)):.1f}s"
  141. except Exception:
  142. pass
  143. try:
  144. agent._buffer_diagnostic_status(
  145. f"⚠️ {provider} stream {kind} ({type(error).__name__}){_suffix} "
  146. f"— attempt {attempt}/{max_attempts} dropped, reconnecting"
  147. )
  148. agent._touch_activity(f"stream retry {attempt}/{max_attempts} after {type(error).__name__}")
  149. except Exception:
  150. pass
  151. # Above this size a refused/reset connect is as likely a body-size limit on the endpoint or a proxy
  152. # in front of it as an outage, so the notice says so (#97548: an ~829 KB request failed twice
  153. # before the stream opened while short chats went through).
  154. LARGE_REQUEST_HINT_BYTES = 256 * 1024
  155. def _failed_request(error: BaseException) -> Any:
  156. """The buffered ``httpx.Request`` carried by the exception chain (SDK wrapper or httpx error), else None."""
  157. link: Optional[BaseException] = error
  158. for _ in range(8):
  159. if link is None:
  160. return None
  161. try:
  162. request = getattr(link, "request", None)
  163. except RuntimeError: # httpx raises when the error was built without a request
  164. request = None
  165. if request is not None:
  166. return request
  167. link = link.__cause__ or link.__context__
  168. return None
  169. def _request_body_bytes(request: Any) -> Optional[int]:
  170. content = getattr(request, "content", None)
  171. if isinstance(content, str):
  172. return len(content.encode("utf-8"))
  173. if isinstance(content, (bytes, bytearray, memoryview)):
  174. return len(content)
  175. return None
  176. def connect_exhausted_notice(error: BaseException, *, attempts: int, base_url: Any) -> str:
  177. """The one user-facing line for \"no stream event ever arrived and the connect retries are spent\":
  178. names the host, the attempt count and the serialized request size so a body-size limit is
  179. distinguishable from an outage without reading agent.log."""
  180. from urllib.parse import urlparse
  181. from agent.stream_max_lifetime import StreamLifetimeExceeded
  182. if isinstance(error, StreamLifetimeExceeded): # the stream DID open; the opt-in cap cut every attempt
  183. return f"❌ {error} ({attempts} attempt{'s' if attempts != 1 else ''})"
  184. request = _failed_request(error)
  185. host = urlparse(str(getattr(request, "url", None) or base_url or "")).hostname or "the endpoint"
  186. size = _request_body_bytes(request)
  187. line = f"❌ Could not open a stream to {host} after {attempts} attempt{'s' if attempts != 1 else ''}"
  188. if size is None:
  189. return line + "; the endpoint looks unreachable — try again in a moment."
  190. line += f" (request {max(1, round(size / 1024))} KB)"
  191. if size >= LARGE_REQUEST_HINT_BYTES:
  192. return line + "; the endpoint or a proxy in front of it may reject requests this large."
  193. return line + "; the endpoint looks unreachable — try again in a moment."
  194. def buffer_connect_exhausted_notice(agent: Any, error: BaseException, *, attempts: int, base_url: Any) -> None:
  195. """Buffer :func:`connect_exhausted_notice` once per turn: the outer retry/fallback loop re-enters the
  196. stream call several times, and every pass exhausting the same budget must not add another copy."""
  197. text = connect_exhausted_notice(error, attempts=attempts, base_url=base_url)
  198. if any(str(msg) == text for _kind, msg in getattr(agent, "_retry_status_buffer", None) or ()):
  199. return
  200. agent._buffer_diagnostic_status(text)
  201. __all__ = [
  202. "STREAM_DIAG_HEADERS",
  203. "buffer_connect_exhausted_notice",
  204. "connect_exhausted_notice",
  205. "emit_stream_drop",
  206. "flatten_exception_chain",
  207. "log_stream_retry",
  208. "stream_diag_capture_response",
  209. "stream_diag_init",
  210. ]