thinking_timeout_guidance.py 2.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051
  1. """Thinking-timeout detection and user-facing guidance for reasoning models.
  2. A known reasoning model hitting a transport-layer error before the first content token was
  3. almost certainly idle-killed mid-think by the upstream proxy — not a context overflow — so
  4. the generic stream-drop guidance in conversation_loop is wrong for that case.
  5. """
  6. from __future__ import annotations
  7. from typing import Optional
  8. # Transport-layer failure signatures: the classifier's server-disconnect set plus the OS-level
  9. # ``broken pipe`` / ``errno 32`` the upstream kill surfaces through the OpenAI SDK wrapper.
  10. _THINKING_TIMEOUT_SUBSTRINGS: tuple[str, ...] = (
  11. "broken pipe", "errno 32", "remote protocol", "connection reset", "connection lost",
  12. "peer closed", "server disconnected",
  13. )
  14. def is_thinking_timeout(classified: object, model: str, error_msg: str) -> bool:
  15. """True when a reasoning model's thinking phase hit a transport kill.
  16. All must hold: ``classified.reason`` is the ``timeout`` FailoverReason (duck-typed via
  17. ``.value`` to avoid importing error_classifier), ``model`` is in the reasoning allowlist,
  18. and ``error_msg`` carries a transport-kill substring. The caller gates on the error having
  19. no HTTP ``status_code``.
  20. """
  21. from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
  22. if getattr(getattr(classified, "reason", None), "value", None) != "timeout":
  23. return False
  24. if get_reasoning_stale_timeout_floor(model) is None:
  25. return False
  26. return any(p in (error_msg or "").lower() for p in _THINKING_TIMEOUT_SUBSTRINGS)
  27. def build_thinking_timeout_guidance(provider: str, model: str, model_label: Optional[str] = None) -> str:
  28. """User-facing guidance appended to the final response: easiest fix first (``/reasoning
  29. low``), the config knob last. ``model`` is used verbatim in the config path so it is
  30. copy-pasteable; ``model_label`` is the optional prose name."""
  31. from hermes_constants import display_hermes_home
  32. label = model_label or model
  33. return (
  34. f"{label} was thinking for so long that the connection timed out before it wrote anything "
  35. "(common for reasoning models behind cloud gateways such as NVIDIA NIM, OpenAI, Anthropic, "
  36. "DeepSeek). Easiest fixes: `/reasoning low`, or switch to a faster model with /model. "
  37. f"Advanced: set `providers.{provider}.models.{model}.stale_timeout_seconds: 900` in "
  38. f"`{display_hermes_home()}/config.yaml` to allow a longer wait."
  39. )