stream_max_lifetime.py 2.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374
  1. """Opt-in total-lifetime cap for one streaming response attempt.
  2. The stale-stream detector bounds the gap BETWEEN chunks and resets on every
  3. chunk, so a stream that keeps yielding data but never finishes (a gateway
  4. trickling chunks, a runaway generation) is never cut. ``agent.stream_max_lifetime``
  5. caps the wall-clock time since the current attempt started; it never resets on
  6. chunk arrival. Default ``0`` = off, so a
  7. legitimately long reasoning stream is never cut unless the user asks for it.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. import math
  12. import time
  13. logger = logging.getLogger(__name__)
  14. def resolve_stream_max_lifetime(stale_timeout: float | None) -> float:
  15. """Cap in seconds, ``0`` when disabled. Floored at a finite stale timeout so the
  16. cap can never cut a stream the stale detector would still be waiting on."""
  17. configured = 0.0
  18. try:
  19. from hermes_cli.config import load_config_readonly
  20. agent_cfg = (load_config_readonly() or {}).get("agent")
  21. value = agent_cfg.get("stream_max_lifetime") if isinstance(agent_cfg, dict) else None
  22. if isinstance(value, (int, float)) and not isinstance(value, bool):
  23. configured = float(value)
  24. except Exception: # unreadable config = the documented default (off)
  25. logger.debug("stream_max_lifetime: config unreadable, cap off", exc_info=True)
  26. if configured <= 0 or not math.isfinite(configured):
  27. return 0.0
  28. if stale_timeout is not None and math.isfinite(stale_timeout):
  29. return max(configured, stale_timeout)
  30. return configured
  31. class StreamLifetimeExceeded(TimeoutError):
  32. """A stream attempt cut by ``agent.stream_max_lifetime``: the stream opened and was still
  33. delivering chunks, so the user-facing error names the setting, not the endpoint."""
  34. def __init__(self, elapsed: float, cap: float):
  35. self.cap = cap
  36. super().__init__(
  37. f"Response still streaming after {elapsed:.0f}s, cut by agent.stream_max_lifetime ({cap:g}s). "
  38. f"Raise agent.stream_max_lifetime, or set it to 0 to turn the cap off, if this model "
  39. f"legitimately streams this long.")
  40. class StreamLifetime:
  41. """Per-call clock: restarts whenever a new stream attempt is dispatched, so each
  42. retry gets the full cap, and after each kill (the killed attempt may unwind slowly)."""
  43. def __init__(self, cap: float):
  44. self.cap = cap
  45. self.attempt = None
  46. self.started = time.time()
  47. def exceeded(self, attempt) -> float | None:
  48. """Elapsed seconds when the cap is exceeded for ``attempt``, else ``None``."""
  49. if not self.cap:
  50. return None
  51. now = time.time()
  52. if attempt != self.attempt:
  53. self.attempt, self.started = attempt, now
  54. return None
  55. elapsed = now - self.started
  56. if elapsed <= self.cap:
  57. return None
  58. self.started = now
  59. return elapsed