conversation_compression_deadline.py 4.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879
  1. """Compression time limits: the configured idle and total limits, and the fence deadline shared live
  2. with the summary stream (sibling of ``conversation_compression``).
  3. Compression waits a total time limit for its summary. That limit stops a summary that trickles
  4. out a token at a time; on a laptop GPU, reading a 150K-token conversation alone can take longer
  5. than the whole limit. Time llama-server spends reading the prompt therefore pushes the fence
  6. deadline back (``CompressionCommitFence.extend_deadline``), and hosts read the deadline from the
  7. fence instead of adding the configured limit to the time they started waiting.
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. from typing import TYPE_CHECKING, Optional
  12. if TYPE_CHECKING:
  13. from agent.conversation_compression import CompressionCommitFence
  14. # Defaults for the in-agent progress-aware wrap; mirror hermes_cli.config.DEFAULT_CONFIG["compression"] keys.
  15. DEFAULT_CONTEXT_TIMEOUT_SECONDS = 120.0
  16. DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS = 600.0
  17. def resolve_context_compression_timeouts(compression_cfg: Optional[dict] = None) -> tuple[float, float]:
  18. """Return ``(idle_timeout_seconds, total_ceiling_seconds)``.
  19. ``idle_timeout_seconds <= 0`` disables the progress-aware wrapper. The ceiling is clamped to at least one
  20. idle window when the idle budget is positive."""
  21. idle = DEFAULT_CONTEXT_TIMEOUT_SECONDS
  22. ceiling = DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS
  23. cfg = compression_cfg
  24. if cfg is None:
  25. cfg = {}
  26. with contextlib.suppress(Exception):
  27. from hermes_cli.config import load_config
  28. raw = load_config()
  29. maybe = raw.get("compression", {}) if isinstance(raw, dict) else {}
  30. cfg = maybe if isinstance(maybe, dict) else {}
  31. if isinstance(cfg, dict):
  32. # Explicit 0/negative idle disables; a non-positive ceiling is ignored.
  33. with contextlib.suppress(TypeError, ValueError):
  34. if cfg.get("context_timeout_seconds") is not None:
  35. idle = float(cfg["context_timeout_seconds"])
  36. with contextlib.suppress(TypeError, ValueError):
  37. if cfg.get("context_total_ceiling_seconds") is not None and float(cfg["context_total_ceiling_seconds"]) > 0:
  38. ceiling = float(cfg["context_total_ceiling_seconds"])
  39. if idle > 0:
  40. ceiling = max(ceiling, idle)
  41. # #114594: the aux summary attempt legitimately needs up to its own budget (floor 300s), while the
  42. # host idle watchdog defaults to 120s. Clamp the idle window up to at least the effective aux
  43. # compression timeout so prep/chunking work with no streamed tokens yet is not cut off. Only raises:
  44. # an explicit cfg value above the aux budget is kept, and idle never exceeds the ceiling.
  45. from agent.auxiliary_client import _effective_aux_timeout
  46. _aux_budget = float(_effective_aux_timeout("compression", None))
  47. if _aux_budget > 0:
  48. ceiling = max(ceiling, _aux_budget)
  49. idle = max(idle, min(_aux_budget, ceiling))
  50. return idle, ceiling
  51. def effective_ceiling(fence: CompressionCommitFence, ceiling: float, wait_started: float) -> float:
  52. """The total limit for a host that started waiting at *wait_started*: the configured
  53. *ceiling*, or longer once prompt reading pushed the fence deadline back."""
  54. deadline = fence.deadline_monotonic
  55. return ceiling if deadline is None else max(ceiling, deadline - wait_started)
  56. @contextlib.contextmanager
  57. def compression_deadline_scope(fence: Optional[CompressionCommitFence]):
  58. """Share *fence*'s deadline with the summary stream, live, and push it back while the
  59. server reads the prompt. A stream checks the deadline as it reads, so a later summary
  60. call in the same attempt sees the extended deadline too."""
  61. from agent.auxiliary_client import aux_stream_deadline
  62. from agent.local_prompt_progress import prompt_reading_hook
  63. if fence is None:
  64. yield
  65. return
  66. with aux_stream_deadline(lambda: fence.deadline_monotonic), prompt_reading_hook(fence.extend_deadline):
  67. yield