context_compressor_prellm.py 4.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. """Pre-LLM summary skips for ContextCompressor: compact deterministically without calling a summary model."""
  2. from __future__ import annotations
  3. import logging
  4. import time
  5. from typing import Any, Dict, List
  6. from agent.model_metadata import estimate_messages_tokens_rough
  7. logger = logging.getLogger(__name__)
  8. # Skip the LLM call when the compressible middle is below this fraction of the
  9. # threshold (and a prior ineffectiveness strike exists); dropping alone suffices.
  10. # See #60451.
  11. _FEASIBILITY_SKIP_MIDDLE_FRACTION = 0.10
  12. _FALLBACK_PROBE_AT_MODEL_CONFIG_KEY = "_fallback_probe_at"
  13. class PreLlmSkipMixin:
  14. """Decides, before the summary call, whether compress() should skip the summary model."""
  15. def _set_fallback_probe_at(self, at: float) -> None:
  16. """Set the summary-model probe deadline, persisting on change only (0 = unarmed) so a rebuilt
  17. compressor resumes the bench window instead of restarting it (same contract as #100185)."""
  18. if at == self._fallback_probe_at:
  19. return
  20. self._fallback_probe_at = at
  21. self._durable_write(
  22. "patch_session_model_config", "summary-model probe deadline", {_FALLBACK_PROBE_AT_MODEL_CONFIG_KEY: at or None},
  23. )
  24. def _load_fallback_probe_at(self) -> None:
  25. self._load_durable(
  26. "_fallback_probe_at", "get_session_model_config_value", "summary-model probe deadline",
  27. float, 0.0, _FALLBACK_PROBE_AT_MODEL_CONFIG_KEY, 0.0,
  28. )
  29. def _fallback_streak_skip(self, telemetry: dict[str, Any]) -> bool:
  30. """Pre-LLM skip while two summaries in a row fell back: compact deterministically instead of
  31. paying for a summary model that keeps failing (#63008). One probe per recovery window; a
  32. healthy summary resets the streak, another fallback benches it again."""
  33. if self._fallback_compression_streak < 2:
  34. self._set_fallback_probe_at(0.0)
  35. return False
  36. # Wall clock: the deadline is persisted on the session row so a gateway rebuild resumes the window.
  37. now = time.time()
  38. if self._fallback_probe_at and now >= self._fallback_probe_at:
  39. self._set_fallback_probe_at(0.0)
  40. if not self.quiet_mode:
  41. logger.info(
  42. "Compression: probing the summary model again after %d fallback summaries in a row",
  43. self._fallback_compression_streak,
  44. )
  45. return False
  46. if not self._fallback_probe_at or self._fallback_probe_at - now > self._ANTI_THRASH_RECOVERY_SECONDS:
  47. self._set_fallback_probe_at(now + self._ANTI_THRASH_RECOVERY_SECONDS)
  48. self._last_feasibility_skip = True
  49. telemetry["failure_class"] = "summary_model_benched"
  50. if not self.quiet_mode:
  51. logger.warning(
  52. "Compression: %d fallback summaries in a row — skipping LLM summarization, proceeding with "
  53. "deterministic message dropping. Next summary-model probe in %.0fs.",
  54. self._fallback_compression_streak, max(0.0, self._fallback_probe_at - now),
  55. )
  56. return True
  57. def _feasibility_skip(
  58. self, telemetry: dict[str, Any], turns_to_summarize: list[dict[str, Any]],
  59. compress_start: int, compress_end: int,
  60. ) -> bool:
  61. """Pre-LLM skip after a real-usage ineffectiveness strike (reads the counter, never writes)."""
  62. if self._ineffective_compression_count < 1:
  63. return False
  64. # Reuse the telemetry estimate so log and telemetry agree; None means the regions helper
  65. # no-op'd (0 is valid).
  66. middle_tokens = telemetry.get("middle_window_tokens")
  67. middle_tokens = estimate_messages_tokens_rough(turns_to_summarize) if middle_tokens is None else middle_tokens
  68. if middle_tokens >= int(self.threshold_tokens * _FEASIBILITY_SKIP_MIDDLE_FRACTION):
  69. return False
  70. self._last_feasibility_skip = True
  71. self._prellm_skip_count += 1
  72. telemetry["prellm_skip_count"] = self._prellm_skip_count
  73. if not self.quiet_mode:
  74. logger.warning(
  75. "Compression: middle section (%d tokens at indices %d-%d) is below %.0f%% of threshold (%d tokens) — "
  76. "skipping LLM summarization, proceeding with deterministic message dropping. prellm_skip_count=%d",
  77. middle_tokens, compress_start, compress_end,
  78. _FEASIBILITY_SKIP_MIDDLE_FRACTION * 100,
  79. self.threshold_tokens, self._prellm_skip_count,
  80. )
  81. return True