context_compressor_telemetry.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. """Content-free per-attempt compression telemetry for ContextCompressor.
  2. Split from ``agent/context_compressor.py`` (file size cap), which imports it at module level; this module
  3. must never import that module at module level (import cycle).
  4. """
  5. from __future__ import annotations
  6. import uuid
  7. from typing import Any
  8. from agent.model_metadata import estimate_messages_tokens_rough
  9. from agent.model_metadata_token_memo import forget_message_token_memo
  10. def _safe_int(value: Any) -> int | None:
  11. """Best-effort integer coercion for telemetry fields."""
  12. try:
  13. return int(value)
  14. except (TypeError, ValueError):
  15. return None
  16. class CompressionTelemetryMixin:
  17. """Per-attempt telemetry dict: seeded at attempt start, filled by each compression phase."""
  18. def _begin_compression_telemetry(
  19. self, *, current_tokens: int | None, attempt_id: str | None = None, session_id: str | None = None,
  20. trigger_source: str | None = None,
  21. ) -> dict[str, Any]:
  22. """Initialize content-free per-attempt compression telemetry."""
  23. seed = getattr(self, "_compression_telemetry_seed", None)
  24. seed = seed if isinstance(seed, dict) else {}
  25. attempt_id = attempt_id or seed.get("attempt_id")
  26. session_id = session_id or seed.get("session_id")
  27. trigger_source = trigger_source or seed.get("trigger_source")
  28. telemetry: dict[str, Any] = {
  29. "event": "compression_attempt", "attempt_id": attempt_id or uuid.uuid4().hex,
  30. "session_id": session_id or "", "trigger_source": trigger_source or "unknown",
  31. "overflow_reason": seed.get("overflow_reason"),
  32. "main_provider": self.provider or "", "main_model": self.model or "",
  33. "main_context_limit": _safe_int(self.context_length),
  34. "current_estimated_tokens": _safe_int(current_tokens),
  35. "effective_threshold": _safe_int(self.threshold_tokens), "protected_head_tokens": None,
  36. "protected_tail_tokens": None, "middle_window_tokens": None, "prellm_skip_count": 0,
  37. "aux_prompt_tokens": None, "aux_output_reservation": None, "aux_provider": "", "aux_model": "",
  38. "effective_aux_context": None, "fit_margin": None, "chunking": False, "chunk_count": 0,
  39. "total_duration_ms": None, "aux_call_duration_ms": None, "queue_wait_ms": None, "prompt_build_ms": None,
  40. "time_to_first_progress_ms": None, "summary_generation_ms": None, "commit_ms": None,
  41. "fallback_used": False, "commit_status": "unknown", "split_status": "unknown", "failure_class": None,
  42. # Lean-sampling coverage (filled by _record_summary_input_coverage; None on the legacy path).
  43. "summary_input_chars": None, "summary_input_sampled_chars": None, "summary_input_omitted_chars": None,
  44. "summary_input_record_count": None, "summary_input_sampled_record_count": None,
  45. "summary_input_elided_record_count": None,
  46. # Candidate effect is captured by the compressor, then replaced with the exact committed shape.
  47. "method": "none", "items_dropped": 0, "messages_before": None, "messages_after": None,
  48. "tokens_before": None, "tokens_after": None, "tokens_reclaimed": None, "token_count_method": None,
  49. "tool_results_pruned": None, "reasoning_items_pruned": None, "has_focus_topic": False,
  50. # True when the summary came from an opt-in prepared (background) pass (compression.prepare_ahead).
  51. "prepared_summary": False,
  52. }
  53. self._active_compression_telemetry = self._last_compression_telemetry = telemetry
  54. return telemetry
  55. def _record_compression_regions(
  56. self, *, head_messages: list[dict[str, Any]], middle_messages: list[dict[str, Any]],
  57. tail_messages: list[dict[str, Any]],
  58. ) -> None:
  59. telemetry = getattr(self, "_active_compression_telemetry", None)
  60. if isinstance(telemetry, dict):
  61. telemetry["protected_head_tokens"] = estimate_messages_tokens_rough(head_messages)
  62. telemetry["middle_window_tokens"] = estimate_messages_tokens_rough(middle_messages)
  63. telemetry["protected_tail_tokens"] = estimate_messages_tokens_rough(tail_messages)
  64. def _record_aux_compression_call(
  65. self, *, prompt_messages: list[dict[str, Any]], max_tokens: int | None, duration_ms: int,
  66. aux_provider: str | None = None, aux_model: str | None = None,
  67. effective_aux_context: int | None = None, phase_timings: dict[str, Any] | None = None,
  68. ) -> None:
  69. telemetry = getattr(self, "_active_compression_telemetry", None)
  70. if not isinstance(telemetry, dict):
  71. return
  72. telemetry["aux_prompt_tokens"] = estimate_messages_tokens_rough(prompt_messages)
  73. # The summary prompt is built fresh per attempt and never estimated again; don't let the memo pin it.
  74. forget_message_token_memo(prompt_messages)
  75. telemetry["aux_output_reservation"] = _safe_int(max_tokens)
  76. if aux_provider:
  77. telemetry["aux_provider"] = aux_provider
  78. if aux_model:
  79. telemetry["aux_model"] = aux_model
  80. if effective_aux_context is not None:
  81. telemetry["effective_aux_context"] = _safe_int(effective_aux_context)
  82. if telemetry["effective_aux_context"] is not None and telemetry["aux_prompt_tokens"] is not None:
  83. telemetry["fit_margin"] = (telemetry["effective_aux_context"] - telemetry["aux_prompt_tokens"]
  84. - (telemetry["aux_output_reservation"] or 0))
  85. telemetry["aux_call_duration_ms"] = (telemetry.get("aux_call_duration_ms") or 0) + max(0, int(duration_ms))
  86. for key in ("queue_wait_ms", "prompt_build_ms", "time_to_first_progress_ms", "summary_generation_ms", "commit_ms"):
  87. if not isinstance(phase_timings, dict) or key not in phase_timings:
  88. continue
  89. value = _safe_int(phase_timings[key])
  90. # Wait and generation phases accumulate across retries; the rest are point readings.
  91. accumulate = key in {"queue_wait_ms", "summary_generation_ms"} and value is not None
  92. telemetry[key] = (telemetry.get(key) or 0) + value if accumulate else value
  93. def _note_aux_route_fallback(self, route: dict[str, str]) -> None:
  94. """``call_llm`` answered the summary from a fallback route after the configured one failed (dead
  95. endpoint, quota): record it like the compressor's own main-model fallback, so telemetry says
  96. ``fallback_used`` and, when the main model answered, the user is told their compression model is broken."""
  97. failed_model = route.get("failed_model")
  98. if not failed_model:
  99. return
  100. telemetry = getattr(self, "_active_compression_telemetry", None)
  101. if isinstance(telemetry, dict):
  102. telemetry["fallback_used"] = True
  103. telemetry["failure_class"] = telemetry.get("failure_class") or "aux_model_fallback"
  104. # The user warning says "summarised with your main model": only the main-agent route earns it, never a
  105. # fallback_chain entry that happens to share the model name on another provider.
  106. if str(route.get("fallback_label") or "").startswith("main-agent("):
  107. self._last_aux_model_failure_model = failed_model
  108. failure = route.get("failure") or "unknown error"
  109. self._last_aux_model_failure_error = failure if len(failure) <= 220 else failure[:217].rstrip() + "..."
  110. def _compression_method(self) -> str:
  111. """How the summary that replaced the middle window was produced."""
  112. if getattr(self, "_last_summary_fallback_used", False):
  113. return "deterministic_fallback"
  114. if getattr(self, "_last_aux_model_failure_error", None):
  115. return "aux_fallback_main"
  116. return "llm_summary"
  117. def _record_compression_effect(
  118. self, messages_before: int, compressed: list[dict[str, Any]], tokens_before: int, tool_results_pruned: int,
  119. reasoning_items_pruned: int,
  120. ) -> None:
  121. """Record what a finished rewrite did. ``tokens_before`` and the after count are rough message-only
  122. estimates (system prompt and tool schemas excluded), so they compare like for like."""
  123. telemetry = getattr(self, "_active_compression_telemetry", None)
  124. if not isinstance(telemetry, dict):
  125. return
  126. tokens_after = estimate_messages_tokens_rough(compressed)
  127. telemetry.update(
  128. method=self._compression_method(), items_dropped=getattr(self, "_last_summary_dropped_count", 0) or 0,
  129. messages_before=messages_before, messages_after=len(compressed), tokens_before=tokens_before,
  130. tokens_after=tokens_after, tokens_reclaimed=tokens_before - tokens_after,
  131. token_count_method="estimate_rough", tool_results_pruned=tool_results_pruned,
  132. reasoning_items_pruned=reasoning_items_pruned,
  133. )
  134. def _record_committed_compression_effect(
  135. self, messages_before: list[dict[str, Any]], messages_after: list[dict[str, Any]],
  136. ) -> None:
  137. """Replace candidate counts with the exact transcript shape that crossed the commit boundary."""
  138. telemetry = getattr(self, "_active_compression_telemetry", None)
  139. if not isinstance(telemetry, dict):
  140. return
  141. tokens_before = estimate_messages_tokens_rough(messages_before)
  142. tokens_after = estimate_messages_tokens_rough(messages_after)
  143. telemetry.update(
  144. messages_before=len(messages_before), messages_after=len(messages_after),
  145. tokens_before=tokens_before, tokens_after=tokens_after,
  146. tokens_reclaimed=tokens_before - tokens_after, token_count_method="estimate_rough",
  147. )