context_engine.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. """Abstract base class for pluggable context engines.
  2. A context engine decides when/how conversation context is compacted near the token
  3. limit, tracks usage, and may expose tools. ContextCompressor is the default;
  4. ``context.engine`` selects a plugin (``plugins/context_engine/<name>/``); one is active.
  5. Lifecycle: on_session_start() -> per API response update_from_response() -> per turn
  6. should_compress() / compress() -> on_session_end() at real session boundaries only
  7. (CLI exit, /reset, gateway expiry), never per-turn.
  8. """
  9. import copy
  10. import json
  11. from abc import ABC, abstractmethod
  12. from typing import Any, Dict, List, Optional
  13. from agent.compression_marker import elide_middle
  14. from agent.redact import redact_sensitive_text
  15. MEMORY_CONTEXT_MAX_CHARS = 6_000
  16. _MEMORY_CONTEXT_HEAD_CHARS = 4_000
  17. _MEMORY_CONTEXT_TAIL_CHARS = 1_500
  18. def sanitize_memory_context(memory_context: str) -> str:
  19. """Prepare provider context for a context-engine/LLM egress boundary."""
  20. sanitized = redact_sensitive_text(memory_context.strip(), force=True, redact_url_credentials=True)
  21. if len(sanitized) <= MEMORY_CONTEXT_MAX_CHARS:
  22. return sanitized
  23. return elide_middle(sanitized, _MEMORY_CONTEXT_HEAD_CHARS, _MEMORY_CONTEXT_TAIL_CHARS)
  24. def automatic_compaction_status_message(engine: Any, *, phase: str, default_message: str, **context: Any) -> str | None:
  25. """Host-visible status for an automatic compaction event; ``None`` = emit nothing.
  26. Engines suppress via ``emit_automatic_compaction_status = False`` or
  27. customize via ``get_automatic_compaction_status_message(...)``.
  28. """
  29. if not getattr(engine, "emit_automatic_compaction_status", True):
  30. return None
  31. formatter = getattr(engine, "get_automatic_compaction_status_message", None)
  32. message = formatter(phase=phase, default_message=default_message, **context) if callable(formatter) else default_message
  33. if message is None:
  34. return None
  35. return str(message).strip() or None
  36. class ContextEngine(ABC):
  37. """Base class all context engines must implement."""
  38. @property
  39. @abstractmethod
  40. def name(self) -> str:
  41. """Short identifier (e.g. 'compressor', 'lcm')."""
  42. # Token state: engines MUST maintain these; run_agent.py reads them directly.
  43. last_prompt_tokens: int = 0
  44. last_completion_tokens: int = 0
  45. last_total_tokens: int = 0
  46. threshold_tokens: int = 0
  47. context_length: int = 0
  48. compression_count: int = 0
  49. # Compaction parameters (read by run_agent.py for preflight). protect_first_n counts
  50. # non-system head messages kept verbatim IN ADDITION to the always-protected system
  51. # prompt (3 keeps the historical head shape).
  52. # These control the preflight compression check. Subclasses may override via __init__ or property;
  53. # defaults are sensible for most engines. See #13754.
  54. threshold_percent: float = 0.75
  55. protect_first_n: int = 3
  56. protect_last_n: int = 6
  57. # False keeps successful automatic compaction passes silent (routine background
  58. # maintenance); warnings, errors and manual /compress still surface.
  59. emit_automatic_compaction_status: bool = True
  60. @abstractmethod
  61. def update_from_response(self, usage: dict[str, Any]) -> None:
  62. """Update tracked token usage after every LLM call.
  63. ``prompt_tokens``/``completion_tokens``/``total_tokens`` are always present; the
  64. canonical buckets (``input_tokens``, ``output_tokens``, ``cache_read_tokens``,
  65. ``cache_write_tokens``, ``reasoning_tokens``) are optional on older hosts.
  66. """
  67. @abstractmethod
  68. def should_compress(self, prompt_tokens: int | None = None) -> bool:
  69. """Return True if compaction should fire this turn."""
  70. def should_compress_info(self, prompt_tokens: int | None = None) -> tuple[bool, str | None]:
  71. """Return ``(should_compress, reason)``.
  72. Engines with block reasons (summary-LLM cooldown, anti-thrashing guard) override
  73. this so callers can warn instead of silently skipping; the default keeps plugin
  74. engines from raising AttributeError.
  75. """
  76. return self.should_compress(prompt_tokens), None
  77. @abstractmethod
  78. def compress(
  79. self, messages: list[dict[str, Any]], current_tokens: Optional[int] = None,
  80. focus_topic: Optional[str] = None, force: bool = False, memory_context: str = "",
  81. ) -> list[dict[str, Any]]:
  82. """Compact ``messages`` into a valid OpenAI-format list that fits the budget.
  83. ``focus_topic`` comes from manual ``/compress <focus>`` (prioritise that topic);
  84. ``force`` asks to bypass an engine-owned cooldown; ``memory_context`` is provider
  85. text for the handoff prompt. Older engines may omit optional parameters — the
  86. host filters them by signature.
  87. """
  88. def prune_tool_results_only(
  89. self, messages: list[dict[str, Any]], current_tokens: int | None = None,
  90. ) -> tuple[list[dict[str, Any]], int]:
  91. """Deterministically trim old tool-result payloads without an LLM call.
  92. Runs on a low, cost-oriented trigger independent of ``should_compress`` so
  93. large-window engines reclaim re-sent tool output long before full compaction.
  94. Returns ``(messages, n_pruned)``; the default no-op keeps older engines safe.
  95. """
  96. return messages, 0
  97. def select_context(
  98. self, request_messages: list[dict[str, Any]], *, conversation_messages: list[dict[str, Any]] | None = None,
  99. incoming_message: dict[str, Any] | None = None, budget_tokens: int = 0,
  100. ) -> list[dict[str, Any]]:
  101. """Optionally *select* (replace) the context for THIS request, pre-generation.
  102. Runs on every provider request (also retries), independent of
  103. ``should_compress()``: ``compress()`` shrinks over-long context, this swaps in a
  104. different one (retrieval, topic routing, branch switching). Return ``None`` to
  105. leave the request unchanged. The returned list is request-only — it MUST NOT be
  106. treated as persisted transcript state (session DB history is untouched); unlike
  107. ``pre_llm_call`` it may replace the list. The host runs it before prompt
  108. cache-control and every request sanitizer, so a malformed replacement never
  109. reaches the provider and the default no-op keeps the request byte-identical;
  110. an engine that replaces the list changes its own cache prefix (breakpoints are
  111. re-derived on the selected list). ``request_messages`` is the assembled request
  112. (system prompt + history + ephemeral prefill); ``conversation_messages`` is the
  113. persisted history for reference only (do not mutate); ``budget_tokens`` is the
  114. model's context length or 0 if unknown.
  115. """
  116. return None
  117. def on_turn_complete(self, messages: list[dict[str, Any]], usage: dict[str, Any] | None = None, **kwargs: Any) -> None:
  118. """Observe a finished turn (complement of ``select_context()``) to index/update
  119. routing state for the next request.
  120. Best-effort, not guaranteed: fires from the normal finalization seam only; some
  121. abnormal early returns (content-policy block, provider terminal failure) skip it.
  122. ``messages`` is a read-only shallow copy (return value ignored; never rely on
  123. transcript mutation). ``usage`` has the ``update_from_response`` shape and is
  124. ``None`` when no provider response was reached (interrupt). ``kwargs`` may include
  125. ``turn_id``, ``task_id``, ``api_call_count``, ``interrupted``, ``failed``, ``turn_exit_reason``.
  126. """
  127. return
  128. def should_compress_preflight(self, messages: list[dict[str, Any]]) -> bool:
  129. """Cheap rough check before the API call (no real token count yet); default skips."""
  130. return False
  131. def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool:
  132. """True when preflight should trust recent real usage over the noisy rough
  133. estimate (avoids re-compacting after a compressed request already fit)."""
  134. return False
  135. def get_automatic_compaction_status_message(
  136. self, *, phase: str, default_message: str, **context: Any,
  137. ) -> str | None:
  138. """User-visible status for automatic compaction, or ``None`` to suppress it.
  139. ``phase`` is the host call site (``"preflight"`` / ``"compress"``); ``context``
  140. carries best-effort ``approx_tokens`` / ``threshold_tokens``. Warnings, errors
  141. and manual ``/compress`` are not governed by this hook.
  142. """
  143. return default_message if self.emit_automatic_compaction_status else None
  144. def has_content_to_compress(self, messages: list[dict[str, Any]]) -> bool:
  145. """Preflight guard for gateway ``/compress``: False reports "nothing to
  146. compress yet" without an LLM call (e.g. transcript entirely protected)."""
  147. return True
  148. def on_session_start(self, session_id: str, **kwargs) -> None:
  149. """Session begins: load persisted state. kwargs may include hermes_home, platform, model."""
  150. def on_session_end(self, session_id: str, messages: list[dict[str, Any]]) -> None:
  151. """Real session boundary (CLI exit, /reset, gateway expiry) — never per-turn."""
  152. def on_session_reset(self) -> None:
  153. """/new or /reset: reset per-session state (default: counters and token tracking)."""
  154. # Reset cross-call calibration state captured under the PREVIOUS model. These fields encode "the
  155. # provider proved this prompt fit" / "preflight can be deferred" decisions that are only valid for
  156. # the model that produced them. Carrying them across a switch to a smaller-context model would let
  157. # should_defer_preflight_to_real_usage() suppress a preflight compression the new model actually
  158. # needs — the exact oversized-send-after-switch failure in #23767. The new model's first response
  159. # repopulates them via update_from_response(). Setting last_prompt_tokens to 0 (NOT -1) is
  160. # deliberate: 0 is the documented "no real usage yet -> use the rough estimate" state, so the post-
  161. # response should_compress path falls back to estimate_request_tokens_rough rather than skipping
  162. # compression. -1 is a different sentinel (#36718, "compression just ran, await real usage") and
  163. # must not be set here.
  164. self.last_prompt_tokens = 0
  165. self.last_completion_tokens = 0
  166. self.last_total_tokens = 0
  167. self.compression_count = 0
  168. def get_tool_schemas(self) -> list[dict[str, Any]]:
  169. """Tool schemas this engine exposes to the agent (default: none)."""
  170. return []
  171. def handle_tool_call(self, name: str, args: dict[str, Any], **kwargs) -> str:
  172. """Handle a call to one of this engine's tools; must return a JSON string.
  173. kwargs may include ``messages`` (live in-memory list)."""
  174. return json.dumps({"error": f"Unknown context engine tool: {name}"})
  175. def get_status(self) -> dict[str, Any]:
  176. """Status dict with the standard fields run_agent.py expects."""
  177. # Clamp the -1 "compression just ran, awaiting real usage" sentinel to 0 so no
  178. # reader sees a negative usage_percent on the transitional turn.
  179. last_prompt = max(self.last_prompt_tokens, 0)
  180. return {
  181. "last_prompt_tokens": last_prompt,
  182. "threshold_tokens": self.threshold_tokens,
  183. "context_length": self.context_length,
  184. "usage_percent": min(100, last_prompt / self.context_length * 100) if self.context_length else 0,
  185. "compression_count": self.compression_count,
  186. }
  187. def clone_for_agent(self) -> "ContextEngine":
  188. """Per-agent instance of a plugin-registered engine (the plugin system holds ONE shared
  189. instance; every AIAgent gets its own so a child's update_model() cannot mutate the parent's).
  190. Override when the engine holds uncopyable state (locks, DB connections): return a fresh
  191. engine sharing the durable backend and copying only mutable budget state."""
  192. return copy.deepcopy(self)
  193. def update_model(
  194. self, model: str, context_length: int, base_url: str = "", api_key: str = "",
  195. provider: str = "", api_mode: str = "",
  196. ) -> None:
  197. """Model switch / fallback: recompute threshold_tokens (override for more).
  198. Per-model threshold override (longest substring match), else the raw config
  199. percent — snapshotted ONCE so repeated switches fall back to the configured
  200. value, not the previous model's override.
  201. """
  202. self.context_length = context_length
  203. from agent.context_compressor import resolve_model_threshold
  204. if not hasattr(self, "_config_threshold_percent"):
  205. self._config_threshold_percent = self.threshold_percent
  206. self._base_threshold_percent = resolve_model_threshold(
  207. model, getattr(self, "model_thresholds", {}), self._config_threshold_percent, provider)
  208. self.threshold_percent = self._base_threshold_percent
  209. self.threshold_tokens = int(context_length * self.threshold_percent)