turn_preflight.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. """Pre-API preflight compression gate and post-tool compression for the turn loop.
  2. Runs once per API call after the request pressure is measured: grow a managed llama.cpp
  3. window (last resort), or compress when over threshold (deferring on noisy estimates, in
  4. failure cooldown, or while the review fork's first request is pending), handle the
  5. provider-proven overflow re-check fail-closed, and emit the deduped blocked/uncompressed
  6. overflow warnings. Nothing here imports ``agent.conversation_loop`` at module level
  7. (cycle); loop-internal helpers resolve lazily."""
  8. from __future__ import annotations
  9. import logging
  10. from dataclasses import dataclass
  11. from typing import Any, Dict, List, Optional
  12. from agent.context_engine import automatic_compaction_status_message
  13. from agent.conversation_compression import (
  14. PRE_API_COMPRESSION_STATUS_TEMPLATE, _reset_read_dedup_caches, compression_blocked_transiently,
  15. compression_skipped_due_to_lock, context_compression_timed_out, conversation_history_after_compression,
  16. )
  17. from agent.conversation_compression_feasibility import ensure_compression_feasibility_checked
  18. from agent.turn_context import _review_fork_first_request_pending
  19. from agent.turn_context_compaction import (
  20. _apply_grown_window, _blocked_compress_reason, _clear_overflow_warn, _refund_api_call,
  21. _reanchor, _reset_retry_state_after_compaction,
  22. )
  23. logger = logging.getLogger("agent.conversation_loop")
  24. @dataclass
  25. class PreflightGateVerdict:
  26. """``action``: ``"fallthrough"`` (make the API call), ``"continue"`` (window grown or
  27. history compacted — the call/budget was refunded, re-enter the turn loop and
  28. re-measure), ``"break"`` (turn ends: compression timeout or non-actionable compaction
  29. handoff — ``final_response``/``failed``/``_turn_exit_reason`` set) or ``"return"``
  30. (``result`` is the turn's result dict). The other fields are the loop locals the gate
  31. rebinds; ``_last_preflight_pressure`` is re-armed only by a compression pass."""
  32. action: str
  33. pending_moa_prepared_request: Any
  34. messages: Any
  35. active_system_prompt: Any
  36. conversation_history: Any
  37. api_call_count: Any
  38. compression_attempts: Any
  39. final_response: Any
  40. failed: Any
  41. _turn_exit_reason: Any
  42. _compression_timeout_exhausted: Any
  43. _preflight_compression_blocked: Any
  44. _provider_overflow_recovery_pending: Any
  45. _last_preflight_pressure: Any
  46. result: Optional[dict[str, Any]] = None
  47. def run_preflight_compression(
  48. agent: Any, v: PreflightGateVerdict, *, compressor: Any, request_pressure_tokens: int,
  49. provider_overflow_preflight: bool, defer_preflight: Any, moa_prepared_request: Any,
  50. system_message: Any, user_message: Any, max_compression_attempts: int, effective_task_id: Any,
  51. ) -> PreflightGateVerdict:
  52. """Mirror of the turn-prologue guard chain (defer on noisy estimate → skip in failure
  53. cooldown → ``should_compress``), rebinding the loop locals on ``v`` and setting
  54. ``v.action``. A compression pass that never reaches the provider refunds the
  55. call/budget in every branch (skip, re-run, timeout) so ``api_call_count`` never
  56. over-reports; a lock/transient skip refunds the attempt and leaves the progress
  57. blocker unarmed. A forced provider-overflow preflight that any gate blocks fails
  58. closed (llama.cpp may silently truncate)."""
  59. from agent.conversation_loop import (
  60. _COMPRESSION_TIMEOUT_FINAL_RESPONSE, _HANDOFF_SKIP_FINAL_RESPONSE,
  61. _compression_deferred_result, _maybe_grow_local_window, _provider_overflow_exhausted_result,
  62. _should_skip_model_call_for_reference_handoff,
  63. )
  64. def _done(action: str, result: Optional[dict[str, Any]] = None) -> PreflightGateVerdict:
  65. v.action, v.result = action, result
  66. return v
  67. def _exhausted_result() -> dict[str, Any]:
  68. return _provider_overflow_exhausted_result(
  69. agent, v.messages, v.conversation_history, v.api_call_count, request_pressure_tokens,
  70. max_compression_attempts,
  71. )
  72. _eligible = (
  73. agent.compression_enabled
  74. and len(v.messages) > 1
  75. and v.compression_attempts < max_compression_attempts
  76. )
  77. if _eligible:
  78. # Aux clamp must land before the first compaction fires on the main-window threshold (#114707).
  79. # Before the cooldown read: a moved compression route clears the previous summariser's cooldown.
  80. ensure_compression_feasibility_checked(agent, request_pressure_tokens)
  81. _compression_cooldown = getattr(
  82. compressor, "get_active_compression_failure_cooldown", lambda: None
  83. )()
  84. if (
  85. _eligible
  86. and not _review_fork_first_request_pending(agent)
  87. and (not v._preflight_compression_blocked or provider_overflow_preflight)
  88. and (not defer_preflight(request_pressure_tokens) or provider_overflow_preflight)
  89. and not _compression_cooldown
  90. and compressor.should_compress(request_pressure_tokens)
  91. ):
  92. # Managed local runtime: grow the context window before compressing (last
  93. # resort). Only for a llamacpp provider at the supervised base_url.
  94. _grown_window = _maybe_grow_local_window(agent, compressor, request_pressure_tokens)
  95. if _grown_window:
  96. # Bigger window granted: recalibrate and skip compression this pass. Never
  97. # reached the provider — refund like the compression path does.
  98. _apply_grown_window(agent, compressor, _grown_window)
  99. v.api_call_count = _refund_api_call(agent, v.api_call_count)
  100. return _done("continue")
  101. if moa_prepared_request is not None:
  102. v.pending_moa_prepared_request = moa_prepared_request
  103. v.compression_attempts += 1
  104. # Compression is running: reset the blocked-overflow warning dedup so a
  105. # later blocked turn warns again.
  106. _clear_overflow_warn(agent)
  107. _threshold = int(getattr(compressor, "threshold_tokens", 0) or 0)
  108. _context_length = int(getattr(compressor, "context_length", 0) or 0)
  109. logger.info(
  110. "Pre-API compression: ~%s request tokens >= %s threshold "
  111. "(context=%s, attempt=%s/%s)",
  112. f"{request_pressure_tokens:,}",
  113. f"{_threshold:,}",
  114. f"{_context_length:,}" if getattr(compressor, "context_length", 0) else "unknown",
  115. v.compression_attempts,
  116. max_compression_attempts,
  117. )
  118. _pre_api_status = automatic_compaction_status_message(
  119. compressor,
  120. phase="pre_api",
  121. default_message=PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
  122. tokens=request_pressure_tokens
  123. ),
  124. approx_tokens=request_pressure_tokens,
  125. threshold_tokens=_threshold,
  126. context_length=_context_length,
  127. model=agent.model,
  128. attempt=v.compression_attempts,
  129. max_attempts=max_compression_attempts,
  130. )
  131. if _pre_api_status:
  132. agent._emit_status(_pre_api_status)
  133. v._last_preflight_pressure = request_pressure_tokens
  134. _pre_api_input = v.messages
  135. v.messages, v.active_system_prompt = agent._compress_context(
  136. v.messages, system_message, approx_tokens=request_pressure_tokens,
  137. task_id=effective_task_id, trigger="pre_api",
  138. )
  139. if context_compression_timed_out(agent):
  140. # Progress-aware timeout: never reached the provider — refund the
  141. # call/budget and stop; an overflow retry would only re-compress.
  142. v.api_call_count = _refund_api_call(agent, v.api_call_count)
  143. v.final_response = _COMPRESSION_TIMEOUT_FINAL_RESPONSE
  144. v.failed = True
  145. v._compression_timeout_exhausted = True
  146. v._turn_exit_reason = "context_compression_timeout"
  147. return _done("break")
  148. if v.messages is _pre_api_input and (
  149. compression_skipped_due_to_lock(agent) or compression_blocked_transiently(agent)
  150. ):
  151. # Temporary DEFER (lock held / cooldown), not evidence about compressibility:
  152. # refund the attempt, leave the progress blocker unarmed and proceed.
  153. v.compression_attempts -= 1
  154. v._last_preflight_pressure = None
  155. if v.pending_moa_prepared_request is moa_prepared_request:
  156. v.pending_moa_prepared_request = None
  157. else:
  158. _reset_retry_state_after_compaction(agent)
  159. # Re-baseline the flush cursor: rotation returns None (child flushes
  160. # whole); in-place returns list(messages) — None would re-append
  161. # persisted rows. See conversation_history_after_compression().
  162. v.conversation_history = conversation_history_after_compression(
  163. agent, v.messages, v.conversation_history
  164. )
  165. # Never reaches the provider on skip or re-run — refund the call/budget
  166. # in BOTH cases, else budget leaks and api_call_count over-reports.
  167. v.api_call_count = _refund_api_call(agent, v.api_call_count)
  168. if _should_skip_model_call_for_reference_handoff(v.messages, user_message):
  169. # Reference-only handoff must not become the active turn after a
  170. # completed assistant response.
  171. logger.info(
  172. "Skipping post-compaction model call: reference-only "
  173. "handoff would be the sole active user turn (#80622)"
  174. )
  175. if not v.final_response:
  176. v.final_response = _HANDOFF_SKIP_FINAL_RESPONSE
  177. v._turn_exit_reason = "compaction_handoff_not_actionable"
  178. return _done("break")
  179. return _done("continue")
  180. elif provider_overflow_preflight and _compression_cooldown:
  181. # Provider proved the request cannot fit and the compressor is unavailable:
  182. # don't resend; let the next user turn retry after cooldown.
  183. agent._persist_session(v.messages, v.conversation_history)
  184. return _done("return", _compression_deferred_result(
  185. agent, v.messages, v.api_call_count, reason="transient_block"
  186. ))
  187. elif provider_overflow_preflight and v.compression_attempts >= max_compression_attempts:
  188. # All recovery passes consumed and still over threshold: fail closed —
  189. # llama.cpp may silently truncate an oversized retry.
  190. return _done("return", _exhausted_result())
  191. elif _eligible and not defer_preflight(request_pressure_tokens) and _compression_cooldown:
  192. # Summary-LLM cooldown blocks compression: deduped warning only when over
  193. # threshold (should_compress_info reason is None below it).
  194. _block_reason = _blocked_compress_reason(compressor, request_pressure_tokens)
  195. if _block_reason:
  196. agent._warn_context_overflow_blocked(
  197. _block_reason, request_pressure_tokens,
  198. int(getattr(compressor, "threshold_tokens", 0) or 0),
  199. )
  200. elif not agent.compression_enabled and len(v.messages) > 1:
  201. # Uncompressed session guard: compression is disabled, so warn (deduped) when
  202. # the request exceeds the context window; the turn-context preflight re-arms.
  203. _ctx_len = getattr(getattr(agent, "context_compressor", None), "context_length", None)
  204. if isinstance(_ctx_len, int) and _ctx_len > 0 and request_pressure_tokens > _ctx_len:
  205. _warn_fn = getattr(agent, "_warn_uncompressed_context_overflow", None)
  206. if callable(_warn_fn):
  207. _warn_fn(request_pressure_tokens, _ctx_len)
  208. if provider_overflow_preflight:
  209. # Any other gate blocking the forced preflight (e.g. uncompressible one-
  210. # message request) must fail closed: the request is proven not to fit.
  211. return _done("return", _exhausted_result())
  212. return _done("fallthrough")
  213. @dataclass
  214. class PostToolCompressionVerdict:
  215. """``end_turn`` True → a reference-only compaction handoff would be the sole active
  216. user turn: stop without another model call (``final_response`` /
  217. ``turn_exit_reason`` set)."""
  218. end_turn: bool
  219. messages: list[dict[str, Any]]
  220. active_system_prompt: Any
  221. conversation_history: Any
  222. compression_attempts: int
  223. final_response: Any
  224. turn_exit_reason: Any
  225. current_turn_user_idx: int
  226. def compress_after_tool_results(
  227. agent: Any, *, messages: list[dict[str, Any]], system_message: Any, user_message: Any,
  228. active_system_prompt: Any, conversation_history: Any, compression_attempts: int,
  229. max_compression_attempts: int, effective_task_id: Any, final_response: Any,
  230. turn_exit_reason: Any, current_turn_user_idx: int,
  231. ) -> PostToolCompressionVerdict:
  232. """Post-tool-call compression decision. Pressure comes from API-reported
  233. ``prompt_tokens`` (a tight lower bound; thinking models inflate completion tokens),
  234. ``0`` right after compression (no real count yet), else the route-aware
  235. overhead-inclusive estimate. Over threshold but blocked → deduped warning plus the
  236. deterministic tool-result-only prune, committed only when the engine returns a NEW
  237. list (never rebuild ``conversation_history`` for it)."""
  238. from agent.conversation_loop import (
  239. _HANDOFF_SKIP_FINAL_RESPONSE, _midturn_request_pressure_tokens,
  240. _should_skip_model_call_for_reference_handoff,
  241. )
  242. from agent.model_metadata import estimate_request_tokens_rough
  243. def _verdict(end_turn: bool) -> PostToolCompressionVerdict:
  244. return PostToolCompressionVerdict(
  245. end_turn=end_turn, messages=messages, active_system_prompt=active_system_prompt,
  246. conversation_history=conversation_history, compression_attempts=compression_attempts,
  247. final_response=final_response, turn_exit_reason=turn_exit_reason,
  248. current_turn_user_idx=current_turn_user_idx,
  249. )
  250. _compressor = agent.context_compressor
  251. # A new checkpoint must reach the provider before stale usage can trigger
  252. # local compression, overflow warnings, or destructive tool-result pruning.
  253. if bool(getattr(_compressor, "awaiting_real_usage_after_compression", False)):
  254. return _verdict(False)
  255. # Real usage decides: the anchor is the provider's last prompt count plus a rough delta for
  256. # ONLY the tool results appended since (the raw last_prompt_tokens ignores them). Right after
  257. # a compaction (-1 sentinel) there is no real count yet: never treat the schema-heavy rough
  258. # figure as pressure. The whole-request rough estimate is the last resort (usage-less
  259. # provider, post-disconnect, gateway restart), kept route-aware (#96995/#97602).
  260. from agent.usage_anchor import anchored_context_tokens
  261. _anchored = anchored_context_tokens(messages, getattr(agent, "_usage_anchor", None))
  262. if _anchored is not None:
  263. _real_tokens = _anchored
  264. elif _compressor.last_prompt_tokens > 0:
  265. # Only prompt_tokens: thinking models inflate completion_tokens with
  266. # reasoning that uses no context → premature compression. (#12026)
  267. _real_tokens = _compressor.last_prompt_tokens
  268. elif _compressor.last_prompt_tokens == -1:
  269. _real_tokens = 0
  270. else:
  271. _real_tokens = _midturn_request_pressure_tokens(
  272. agent, messages, active_system_prompt or "",
  273. estimate_request_tokens_rough(messages, tools=agent.tools or None),
  274. )
  275. if agent.compression_enabled and compression_attempts < max_compression_attempts:
  276. ensure_compression_feasibility_checked(agent, _real_tokens)
  277. if (
  278. agent.compression_enabled
  279. and compression_attempts < max_compression_attempts
  280. and not bool(
  281. getattr(_compressor, "awaiting_real_usage_after_compression", False)
  282. )
  283. and _compressor.should_compress(_real_tokens)
  284. ):
  285. compression_attempts += 1
  286. # Compression is running: reset blocked-overflow warning dedup so a
  287. # future blocked turn can warn again.
  288. _clear_overflow_warn(agent)
  289. agent._safe_print(" ⟳ compacting context…")
  290. _post_tool_input = messages
  291. # Pass overhead-aware _real_tokens, not last_prompt_tokens (0 in the
  292. # no-usage fallback), so the overflow guard sees the true size.
  293. messages, active_system_prompt = agent._compress_context(
  294. messages, system_message, approx_tokens=_real_tokens, task_id=effective_task_id, trigger="post_tool",
  295. )
  296. if messages is _post_tool_input and compression_skipped_due_to_lock(agent):
  297. # Lock-skip no-op is a temporary defer, not evidence about compressibility:
  298. # refund so a lock-loser loop doesn't burn the budget toward exhausted.
  299. # #69870 lock-skip / #97488 transient-block: this pass no-oped for a TEMPORARY reason (another
  300. # path holds the compression lock, or a timed cooldown/backoff guard is active). That is a
  301. # temporary DEFER, not evidence about compressibility — refund the attempt (it must not burn the
  302. # shared overflow-recovery budget toward compression_exhausted → gateway auto-reset,
  303. # #9893/#35809) and leave the insufficient-progress blocker unarmed. Proceed with the current
  304. # request: if it truly does not fit, the provider's 413/overflow handler returns the soft
  305. # compression_deferred result with that stronger signal.
  306. # #69870 lock-skip: the provider proved the request does not fit, but this compression pass
  307. # no-oped only because another path holds the session's compression lock. Temporary defer, not
  308. # exhaustion — refund the attempt and end the turn softly so the gateway does NOT auto-reset the
  309. # session (#9893/#35809).
  310. # #97488 transient-block: compression no-oped because a timed guard (host-timeout cooldown /
  311. # structural backoff) is active — a temporary defer, not evidence of incompressibility. Never
  312. # classify it as compression_exhausted (gateway auto-reset).
  313. # bypass_cooldown=True, # #100661 provider-proven overflow
  314. # #97488: timed transient guard — defer, never exhaustion (gateway auto-reset).
  315. # #69870 lock-skip: the provider proved the request does not fit, but this compression pass
  316. # no-oped only because another path holds the session's compression lock. Temporary defer, not
  317. # exhaustion — refund the attempt and end the turn softly so the gateway does NOT auto-reset the
  318. # session (#9893/#35809).
  319. # #97488 transient-block: a timed guard (host-timeout cooldown / structural backoff) no-oped
  320. # this pass — defer softly, never compression_exhausted (which would auto-reset the session).
  321. # #69870 lock-skip: this pass no-oped because another path holds the session's compression lock
  322. # — a temporary defer, not evidence about compressibility.
  323. compression_attempts -= 1
  324. else:
  325. conversation_history = conversation_history_after_compression(
  326. agent, messages, conversation_history
  327. )
  328. if _should_skip_model_call_for_reference_handoff(messages, user_message):
  329. logger.info(
  330. "Skipping post-tool compaction model call: "
  331. "reference-only handoff would be the sole "
  332. "active user turn (#80622)"
  333. )
  334. if not final_response:
  335. final_response = _HANDOFF_SKIP_FINAL_RESPONSE
  336. turn_exit_reason = "compaction_handoff_not_actionable"
  337. return _verdict(True)
  338. current_turn_user_idx = _reanchor(agent, messages, user_message)
  339. elif agent.compression_enabled:
  340. # Over threshold but compression blocked (cooldown/anti-thrash): deduped
  341. # warning so context can't silently overflow. ``attempts_spent`` names the
  342. # attempts_exhausted lockout when the engine says RUN but the per-turn
  343. # budget is spent (#101889).
  344. _block_reason = _blocked_compress_reason(
  345. _compressor, _real_tokens, attempts_spent=compression_attempts
  346. )
  347. if _block_reason:
  348. agent._warn_context_overflow_blocked(
  349. _block_reason, _real_tokens, int(getattr(_compressor, "threshold_tokens", 0) or 0)
  350. )
  351. # Proactive tool-result prune (deterministic, no LLM, keeps tail): no-op unless
  352. # proactive_prune_tokens is exceeded; commits only past
  353. # proactive_prune_min_reclaim_tokens so cache breaks stay episodic.
  354. _prune = getattr(_compressor, "prune_tool_results_only", None)
  355. if callable(_prune):
  356. try:
  357. _pruned_msgs, _pruned_n = _prune(messages, current_tokens=_real_tokens)
  358. except Exception:
  359. logger.debug("proactive tool-result prune failed; skipping", exc_info=True)
  360. _pruned_msgs, _pruned_n = messages, 0
  361. # Standard no-op caller contract: only commit when the engine returned a
  362. # NEW list object with a non-zero count. Do NOT rebuild
  363. # conversation_history: rows already carry _DB_PERSISTED_MARKER, and on a
  364. # stale in-place flag the helper could seed unpersisted rows.
  365. if _pruned_n and _pruned_msgs is not messages:
  366. from agent.compaction_events import publish_prune
  367. publish_prune(agent, messages, _pruned_msgs, _pruned_n)
  368. messages = _pruned_msgs
  369. # A committed prune is a content-loss boundary like compaction: demoted skill_view /
  370. # read_file bodies survive only as one-line markers, so the repeat-read dedup must
  371. # stop answering "unchanged" for them or the reload the marker asks for is refused.
  372. _reset_read_dedup_caches(effective_task_id, session_id=agent.session_id or "")
  373. return _verdict(False)