compression_facade.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Host-side ``AIAgent._compress_context`` wrapper.
  2. Publishes the commit fence ``hard_interrupt()`` reads, runs the compressor on a snapshot under the progress
  3. timeout, mirrors ``_DB_PERSISTED_MARKER`` stamps back onto the live lists and rebinds the session context.
  4. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
  5. """
  6. import contextlib
  7. import copy
  8. import functools
  9. import logging
  10. import threading
  11. from agent.session_activity import ActivityProvenance
  12. # Same logger name as the origin module so log records / caplog filters are unchanged.
  13. logger = logging.getLogger("run_agent")
  14. class _CommitFenceRegistration:
  15. """One ``_compress_context`` attempt's entry on the agent's commit-fence stack.
  16. The fence slot is a stack, not a save/restore cell: overlapping attempts register in
  17. order but may complete in any order, so each attempt removes only its own entry and the
  18. published slot always tracks the newest *live* attempt. The stall-fallback retry swaps
  19. ``fence`` in place so the slot update stays tied to the owning registration.
  20. """
  21. __slots__ = ("fence",)
  22. def __init__(self, fence) -> None:
  23. self.fence = fence
  24. def _timeout_fallback_prompt(agent, system_message: str) -> str:
  25. """Cached prompt, else a fresh build, else the raw ``system_message`` (never raises).
  26. Resolved lazily by the timeout wrapper: an eager rebuild would raise before compress_context runs when
  27. ``_cached_system_prompt`` is unset and the builder fails."""
  28. if cached := getattr(agent, "_cached_system_prompt", None):
  29. return cached
  30. try:
  31. return agent._build_system_prompt(system_message)
  32. except Exception:
  33. logger.debug("compress_context timeout fallback prompt rebuild failed; using raw system_message", exc_info=True)
  34. return system_message or ""
  35. def _report_compression_timeout(
  36. agent, *, idle: float, waited: float, since_progress: float, total_ceiling: float, total_exhausted: bool,
  37. progress_observed: bool,
  38. ) -> None:
  39. """Host-side timeout bookkeeping: log, activity stamp, cooldown ladder, user warning."""
  40. from agent.conversation_compression import mark_context_compression_timed_out
  41. mark_context_compression_timed_out(agent)
  42. if total_exhausted:
  43. logger.warning(
  44. "Context compression reached its total ceiling after %.1fs (progress observed=%s); continuing without compression",
  45. waited, progress_observed,
  46. )
  47. else:
  48. logger.warning(
  49. "Context compression made no progress for %.1fs (total wait %.1fs, ceiling %.1fs); continuing without compression",
  50. since_progress, waited, total_ceiling,
  51. )
  52. touch = getattr(agent, "_touch_activity", None)
  53. if callable(touch):
  54. try:
  55. touch("context compression timed out", provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT)
  56. except Exception:
  57. logger.debug("compress_context timeout activity touch failed", exc_info=True)
  58. # Same timeout cooldown ladder as summary-LLM timeouts: avoid re-burning the full idle budget every turn.
  59. record = getattr(getattr(agent, "context_compressor", None), "record_timeout_failure", None)
  60. if callable(record):
  61. try:
  62. if total_exhausted:
  63. record("host compress_context total ceiling exhausted", failure_kind="ceiling_exhausted")
  64. else:
  65. record("host compress_context timeout (no summary progress)", failure_kind="stalled")
  66. except Exception:
  67. logger.debug("failed to record compress_context timeout cooldown", exc_info=True)
  68. emit = getattr(agent, "_emit_warning", None)
  69. if not callable(emit):
  70. return
  71. if total_exhausted:
  72. progress = " after summary output was observed" if progress_observed else ""
  73. emit(
  74. "⚠ Context compression reached its total ceiling "
  75. f"after {waited:.1f}s{progress}. No messages were "
  76. "dropped — continuing without compression. Run /compress to retry or /new for a clean session."
  77. )
  78. else:
  79. emit(
  80. f"⚠ Context compression timed out after {idle:.1f}s with no output from the summary "
  81. "model. No messages were dropped — continuing without compression. Run /compress to retry, /new "
  82. "for a clean session, or check auxiliary.compression."
  83. )
  84. def _warn_commit_overrun(agent, waited: float, ceiling: float) -> None:
  85. """Commit-phase ceiling breach: the SessionDB mutation must complete, so only surface it."""
  86. emit = getattr(agent, "_emit_warning", None)
  87. if callable(emit):
  88. emit(
  89. f"⚠ Context compression commit is taking unusually long ({waited:.0f}s, ceiling {ceiling:.0f}s). "
  90. "Waiting for it to finish safely — if this persists, check SessionDB health (disk / lock contention)."
  91. )
  92. def _sync_persisted_markers(target_messages, source_messages) -> None:
  93. """Mirror ``_DB_PERSISTED_MARKER`` stamps from the worker's snapshot onto a live list.
  94. Matched by scoped identity; timestamp-less repeated content is ambiguous, so every scoped match is
  95. stamped. Imported UNCONDITIONALLY: a silent fallback literal would split the stamping key from the flush's
  96. and resurrect the duplicate-row bug."""
  97. from agent.context_compressor import _DB_PERSISTED_MARKER
  98. from agent.conversation_compression import _stamp_scoped_twins
  99. if not isinstance(target_messages, list) or not isinstance(source_messages, list):
  100. return
  101. for source_message in source_messages:
  102. if isinstance(source_message, dict) and source_message.get(_DB_PERSISTED_MARKER):
  103. _stamp_scoped_twins(target_messages, source_message)
  104. def _run_under_progress_timeout(
  105. agent, run, messages, system_message, *, active_fence, registration, fence_registration_lock,
  106. idle_timeout, total_ceiling, approx_tokens=None,
  107. ):
  108. """Run ``run(fence, target_messages=snapshot)`` on the pool under the progress-aware timeout.
  109. The pooled worker must NEVER share the caller's live transcript — a late engine after a host timeout could
  110. rewrite it. It deep-snapshots on the worker and publishes only via an ADMITTED commit; a no-op/abort
  111. returns the snapshot unchanged, so the ORIGINAL list is handed back to keep identity semantics."""
  112. from agent.conversation_compression import (
  113. CompressionCommitFence, request_exceeds_model_window, run_compress_context_with_progress_timeout,
  114. )
  115. def _snapshot_worker(fence=None, *, same_turn_fallback_recovery=False):
  116. # #76354 review F3: the pooled worker must NEVER share the caller's live transcript. Plugin/legacy
  117. # context engines are allowed to mutate their input list in place; after a host timeout the worker
  118. # stays alive, so a shared list would let a late engine rewrite the live conversation (roles,
  119. # ordering, persisted content) behind the caller's back. Deep-snapshot here, on the worker thread,
  120. # so the caller's list object is never touched by pooled code. Results are published to
  121. # caller-visible state only via the returned value of an ADMITTED commit (the host discards results
  122. # on timeout/cancel); durable SessionDB mutation is already gated behind the commit fence inside
  123. # compress_context.
  124. snapshot = copy.deepcopy(messages)
  125. result_msgs, result_prompt = run(
  126. fence, target_messages=snapshot, same_turn_fallback_recovery=same_turn_fallback_recovery
  127. )
  128. return (messages if result_msgs is snapshot else result_msgs), result_prompt
  129. # The stall-fallback retry is the same recovery attempt as the stalled primary, but the cancelled primary
  130. # worker records its stall_interrupted cooldown while unwinding — racing the retry's automatic gate
  131. # (#112387). The retry therefore bypasses ONLY the summary-failure cooldown (never clears it; the
  132. # structural breakers stay in force), exactly like provider-proven overflow recovery.
  133. _same_turn_fallback_worker = functools.partial(_snapshot_worker, same_turn_fallback_recovery=True)
  134. timeout_cause = {"total_exhausted": False, "progress_observed": False}
  135. def _on_timeout_cause(total_exhausted, progress_observed):
  136. timeout_cause.update(total_exhausted=total_exhausted, progress_observed=progress_observed)
  137. def _on_timeout(idle, waited, since_progress):
  138. _report_compression_timeout(
  139. agent, idle=idle, waited=waited, since_progress=since_progress, total_ceiling=total_ceiling, **timeout_cause
  140. )
  141. def _publish_new_fence():
  142. # The stall-fallback retry needs a fence the aborted attempt cannot veto; publish
  143. # it on the slot hard_interrupt() reads, but only while this attempt still owns the
  144. # top registration. A later attempt's live fence must not be clobbered.
  145. retry_fence = CompressionCommitFence()
  146. with fence_registration_lock:
  147. registration.fence = retry_fence
  148. stack = vars(agent).get("_compression_commit_fence_stack") or ()
  149. if stack and stack[-1] is registration:
  150. agent._active_compression_commit_fence = retry_fence
  151. return retry_fence
  152. return run_compress_context_with_progress_timeout(
  153. worker=_snapshot_worker, messages=messages,
  154. system_prompt_fallback=lambda: _timeout_fallback_prompt(agent, system_message),
  155. idle_timeout_seconds=idle_timeout, total_ceiling_seconds=total_ceiling, on_timeout=_on_timeout,
  156. on_timeout_cause=_on_timeout_cause,
  157. on_commit_overrun=lambda waited, ceiling: _warn_commit_overrun(agent, waited, ceiling), fence=active_fence,
  158. telemetry_agent=agent, new_fence=_publish_new_fence, fallback_worker=_same_turn_fallback_worker,
  159. request_exceeds_window=request_exceeds_model_window(agent, approx_tokens) is True,
  160. )
  161. def _mirror_result_onto_live_lists(agent, result, messages, *, direct_path: bool) -> None:
  162. """Mirror persisted-marker stamps from the result list onto the live list(s)."""
  163. if not (isinstance(result, tuple) and result and isinstance(result[0], list)):
  164. return
  165. result_messages = result[0]
  166. # Direct-path callers bypass the snapshot worker but still need the post-publish mirror.
  167. if direct_path or result_messages is not messages:
  168. _sync_persisted_markers(messages, result_messages)
  169. session_messages = getattr(agent, "_session_messages", None)
  170. if isinstance(session_messages, list) and session_messages is not messages:
  171. # Durable-parent adoption can leave `_session_messages` on the pre-adoption list.
  172. _sync_persisted_markers(session_messages, result_messages)
  173. if result_messages is not messages:
  174. # The token-estimate memo pins message strings; bodies the summary replaced can never hit again,
  175. # and close() only sees the compressed history.
  176. from agent.model_metadata_token_memo import forget_message_token_memo
  177. with contextlib.suppress(Exception):
  178. forget_message_token_memo(messages, keep=result_messages)
  179. def _rebind_caller_session_context(agent) -> None:
  180. """Propagate a rotated session id to the CALLER's thread/ContextVar (idempotent otherwise).
  181. The worker thread rotated hermes_logging's thread-local id; post-compression tools must resolve
  182. HERMES_SESSION_ID to the child id."""
  183. with contextlib.suppress(Exception):
  184. from hermes_logging import set_session_context
  185. set_session_context(agent.session_id)
  186. try:
  187. from gateway.session_context import set_current_session_id
  188. if agent.session_id:
  189. set_current_session_id(agent.session_id)
  190. except Exception:
  191. logger.debug("post-compression session ContextVar rebind failed", exc_info=True)
  192. class CompressionFacadeMixin:
  193. """``_compress_context`` (see module docstring)."""
  194. def _compress_context(
  195. self, messages: list, system_message: str, *, approx_tokens: int | None = None, task_id: str = "default",
  196. focus_topic: str | None = None, force: bool = False, bypass_cooldown: bool = False,
  197. defer_context_engine_notification: bool = False, commit_fence=None, verbatim_tail: list | None = None,
  198. trigger: str | None = None, snapshot_is_current=None, overflow_reason: str | None = None,
  199. ) -> tuple:
  200. """Forwarder — see ``agent.conversation_compression.compress_context``.
  201. ``force=True`` (manual /compress) bypasses the summary-failure cooldown; ``bypass_cooldown=True``
  202. (provider-proven overflow recovery) runs one real attempt while the cooldown stays armed.
  203. ``force=True`` is passed by the manual ``/compress`` slash command so users can bypass the
  204. summary-failure cooldown after an auto-compress abort. Auto-compress callers use the default
  205. ``force=False``. See #100661.
  206. """
  207. # Per-attempt timeout signal for turn-start preflight and in-loop consumers: a stalled
  208. # compression must not be mistaken for a structural no-op. Thread-local + per-agent lock.
  209. # A stalled compression must not be mistaken for a structural no-op and followed by the oversized
  210. # provider request it was meant to prevent. The typed helper upgrades the simple attribute to
  211. # thread-local state guarded by a per-agent lock so overlapping automatic/manual entrypoints cannot
  212. # clobber each other's outcome (#98741).
  213. from agent.conversation_compression import (
  214. CompressionCommitFence, compress_context, reset_context_compression_timeout_outcome,
  215. resolve_context_compression_timeouts,
  216. )
  217. reset_context_compression_timeout_outcome(self)
  218. from agent.portal_tags import (
  219. get_affinity_scope, get_conversation_context, reset_affinity_scope, reset_conversation_context,
  220. set_affinity_scope, set_conversation_context,
  221. )
  222. from agent.prompt_cache_scope import declared_conversation_scope_safe
  223. # Out-of-turn compaction (/compact, gateway /compress, partial head compression) runs outside
  224. # run_conversation's ambient scope; publish the root as a fallback so the summarizer's call carries
  225. # the conversation tag. No-op for in-turn callers. Same for the ROUTING scope when declared.
  226. token = None
  227. if get_conversation_context() is None:
  228. root = self._conversation_root_id()
  229. if root:
  230. token = set_conversation_context(root)
  231. # Initialized alongside `token`: the turn-lease timeout/interrupt early returns leave the try block
  232. # before set_affinity_scope() runs, and the finally reads this name unconditionally
  233. # (UnboundLocalError otherwise — the 4 red cross-process lease tests on PR #97158).
  234. affinity_token = None
  235. if get_affinity_scope() is None:
  236. declared = declared_conversation_scope_safe(self)
  237. if declared:
  238. affinity_token = set_affinity_scope(declared)
  239. # Every compression has a fence; hard_interrupt() uses this exact instance to serialize cancel
  240. # admission against begin_commit(). Publication is serialized so overlapping automatic/manual
  241. # entrypoints cannot replace the fence of the attempt currently committing.
  242. active_fence = commit_fence or CompressionCommitFence()
  243. fence_registration_lock = vars(self).setdefault("_compression_commit_fence_lock", threading.RLock())
  244. registration = _CommitFenceRegistration(active_fence)
  245. try:
  246. with fence_registration_lock:
  247. fence_stack = vars(self).setdefault("_compression_commit_fence_stack", [])
  248. fence_stack.append(registration)
  249. self._active_compression_commit_fence = active_fence
  250. def _run(fence=None, target_messages=None, same_turn_fallback_recovery=False):
  251. return compress_context(
  252. self, target_messages if target_messages is not None else messages, system_message,
  253. approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, force=force,
  254. bypass_cooldown=bypass_cooldown or same_turn_fallback_recovery,
  255. defer_context_engine_notification=(defer_context_engine_notification), commit_fence=fence,
  256. verbatim_tail=verbatim_tail, trigger=trigger, snapshot_is_current=snapshot_is_current,
  257. overflow_reason=overflow_reason,
  258. )
  259. # Callers that already own a progress-aware wait (gateway session
  260. # hygiene) pass commit_fence and must not be double-wrapped.
  261. direct_path = commit_fence is not None
  262. if not direct_path:
  263. idle_timeout, total_ceiling = resolve_context_compression_timeouts()
  264. direct_path = idle_timeout <= 0
  265. if direct_path:
  266. result = _run(active_fence)
  267. else:
  268. result = _run_under_progress_timeout(
  269. self, _run, messages, system_message,
  270. active_fence=active_fence, registration=registration,
  271. fence_registration_lock=fence_registration_lock,
  272. idle_timeout=idle_timeout, total_ceiling=total_ceiling, approx_tokens=approx_tokens,
  273. )
  274. _mirror_result_onto_live_lists(self, result, messages, direct_path=direct_path)
  275. _rebind_caller_session_context(self)
  276. return result
  277. finally:
  278. # Remove only THIS attempt's registration. Completion order is not registration
  279. # order: restoring a saved "previous" fence would resurrect a dead fence over a
  280. # live newer attempt (and popping the slot outright would delete that attempt's
  281. # fence mid-run). The slot always tracks the newest live registration.
  282. with fence_registration_lock:
  283. # Re-read rather than reuse the try-block local: an early exception before
  284. # the append would otherwise raise NameError here and mask the real failure.
  285. fence_stack = vars(self).get("_compression_commit_fence_stack") or ()
  286. for idx in range(len(fence_stack) - 1, -1, -1):
  287. if fence_stack[idx] is registration:
  288. del fence_stack[idx]
  289. break
  290. if fence_stack:
  291. self._active_compression_commit_fence = fence_stack[-1].fence
  292. else:
  293. vars(self).pop("_active_compression_commit_fence", None)
  294. # Restore whatever the caller had, so a compaction never leaks its tag into the surrounding scope.
  295. if token is not None:
  296. reset_conversation_context(token)
  297. if affinity_token is not None:
  298. reset_affinity_scope(affinity_token)