turn_request_assembly.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. """Per-iteration API request assembly for the conversation turn loop: build ``api_messages``
  2. from the transcript, append MoA context, inject prefills, run the context-engine selection
  3. hook and the send-time sanitizers, canonicalize for bit-perfect cache prefixes, build the
  4. request-local prompt-cache plan LAST (after every transcript mutation), prepare the
  5. persistent-MoA request, then measure request pressure. Nothing here imports
  6. ``agent.conversation_loop`` at module level (cycle) — loop-internal helpers resolve lazily
  7. so ``patch("agent.conversation_loop.X")`` sites keep intercepting.
  8. """
  9. from __future__ import annotations
  10. from dataclasses import dataclass
  11. import logging
  12. from typing import Any
  13. from agent.message_sanitization import _sanitize_messages_surrogates
  14. from agent.usage_anchor import anchored_context_tokens
  15. from agent.prompt_caching import build_prompt_cache_plan, effective_cache_ttl
  16. from agent.turn_context import build_api_messages
  17. logger = logging.getLogger("agent.conversation_loop")
  18. @dataclass
  19. class AssembledRequest:
  20. """Always ``action == "fallthrough"``; the fields are the iteration locals the assembly
  21. produces (``api_messages``/``tools_for_api`` are the decorated request copies — the
  22. canonical ``messages``/``agent.tools`` stay undecorated)."""
  23. action: str
  24. api_messages: Any
  25. tools_for_api: Any
  26. _moa_prepared_request: Any
  27. pending_moa_prepared_request: Any
  28. approx_tokens: Any
  29. request_pressure_tokens: Any
  30. total_chars: Any
  31. def _append_moa_context(agent: Any, api_messages: Any, moa_config: Any, original_user_message: Any) -> None:
  32. """Run the MoA reference models and append their aggregated context to the last user
  33. message (as a trailing text part on multimodal turns). Fail-open."""
  34. try:
  35. from agent.message_content import flatten_message_text as _flatten_mt
  36. from agent.moa_loop import _preset_temperature, aggregate_moa_context
  37. _moa_context = aggregate_moa_context(
  38. user_prompt=(
  39. original_user_message
  40. if isinstance(original_user_message, str)
  41. # Multimodal content list: extract visible text rather than
  42. # str()-ing parts, which would leak base64 image payloads.
  43. else _flatten_mt(original_user_message)
  44. ),
  45. api_messages=api_messages,
  46. reference_models=moa_config.get("reference_models") or [],
  47. aggregator=moa_config.get("aggregator") or {},
  48. temperature=_preset_temperature(moa_config, "reference_temperature"),
  49. aggregator_temperature=_preset_temperature(moa_config, "aggregator_temperature"),
  50. # None = no per-preset override; inherit auxiliary.moa_reference.timeout.
  51. reference_timeout=(
  52. float(moa_config["reference_timeout"])
  53. if moa_config.get("reference_timeout")
  54. else None
  55. ),
  56. degraded_reference_policy=str(
  57. moa_config.get("degraded_reference_policy") or "loud"
  58. ),
  59. agent=agent,
  60. )
  61. if not _moa_context:
  62. return
  63. for _msg in reversed(api_messages):
  64. if _msg.get("role") == "user":
  65. _base = _msg.get("content", "")
  66. if isinstance(_base, str):
  67. _msg["content"] = _base + "\n\n" + _moa_context
  68. elif isinstance(_base, list):
  69. _msg["content"] = [*_base, {"type": "text", "text": "\n\n" + _moa_context}]
  70. break
  71. except Exception as _moa_exc:
  72. logger.warning("MoA context aggregation failed: %s", _moa_exc)
  73. def _prepare_moa_request(agent: Any, api_messages: Any, pending_moa_prepared_request: Any) -> tuple:
  74. """Persistent-MoA request: rebase the pending prepared request onto the new messages
  75. when the client supports it, else prepare a fresh one. Returns
  76. ``(prepared_request, api_messages, pending_moa_prepared_request)``."""
  77. _moa_completions = getattr(getattr(agent.client, "chat", None), "completions", None)
  78. prepared: Any = None
  79. if pending_moa_prepared_request is not None:
  80. _rebase = getattr(_moa_completions, "rebase_prepared_request", None)
  81. if callable(_rebase):
  82. prepared = _rebase(pending_moa_prepared_request, api_messages)
  83. pending_moa_prepared_request = None
  84. if prepared is None:
  85. _prepare = getattr(_moa_completions, "prepare", None)
  86. if callable(_prepare):
  87. prepared = _prepare(api_messages)
  88. if prepared is not None:
  89. api_messages = prepared["messages"]
  90. return prepared, api_messages, pending_moa_prepared_request
  91. def assemble_api_request(
  92. agent: Any, *, messages: Any, current_turn_user_idx: Any, _ext_prefetch_cache: Any,
  93. _plugin_user_context: Any, moa_config: Any, active_system_prompt: Any,
  94. original_user_message: Any, pending_moa_prepared_request: Any, request_logger: Any,
  95. ) -> AssembledRequest:
  96. """Assemble the request in the original order. ORDER IS LOAD-BEARING: cache breakpoints
  97. are injected only after whitespace normalization, the orphan sweep, thinking-only drop /
  98. user merge and surrogate stripping, so the same row's bytes never vary across turns."""
  99. from agent.conversation_loop import (
  100. _CODEX_INCOMPLETE_NUDGE, _apply_context_engine_selection, _canonicalize_api_tool_calls,
  101. _clone_message_for_send, _midturn_request_pressure_tokens, _pressure_with_real_floor,
  102. )
  103. from agent.model_metadata import (
  104. estimate_messages_tokens_rough,
  105. estimate_native_anthropic_messages_tokens_rough,
  106. )
  107. api_messages, effective_system = build_api_messages(
  108. agent, messages, current_turn_user_idx=current_turn_user_idx,
  109. ext_prefetch_cache=_ext_prefetch_cache, plugin_user_context=_plugin_user_context,
  110. moa_config=moa_config, active_system_prompt=active_system_prompt,
  111. )
  112. if moa_config:
  113. _append_moa_context(agent, api_messages, moa_config, original_user_message)
  114. # Ephemeral prefill messages go right after the system prompt, API-call-time only.
  115. if agent.prefill_messages:
  116. sys_offset = 1 if (api_messages and api_messages[0].get("role") == "system") else 0
  117. for idx, pfm in enumerate(agent.prefill_messages):
  118. # Structural clone: the in-place sanitizers below must not write
  119. # through into agent.prefill_messages' nested containers.
  120. api_messages.insert(sys_offset + idx, _clone_message_for_send(pfm))
  121. # Per-turn context selection hook: an engine may select/replace context for THIS
  122. # call only — request-only, fail-open, and independent of should_compress().
  123. _sel_incoming = (
  124. messages[current_turn_user_idx] if 0 <= current_turn_user_idx < len(messages) else None
  125. )
  126. api_messages = _apply_context_engine_selection(
  127. agent, api_messages, messages, _sel_incoming, logger=request_logger
  128. )
  129. # Context selection may replace the request with a fresh clone of canonical history.
  130. # Re-apply durable rejection suppression after that final replacement hook.
  131. from agent.anthropic_thinking_replay import apply_rejected_thinking_suppression
  132. apply_rejected_thinking_suppression(agent, api_messages)
  133. # Runs unconditionally (not gated on context_compressor) so orphaned tool
  134. # results from session loading or manual message edits are always caught.
  135. api_messages = agent._sanitize_api_messages(api_messages)
  136. # Send-path vision eviction (#89296): compression only strips stale screenshots
  137. # when prune fires, and the Anthropic adapter's keep-window never sees
  138. # OpenAI-style tool-result image_url parts. The per-call clone is rewritten in
  139. # place; persisted history is untouched.
  140. from agent.context_compressor import evict_stale_outbound_tool_images
  141. from agent.image_eviction_policy import outbound_retired_image_ids
  142. evict_stale_outbound_tool_images(api_messages, outbound_retired_image_ids(agent))
  143. # One-time repeated-heal notice goes out via the status/warning callback, NEVER
  144. # appended to messages: the cached prompt prefix stays byte-identical.
  145. try:
  146. from agent.agent_runtime_helpers import consume_pending_sanitizer_heal_notice
  147. _heal_notice = consume_pending_sanitizer_heal_notice()
  148. if _heal_notice:
  149. agent._emit_warning(_heal_notice)
  150. except Exception:
  151. logger.debug("sanitizer heal notice delivery failed", exc_info=True)
  152. # Drop thinking-only assistant turns + merge adjacent users, API copy only:
  153. # Anthropic-style backends 400 on a trailing `thinking` block; history keeps it.
  154. # Off the Codex wire (e.g. after a reasoning-only stall fell over to a Chat Completions
  155. # provider, #67321) the synthetic continuation nudge is Codex-only control text: drop it
  156. # alongside the opaque replay state.
  157. _cross_protocol = agent.api_mode != "codex_responses"
  158. api_messages = agent._drop_thinking_only_and_merge_users(
  159. api_messages, drop_codex_reasoning_items=_cross_protocol,
  160. drop_nudge_marker=_CODEX_INCOMPLETE_NUDGE if _cross_protocol else None,
  161. )
  162. # Normalize whitespace and tool-call JSON for bit-perfect prefixes across turns
  163. # (KV-cache reuse on local servers, better cloud cache hits); API copy only.
  164. for am in api_messages:
  165. if isinstance(am.get("content"), str):
  166. am["content"] = am["content"].strip()
  167. _canonicalize_api_tool_calls(api_messages)
  168. # Strip lone surrogates (U+D800-U+DFFF) that some Ollama-served models emit;
  169. # they crash json.dumps() inside the OpenAI SDK and trigger the 3-retry cycle.
  170. _sanitize_messages_surrogates(api_messages)
  171. # No send-time pad loop here: ``repair_empty_non_final_messages`` (inside
  172. # ``_sanitize_api_messages``) is the single owner of empty-turn repair.
  173. # Build the request-local cache sections LAST, after every transcript mutation;
  174. # the canonical tool registry stays undecorated. Marked ``content`` becomes text
  175. # blocks the whitespace pass skips, so the same row's bytes vary across turns.
  176. tools_for_api = agent.tools
  177. if agent._use_prompt_caching and agent.provider != "moa":
  178. from agent.prompt_caching import envelope_tool_part_cache_markers_supported
  179. _static_system_prefix = getattr(agent, "_cached_system_prompt_static", None)
  180. _initial_cache_plan = build_prompt_cache_plan(
  181. api_messages,
  182. tools_for_api,
  183. # Clamp per-destination: a configured 1h regresses to 5m on
  184. # Qwen/Alibaba routes, whose context cache is 5m-only.
  185. cache_ttl=effective_cache_ttl(
  186. agent._cache_ttl, provider=agent.provider, model=agent.model
  187. ),
  188. native_anthropic=agent._use_native_cache_layout,
  189. static_system_prefix=(
  190. _static_system_prefix if isinstance(_static_system_prefix, str) else None
  191. ),
  192. direct_native_tool_cache=agent._direct_native_anthropic_tool_cache_capability(),
  193. # LiteLLM-style envelope routes forward part-level markers into
  194. # tool_result.content[] → non-retryable 400.
  195. tool_part_markers=envelope_tool_part_cache_markers_supported(
  196. getattr(agent, "provider", ""), getattr(agent, "base_url", "")
  197. ),
  198. )
  199. api_messages = _initial_cache_plan.messages
  200. tools_for_api = _initial_cache_plan.tools
  201. # Prepare the persistent-MoA request before measuring compression pressure: the
  202. # ephemeral advisor output is absent from ``messages``; ``create()`` reuses the
  203. # prepared request instead of running the advisors again.
  204. _moa_prepared_request = None
  205. if agent.provider == "moa":
  206. _moa_prepared_request, api_messages, pending_moa_prepared_request = _prepare_moa_request(
  207. agent, api_messages, pending_moa_prepared_request
  208. )
  209. # One image-stripped estimate feeds both figures; tools counted separately (50+
  210. # tools ≈ 20-30K tokens); total_chars is a rough proxy for logs/hooks only.
  211. # Charge stale thinking only when the active route replays it.
  212. from agent.turn_context import _agent_stale_thinking_on_wire
  213. if _agent_stale_thinking_on_wire(agent):
  214. if getattr(agent, "api_mode", "") == "anthropic_messages":
  215. from agent.anthropic_thinking_policy import native_anthropic_preserves_prior_thinking
  216. if native_anthropic_preserves_prior_thinking(
  217. getattr(agent, "base_url", ""), getattr(agent, "model", "")
  218. ):
  219. approx_tokens = estimate_native_anthropic_messages_tokens_rough(api_messages)
  220. else:
  221. approx_tokens = estimate_messages_tokens_rough(api_messages)
  222. else:
  223. approx_tokens = estimate_messages_tokens_rough(api_messages)
  224. else:
  225. approx_tokens = estimate_messages_tokens_rough(api_messages, charge_stale_thinking=False)
  226. # Route-aware: native Responses compaction prunes the wire payload, so the raw
  227. # history figure overstates it and fires needless local compression.
  228. # Route-aware pressure: when the upcoming request is eligible for native Responses compaction the
  229. # transport will checkpoint-prune the payload before sending — the generic durable-history figure
  230. # overstates the wire by orders of magnitude on a compacted session and fires a 600s local compression
  231. # the main request never needed (#96995, mirroring the turn-prologue preflight #96644/#96155).
  232. request_pressure_tokens = _midturn_request_pressure_tokens(
  233. agent, api_messages, effective_system or "", approx_tokens
  234. )
  235. # Usage-anchored override: real prompt_tokens (incl. system + tool schemas) +
  236. # delta estimate replaces the whole-history heuristic when the anchor is fresh.
  237. _anchored_pressure = anchored_context_tokens(messages, getattr(agent, "_usage_anchor", None))
  238. agent._request_pressure_anchored = _anchored_pressure is not None
  239. if _anchored_pressure is not None:
  240. request_pressure_tokens = _anchored_pressure
  241. else:
  242. # Rough fallback only: floor at the provider's last REAL prompt size (an anchored
  243. # figure is provider-exact and is never floored — on MoA turns that would re-add
  244. # the fan-out tokens the anchor excludes).
  245. request_pressure_tokens = _pressure_with_real_floor(
  246. agent.context_compressor, request_pressure_tokens
  247. )
  248. return AssembledRequest(
  249. "fallthrough", api_messages, tools_for_api, _moa_prepared_request,
  250. pending_moa_prepared_request, approx_tokens, request_pressure_tokens, approx_tokens * 4,
  251. )