turn_api_request.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. """Per-attempt request assembly for the conversation turn's retry loop: re-apply the
  2. reasoning echo pad and prompt-cache decoration for the CURRENT provider (a fallback may
  3. differ from the primary), build ``api_kwargs``, run the surrogate/ASCII chokepoints, Codex
  4. preflight, OpenRouter cache bypass, provider per-request headers, the LLM request middleware,
  5. the ``pre_api_request`` hook and the debug dump. Nothing here imports
  6. ``agent.conversation_loop`` at module level.
  7. """
  8. from __future__ import annotations
  9. from dataclasses import dataclass
  10. import logging
  11. from typing import Any
  12. from agent.message_sanitization import sanitize_outbound_kwargs, strip_images_for_rejecting_model
  13. from hermes_cli.observability.shared_metrics_efficiency import observe_request_tools
  14. from utils import env_var_enabled
  15. logger = logging.getLogger("agent.conversation_loop")
  16. @dataclass
  17. class ApiRequestBuild:
  18. """Always ``action == "fallthrough"``; the fields are the request-local values the caller
  19. rebinds for the attempt."""
  20. action: str
  21. api_messages: Any
  22. _moa_prepared_request: Any
  23. tools_for_api: Any
  24. api_kwargs: Any
  25. _original_api_kwargs: Any
  26. _llm_middleware_trace: Any
  27. def _set_extra_header(api_kwargs: Any, key: str, value: str) -> None:
  28. """Copy-on-write header set (the dict may be shared with the transport's defaults)."""
  29. _xh = dict(api_kwargs.get("extra_headers") or {})
  30. _xh[key] = value
  31. api_kwargs["extra_headers"] = _xh
  32. def _apply_user_turn_request_headers(agent: Any, api_kwargs: Any) -> None:
  33. """Provider per-request headers (turn attribution). Only the first payload built in a user turn (the
  34. first attempt of its first API call) is user-initiated, and it consumes the turn flag exactly once,
  35. whatever the route answers (or if its hook fails). Every later build of the turn (a 429/5xx retry, a
  36. fallback activation, a tool-loop follow-up) gets the route's default: the historical billing
  37. behaviour, kept on purpose."""
  38. if not getattr(agent, "_is_user_initiated_turn", False):
  39. return
  40. agent._is_user_initiated_turn = False
  41. from agent.provider_route_hooks import route_answer
  42. _request_headers = route_answer(
  43. "request_headers", provider=agent.provider or "", base_url=agent.base_url or "",
  44. model=agent.model or "", user_initiated=True, scope=agent)
  45. for _key, _value in (_request_headers or {}).items():
  46. _set_extra_header(api_kwargs, _key, _value)
  47. def _fire_pre_api_request_hook(
  48. agent: Any, api_kwargs: Any, api_messages: Any, _llm_middleware_trace: Any, *, messages: Any,
  49. original_user_message: Any, approx_tokens: Any, total_chars: Any, retry_count: Any,
  50. api_call_count: Any, api_request_id: Any, api_start_time: Any, effective_task_id: Any,
  51. turn_id: Any,
  52. ) -> None:
  53. from agent.conversation_loop import _system_prompt_for_hooks
  54. try:
  55. from hermes_cli.lifecycle import has_hook, invoke_hook as _invoke_hook
  56. if has_hook("pre_api_request"):
  57. request_messages = api_kwargs.get("messages")
  58. if not isinstance(request_messages, list):
  59. request_messages = api_kwargs.get("input")
  60. if not isinstance(request_messages, list):
  61. request_messages = api_messages
  62. # Shallow copies: plugins may retain the lists; deepcopy is costly.
  63. # ``request_messages``/``conversation_history`` are raw langfuse passthroughs.
  64. # Anthropic (``system``) and Responses/Codex (``instructions``) move the system
  65. # prompt out of messages; pass it for observability.
  66. _invoke_hook(
  67. "pre_api_request",
  68. task_id=effective_task_id,
  69. turn_id=turn_id,
  70. api_request_id=api_request_id,
  71. session_id=agent.session_id or "",
  72. user_message=original_user_message,
  73. conversation_history=list(messages),
  74. platform=agent.platform or "",
  75. model=agent.model,
  76. provider=agent.provider,
  77. base_url=agent.base_url,
  78. api_mode=agent.api_mode,
  79. api_call_count=api_call_count,
  80. retry_count=retry_count,
  81. request_messages=list(request_messages) if isinstance(request_messages, list) else [],
  82. system_prompt=_system_prompt_for_hooks(api_kwargs, request_messages),
  83. message_count=len(api_messages),
  84. tool_count=len(agent.tools or []),
  85. approx_input_tokens=approx_tokens,
  86. request_char_count=total_chars,
  87. max_tokens=agent.max_tokens,
  88. started_at=api_start_time,
  89. middleware_trace=list(_llm_middleware_trace),
  90. request=agent._api_request_payload_for_hook(api_kwargs),
  91. )
  92. except Exception:
  93. pass
  94. def build_request_payload(
  95. agent: Any, *, api_messages: Any, _moa_prepared_request: Any, tools_for_api: Any,
  96. system_message: Any, api_call_count: Any, api_request_id: Any, effective_task_id: Any,
  97. turn_id: Any,
  98. ) -> ApiRequestBuild:
  99. """The provider payload through the LLM request middleware, with no turn hook fired: the
  100. turn loop's :func:`build_api_request` and off-turn side requests (``ctx.side_question``,
  101. which reports through ``pre/post_auxiliary_call``) share it."""
  102. from agent.conversation_loop import _redecorate_prompt_cache_for_provider
  103. agent._reset_stream_delivery_tracking()
  104. # Per-attempt first-chunk timestamp so a stale value never leaks into post_api_request.
  105. agent._last_api_first_chunk_at = None
  106. # api_messages was built for the primary; a fallback (DeepSeek / Kimi / MiMo) may
  107. # require reasoning_content — re-apply the echo-back pad (idempotent) and re-render
  108. # the prompt-cache decoration for the current provider.
  109. agent._reapply_reasoning_echo_for_provider(api_messages)
  110. api_messages, _moa_prepared_request, tools_for_api = (
  111. _redecorate_prompt_cache_for_provider(
  112. agent, api_messages, system_message=system_message, moa_prepared=_moa_prepared_request,
  113. tools_for_api=tools_for_api,
  114. )
  115. )
  116. # A model that rejected image content gets text only; history keeps the images.
  117. strip_images_for_rejecting_model(agent, api_messages)
  118. observe_request_tools(agent, tools_for_api)
  119. if tools_for_api == agent.tools:
  120. api_kwargs = agent._build_api_kwargs(api_messages)
  121. else:
  122. api_kwargs = agent._build_api_kwargs(api_messages, tools_for_api=tools_for_api)
  123. # Messages were scrubbed above; this walk covers the rest of the payload (tool descriptions,
  124. # extra_body, kwargs strings) — see sanitize_outbound_kwargs for the #50959 rationale.
  125. sanitize_outbound_kwargs(agent, api_kwargs)
  126. if agent.api_mode == "codex_responses":
  127. from agent.codex_responses_adapter import drops_replay_item_ids
  128. api_kwargs = agent._get_transport().preflight_kwargs(
  129. api_kwargs, allow_stream=False, drop_replay_item_ids=drops_replay_item_ids(agent.base_url),
  130. sanitize_harmony_tokens=agent._is_codex_backend(),
  131. )
  132. # OpenRouter caching replays identical responses, even empty ones; an empty-response
  133. # retry must bypass the cache.
  134. if agent._empty_content_retries > 0 and agent._is_openrouter_url():
  135. _set_extra_header(api_kwargs, "X-OpenRouter-Cache", "false")
  136. _apply_user_turn_request_headers(agent, api_kwargs)
  137. try:
  138. from hermes_cli.middleware import apply_llm_request_middleware
  139. _llm_request_mw = apply_llm_request_middleware(
  140. api_kwargs, task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
  141. session_id=agent.session_id or "", platform=agent.platform or "", model=agent.model,
  142. provider=agent.provider, base_url=agent.base_url, api_mode=agent.api_mode,
  143. api_call_count=api_call_count,
  144. )
  145. api_kwargs = _llm_request_mw.payload
  146. _original_api_kwargs = _llm_request_mw.original_payload
  147. _llm_middleware_trace = _llm_request_mw.trace
  148. except Exception:
  149. logger.warning("LLM request middleware failed; sending the request unmodified", exc_info=True)
  150. _original_api_kwargs = dict(api_kwargs)
  151. _llm_middleware_trace = []
  152. return ApiRequestBuild(
  153. "fallthrough", api_messages, _moa_prepared_request, tools_for_api, api_kwargs,
  154. _original_api_kwargs, _llm_middleware_trace,
  155. )
  156. def build_api_request(
  157. agent: Any, *, api_messages: Any, _moa_prepared_request: Any, tools_for_api: Any,
  158. system_message: Any, messages: Any, original_user_message: Any, approx_tokens: Any,
  159. total_chars: Any, retry_count: Any, api_call_count: Any, api_request_id: Any,
  160. api_start_time: Any, effective_task_id: Any, turn_id: Any,
  161. ) -> ApiRequestBuild:
  162. """Assemble the attempt's request in the original order (every mutation happens BEFORE
  163. middleware/hooks/debug dumps observe the payload)."""
  164. from agent.conversation_loop import _moa_client_consumes_prepared_request
  165. built = build_request_payload(
  166. agent, api_messages=api_messages, _moa_prepared_request=_moa_prepared_request,
  167. tools_for_api=tools_for_api, system_message=system_message, api_call_count=api_call_count,
  168. api_request_id=api_request_id, effective_task_id=effective_task_id, turn_id=turn_id,
  169. )
  170. api_kwargs, api_messages, _moa_prepared_request = built.api_kwargs, built.api_messages, built._moa_prepared_request
  171. _fire_pre_api_request_hook(
  172. agent, api_kwargs, api_messages, built._llm_middleware_trace, messages=messages,
  173. original_user_message=original_user_message, approx_tokens=approx_tokens,
  174. total_chars=total_chars, retry_count=retry_count, api_call_count=api_call_count,
  175. api_request_id=api_request_id, api_start_time=api_start_time,
  176. effective_task_id=effective_task_id, turn_id=turn_id,
  177. )
  178. if env_var_enabled("HERMES_DUMP_REQUESTS"):
  179. agent._dump_api_request_debug(api_kwargs, reason="preflight")
  180. # Private to the in-process MoA facade; added after middleware/hooks/debug dumps so
  181. # none serializes it into the provider payload. Re-read the live client:
  182. # rotation/fallback/cleanup rebuild agent.client between attempts; a native OpenAI
  183. # client rejects this key (TypeError).
  184. if _moa_prepared_request is not None and agent.provider == "moa":
  185. if _moa_client_consumes_prepared_request(agent.client):
  186. api_kwargs["_moa_prepared_request"] = _moa_prepared_request
  187. else:
  188. logger.warning(
  189. "MoA client replaced mid-turn (client=%s); sending the "
  190. "prepared prompt without the MoA handshake",
  191. type(agent.client).__name__,
  192. )
  193. return built