side_question_request.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. """One text-only request on a detached cache-parity fork of the live session agent, for plugins
  2. (``ctx.side_question``).
  3. The fork replays the turn snapshot captured at finalization (system prompt, ``tools[]``,
  4. messages) verbatim plus one user prompt, so the request reads the parent's warm prompt cache.
  5. It runs the turn loop's own request phases (assembly, the payload build with its LLM
  6. middleware, ``perform_api_call``) exactly once: no tool dispatch (tool calls in the response
  7. are dropped), no retries, no fallback model, no compression, no persistence. Like any other
  8. side call it reports through ``pre/post_auxiliary_call`` (``aux_task="plugin_side_question"``
  9. plus the asking ``plugin_id``), never the turn-scoped ``pre/post_api_request``, and its usage is
  10. billed to the session the token was issued for as ``plugin_side_question:<plugin_id>`` (``/btw``
  11. keeps ``side_question``).
  12. """
  13. from __future__ import annotations
  14. import contextvars
  15. import logging
  16. import threading
  17. import time
  18. import uuid
  19. from dataclasses import dataclass, field
  20. from typing import Any, Optional
  21. logger = logging.getLogger(__name__)
  22. SIDE_QUESTION_ORIGIN = "side_question"
  23. # Hook/usage task for plugin side questions, distinct from /btw's ``side_question``.
  24. PLUGIN_SIDE_QUESTION_TASK = "plugin_side_question"
  25. # Tool entries the client executes; any other ``type`` (Responses ``web_search``, Anthropic
  26. # ``web_search_20250305``, ...) runs server-side and could act without Hermes seeing it.
  27. _CLIENT_TOOL_TYPES = frozenset({"function", "custom"})
  28. # OpenRouter request plugins that only pick the model; anything else (``web``, ...) executes.
  29. _ROUTING_ONLY_OPENROUTER_PLUGINS = frozenset({"pareto-router"})
  30. @dataclass(frozen=True)
  31. class SideQuestionCapture:
  32. """The parent's request inputs frozen at turn finalization (the live parent moves on).
  33. ``parent_ref`` is weak: a lease must not keep an evicted session's agent alive."""
  34. parent_ref: Any # weakref.ref[AIAgent]
  35. messages: list[dict[str, Any]]
  36. system_prompt: Optional[str]
  37. system_prompt_static: Optional[str]
  38. tools: list[dict[str, Any]]
  39. session_id: str
  40. turn_id: str
  41. runtime: tuple # runtime_key(parent) at capture
  42. context: contextvars.Context = field(repr=False)
  43. def runtime_key(agent: Any) -> tuple:
  44. """What decides which model the fork talks to and how: a turn routed elsewhere (voice chat)
  45. restores the main runtime after ``post_llm_call``, and the captured prompt must not go out
  46. on it."""
  47. return tuple(getattr(agent, name, None) for name in ("model", "provider", "base_url", "api_mode", "reasoning_config"))
  48. @dataclass
  49. class _Audit:
  50. plugin_id: str
  51. started: float = field(default_factory=time.monotonic)
  52. outcome: str = "error"
  53. usage: dict[str, Any] = field(default_factory=dict)
  54. def _agent_provider_refusal(agent: Any) -> Optional[str]:
  55. """Why ``agent``'s runtime cannot serve a text-only side request, or None.
  56. MoA fans out to advisor models; an agent-as-provider (ACP CLI shim, external-process
  57. provider, codex app-server, a client declaring ``SUPPORTS_HERMES_TOOL_CALLS = False``)
  58. runs its own tools on the remote side, which no client-side drop can stop."""
  59. from agent.background_review import _parent_can_emit_tool_calls
  60. from hermes_cli.runtime_provider_backends import _is_external_process_provider
  61. if agent.provider == "moa":
  62. return "moa"
  63. if (
  64. str(agent.base_url or "").lower().startswith(("acp://", "acp+tcp://"))
  65. or agent.api_mode == "codex_app_server"
  66. or _is_external_process_provider(agent.provider)
  67. or not _parent_can_emit_tool_calls(agent)
  68. ):
  69. return "agent_provider"
  70. return None
  71. def has_server_side_tools(api_kwargs: dict[str, Any]) -> bool:
  72. """True when the FINAL request asks the provider to execute something itself: a tool entry
  73. with a non-client ``type`` (transports rewrite client tools, e.g. Codex/xAI swap a client
  74. ``web_search`` for ``{"type": "web_search"}``), OpenAI ``web_search_options``, or an
  75. OpenRouter plugin other than a routing-only one."""
  76. extra_body = api_kwargs.get("extra_body") if isinstance(api_kwargs.get("extra_body"), dict) else {}
  77. # ``request_overrides`` land in extra_body, whose ``tools`` the SDK merge sends over the top level.
  78. for tool in [*(api_kwargs.get("tools") or ()), *(extra_body.get("tools") or ())]:
  79. tool_type = tool.get("type") if isinstance(tool, dict) else None
  80. if tool_type and tool_type not in _CLIENT_TOOL_TYPES:
  81. return True
  82. if "web_search_options" in api_kwargs or "web_search_options" in extra_body:
  83. return True
  84. plugins = [*(api_kwargs.get("plugins") or ()), *(extra_body.get("plugins") or ())]
  85. return any(not (isinstance(p, dict) and p.get("id") in _ROUTING_ONLY_OPENROUTER_PLUGINS) for p in plugins)
  86. def _build_fork(parent: Any, capture: SideQuestionCapture) -> Any:
  87. """A never-routed cache-parity fork pinned to the captured request inputs."""
  88. from agent.background_review import build_cache_parity_fork
  89. # task_cfg={} = never route: the side question must hit the parent's warm cache.
  90. fork, _rt, _routed = build_cache_parity_fork(
  91. parent, {}, max_iterations=1, write_origin=SIDE_QUESTION_ORIGIN,
  92. )
  93. fork._cached_system_prompt = capture.system_prompt
  94. # The static tier decides where the system-prompt cache breakpoint splits; rebuilding it on
  95. # the fork re-reads prompt files and can land the marker elsewhere.
  96. fork._cached_system_prompt_static = capture.system_prompt_static
  97. fork.tools = capture.tools
  98. fork.valid_tool_names = {tool["function"]["name"] for tool in capture.tools}
  99. fork.session_id = capture.session_id
  100. return fork
  101. def _send(fork: Any, capture: SideQuestionCapture, prompt: str, audit: _Audit) -> Any:
  102. """Assemble, build and perform ONE request on the fork; the normalized reply, or None on
  103. refusal / a cancelled call."""
  104. from agent import relay_llm
  105. from agent.auxiliary_hooks import finish_aux_call_hooks, start_aux_call_hooks
  106. from agent.side_question import trim_snapshot_for_fork
  107. from agent.turn_api_call import perform_api_call
  108. from agent.turn_api_request import build_request_payload
  109. from agent.turn_context import _PER_TURN_RESET_STATE
  110. from agent.turn_request_assembly import assemble_api_request
  111. from agent.turn_retry_state import TurnRetryState
  112. from agent.turn_truncation import normalize_response_for_agent
  113. from agent.turn_usage import record_response_usage
  114. turn_id = f"{capture.turn_id}:{SIDE_QUESTION_ORIGIN}"
  115. task_id = str(uuid.uuid4())
  116. api_request_id = f"{turn_id}:api:1"
  117. # Turn state a fork that never ran a turn lacks: the per-turn counters the request phases
  118. # read, the replay-canonicalization clock (build_api_messages), the ids middleware and relay
  119. # key on, and a non-streaming transport.
  120. for name, value in _PER_TURN_RESET_STATE:
  121. setattr(fork, name, value)
  122. fork._current_turn_timestamp = time.time()
  123. fork._current_turn_id, fork._current_task_id, fork._current_api_request_id = turn_id, task_id, api_request_id
  124. fork._disable_streaming = True
  125. messages = trim_snapshot_for_fork(capture.messages) + [{"role": "user", "content": prompt}]
  126. assembled = assemble_api_request(
  127. fork, messages=messages, current_turn_user_idx=len(messages) - 1, _ext_prefetch_cache=None,
  128. _plugin_user_context=None, moa_config=None, active_system_prompt=fork._cached_system_prompt,
  129. original_user_message=prompt, pending_moa_prepared_request=None, request_logger=logger,
  130. )
  131. built = build_request_payload(
  132. fork, api_messages=assembled.api_messages, _moa_prepared_request=None,
  133. tools_for_api=assembled.tools_for_api, system_message=None, api_call_count=1,
  134. api_request_id=api_request_id, effective_task_id=task_id, turn_id=turn_id,
  135. )
  136. if has_server_side_tools(built.api_kwargs):
  137. audit.outcome = "refused:server_side_tools"
  138. return None
  139. hooks = start_aux_call_hooks(
  140. aux_task=PLUGIN_SIDE_QUESTION_TASK, metadata={"api_request_id": api_request_id, "plugin_id": audit.plugin_id},
  141. client=fork.client,
  142. kwargs=built.api_kwargs, provider=str(fork.provider or ""), model=str(fork.model or ""),
  143. api_mode=str(fork.api_mode or ""), streaming=False,
  144. )
  145. api_start_time = time.time()
  146. # One hook bracket around the whole attempt: a reply that fails after the provider returned
  147. # (accounting, normalization) still closes the observer's pre_auxiliary_call with its error.
  148. try:
  149. verdict = perform_api_call(
  150. fork, api_kwargs=built.api_kwargs, _original_api_kwargs=built._original_api_kwargs,
  151. _llm_middleware_trace=built._llm_middleware_trace, _moa_prepared_request=None,
  152. _retry=TurnRetryState(), thinking_spinner=None, retry_count=0, api_call_count=1,
  153. api_request_id=api_request_id, effective_task_id=task_id, turn_id=turn_id, interrupted=False,
  154. )
  155. response = None if verdict.action == "break" else verdict.response
  156. if response is not None:
  157. record_response_usage(
  158. fork, response, messages=messages, api_call_count=1,
  159. api_duration=time.time() - api_start_time, compression_attempts=0, max_compression_attempts=0,
  160. )
  161. assistant_message = normalize_response_for_agent(fork, response)
  162. # The reply is in the transport's native shape: observers get the normalized view.
  163. normalized = (assistant_message, assistant_message.finish_reason,
  164. fork._usage_summary_for_api_request_hook(response))
  165. except BaseException as exc:
  166. finish_aux_call_hooks(hooks, error=exc)
  167. relay_llm.complete_logical_call(api_request_id, outcome="failed")
  168. raise
  169. if response is None:
  170. finish_aux_call_hooks(hooks, error=InterruptedError("side question cancelled"))
  171. relay_llm.complete_logical_call(api_request_id, outcome="cancelled")
  172. audit.outcome = "error"
  173. return None
  174. relay_llm.complete_logical_call(api_request_id, outcome="success")
  175. finish_aux_call_hooks(hooks, response, normalized=normalized)
  176. return assistant_message
  177. def _answer_text(fork: Any, assistant_message: Any) -> Optional[str]:
  178. """Visible text of the reply; its ``tool_calls`` are dropped, never dispatched."""
  179. from agent.message_content import flatten_message_text
  180. content = assistant_message.content
  181. text = content if isinstance(content, str) else flatten_message_text(content)
  182. return fork._strip_think_blocks(text or "").strip() or None
  183. def _log_audit(audit: _Audit) -> None:
  184. usage = audit.usage
  185. logger.info(
  186. "side_question plugin=%s outcome=%s calls=%d in=%d cache_read=%d cache_write=%d out=%d latency=%.1fs",
  187. audit.plugin_id or "?", audit.outcome,
  188. *(int(usage.get(key) or 0) for key in (
  189. "api_calls", "input_tokens", "cache_read_tokens", "cache_write_tokens", "output_tokens")),
  190. time.monotonic() - audit.started,
  191. )
  192. def ask_text_only(
  193. capture: SideQuestionCapture, prompt: str, *, plugin_id: str = "", timeout: float = 15.0,
  194. ) -> Optional[str]:
  195. """Ask the session's own agent ``prompt`` after the captured turn; its text answer or None.
  196. None on refusal (MoA, agent-as-provider, server-side tools in the final request), timeout,
  197. provider error or an empty / tool-call-only reply. The request parameters (tools, reasoning,
  198. max_tokens, tool_choice) are the parent's verbatim: changing any of them changes the cache
  199. key, and an inherited thinking budget 400s against a smaller max_tokens."""
  200. from agent.background_review import release_cache_parity_fork
  201. audit = _Audit(plugin_id)
  202. parent = capture.parent_ref()
  203. refusal = "parent_gone" if parent is None else _agent_provider_refusal(parent)
  204. if refusal is not None:
  205. audit.outcome = f"refused:{refusal}"
  206. _log_audit(audit)
  207. return None
  208. fork = None
  209. built: list = [] # the fork once built, for the timer
  210. timed_out = threading.Event()
  211. def _on_timeout() -> None:
  212. timed_out.set()
  213. # Soft interrupt with a system reason: aborts the in-flight request without being
  214. # recorded as a user Stop.
  215. for target in built:
  216. target.interrupt(None, tool_reason="side question timed out")
  217. # The budget starts before the build: runtime resolution (credentials, client setup) is part
  218. # of the caller's wait. A build cannot be interrupted, so one that outlasts the budget is
  219. # caught right after it and sends nothing.
  220. timer = threading.Timer(timeout, _on_timeout)
  221. timer.daemon = True
  222. timer.start()
  223. try:
  224. fork = _build_fork(parent, capture)
  225. built.append(fork)
  226. if timed_out.is_set():
  227. audit.outcome = "timeout"
  228. return None
  229. if runtime_key(parent) != capture.runtime:
  230. audit.outcome = "refused:runtime_changed"
  231. return None
  232. assistant_message = _send(fork, capture, prompt, audit)
  233. if assistant_message is None:
  234. return None
  235. answer = _answer_text(fork, assistant_message)
  236. audit.outcome = "ok" if answer else "empty"
  237. return answer
  238. except Exception:
  239. # Boundary: a plugin's side question returns None, it never raises into the plugin.
  240. audit.outcome = "timeout" if timed_out.is_set() else "error"
  241. logger.warning("side question %s (plugin=%s)", audit.outcome, plugin_id or "?", exc_info=True)
  242. return None
  243. finally:
  244. timer.cancel()
  245. if fork is not None:
  246. # Billed to the session the token was issued for: the parent's id may have rotated
  247. # (``/compress``, a session reset) since the turn ended.
  248. audit.usage = release_cache_parity_fork(
  249. parent, fork, task=f"{PLUGIN_SIDE_QUESTION_TASK}:{plugin_id or '?'}",
  250. session_id=capture.session_id)
  251. _log_audit(audit)