auxiliary_hooks.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. """Plugin events for auxiliary LLM calls (#79733).
  2. ``pre_auxiliary_call`` / ``post_auxiliary_call`` fire once per physical provider attempt at the
  3. relay boundary of ``agent.auxiliary_client`` — the funnel every auxiliary task (titling,
  4. compression, MoA advisors/aggregator, vision, approval, ...) shares, retries and fallbacks
  5. included — carrying the ``pre_api_request`` / ``post_api_request`` payload shape plus
  6. ``aux_task``. They are deliberately DISTINCT events: the main-loop ``*_api_request`` events stay
  7. turn-scoped, so observability plugins keyed on turn identity never see auxiliary traffic unless
  8. they subscribe to these. Observer-only (returns ignored) and fail-open: a raising or hung
  9. callback is logged and the auxiliary call proceeds untouched.
  10. """
  11. from __future__ import annotations
  12. import logging
  13. import time
  14. from typing import Any, Awaitable, Callable, Dict, Optional
  15. logger = logging.getLogger(__name__)
  16. PRE_AUXILIARY_CALL = "pre_auxiliary_call"
  17. POST_AUXILIARY_CALL = "post_auxiliary_call"
  18. def _parent_turn_identity() -> dict[str, str]:
  19. """``session_id`` / ``task_id`` / ``turn_id`` / ``platform`` of the main turn this auxiliary
  20. call runs under, or empty strings for turn-less callers (cron, gateway idle work)."""
  21. ident = {"session_id": "", "task_id": "", "turn_id": "", "platform": ""}
  22. try:
  23. from agent.relay_runtime import current_turn
  24. turn = current_turn()
  25. except Exception:
  26. return ident
  27. if turn is None:
  28. return ident
  29. lease = getattr(turn, "lease", None)
  30. ident["session_id"] = str(getattr(lease, "session_id", "") or "")
  31. ident["platform"] = str(getattr(lease, "platform", "") or "")
  32. ident["task_id"] = str(getattr(turn, "task_id", "") or "")
  33. ident["turn_id"] = str(getattr(turn, "turn_id", "") or "")
  34. return ident
  35. def _system_prompt(messages: Any, kwargs: dict[str, Any]) -> str:
  36. if isinstance(kwargs.get("system"), str): # Anthropic
  37. return kwargs["system"]
  38. if isinstance(kwargs.get("instructions"), str): # Responses
  39. return kwargs["instructions"]
  40. if isinstance(messages, list) and messages and isinstance(messages[0], dict):
  41. first = messages[0]
  42. if first.get("role") == "system" and isinstance(first.get("content"), str):
  43. return first["content"]
  44. return ""
  45. def _usage_summary(response: Any) -> Optional[dict[str, Any]]:
  46. raw_usage = getattr(response, "usage", None)
  47. if response is None or not raw_usage:
  48. return None
  49. from dataclasses import asdict
  50. from agent.usage_pricing import normalize_usage
  51. # Every auxiliary attempt returns from ``chat.completions.create``: native adapters
  52. # (Codex Responses, Anthropic Messages) translate their usage to prompt/completion
  53. # tokens first, so the native api_mode/provider shape would read zeros here.
  54. cu = normalize_usage(raw_usage, api_mode="chat_completions")
  55. summary = asdict(cu)
  56. summary.pop("raw_usage", None)
  57. summary["prompt_tokens"] = cu.prompt_tokens
  58. summary["total_tokens"] = cu.total_tokens
  59. return summary
  60. def _first_choice_message(response: Any) -> Any:
  61. choices = getattr(response, "choices", None)
  62. if isinstance(response, dict):
  63. choices = response.get("choices")
  64. if not choices:
  65. return None, None
  66. choice = choices[0]
  67. if isinstance(choice, dict):
  68. return choice.get("message"), choice.get("finish_reason")
  69. return getattr(choice, "message", None), getattr(choice, "finish_reason", None)
  70. def _field(obj: Any, name: str) -> Any:
  71. return obj.get(name) if isinstance(obj, dict) else getattr(obj, name, None)
  72. class _AuxCallHooks:
  73. """Fires the pre/post pair for one provider attempt; the base payload is built once."""
  74. def __init__(
  75. self, *, aux_task: str, metadata: dict[str, Any], client: Any, kwargs: dict[str, Any],
  76. provider: str, model: str, api_mode: str, streaming: bool,
  77. ) -> None:
  78. self.streaming = streaming
  79. self.kwargs = kwargs
  80. self.started_at = time.time()
  81. self.base: dict[str, Any] = dict(_parent_turn_identity())
  82. self.base.update(
  83. aux_task=aux_task,
  84. api_request_id=str(metadata.get("api_request_id") or ""),
  85. retry_count=int(metadata.get("retry_count") or 0),
  86. api_call_count=int(metadata.get("retry_count") or 0) + 1,
  87. model=model,
  88. provider=provider,
  89. base_url=str(getattr(client, "base_url", "") or ""),
  90. api_mode=api_mode,
  91. streaming=streaming,
  92. started_at=self.started_at,
  93. message_count=len(kwargs.get("messages") or kwargs.get("input") or []),
  94. )
  95. if metadata.get("plugin_id"): # a call a plugin asked for (ctx.side_question)
  96. self.base["plugin_id"] = str(metadata["plugin_id"])
  97. def pre(self) -> None:
  98. if not _has_hook(PRE_AUXILIARY_CALL):
  99. return
  100. from agent.api_request_hooks import ApiRequestHooksMixin as _Sanitize
  101. kwargs = self.kwargs
  102. messages = kwargs.get("messages")
  103. if not isinstance(messages, list):
  104. messages = kwargs.get("input") # Responses API
  105. if not isinstance(messages, list):
  106. messages = []
  107. body = {k: v for k, v in kwargs.items() if k not in {"timeout", "http_client"}}
  108. total_chars = sum(len(str(_field(m, "content") or "")) for m in messages)
  109. _fire(
  110. PRE_AUXILIARY_CALL, **self.base,
  111. request_messages=list(messages),
  112. system_prompt=_system_prompt(messages, kwargs),
  113. tool_count=len(kwargs.get("tools") or []),
  114. approx_input_tokens=total_chars // 4,
  115. request_char_count=total_chars,
  116. max_tokens=kwargs.get("max_tokens") or kwargs.get("max_completion_tokens"),
  117. request=_Sanitize._sanitize_hook_payload({"method": "POST", "body": body}),
  118. )
  119. def post(
  120. self, response: Any = None, error: Optional[BaseException] = None, *,
  121. normalized: Optional[tuple[Any, Any, Optional[dict[str, Any]]]] = None,
  122. ) -> None:
  123. """``normalized`` = ``(assistant_message, finish_reason, usage_summary)`` for a response
  124. that is not chat.completions-shaped (an agent transport's native reply)."""
  125. if not _has_hook(POST_AUXILIARY_CALL):
  126. return
  127. from agent.api_request_hooks import ApiRequestHooksMixin as _Sanitize
  128. ended_at = time.time()
  129. payload: dict[str, Any] = dict(
  130. self.base, ended_at=ended_at, api_duration=max(0.0, ended_at - self.started_at),
  131. error=None if error is None else f"{type(error).__name__}: {error}"[:2000],
  132. error_type=None if error is None else type(error).__name__,
  133. )
  134. # A streamed response is handed back unconsumed (the MoA facade owns reassembly), so
  135. # there is no usage/finish_reason to report yet; ``streaming`` tells the observer why.
  136. if error is not None or self.streaming:
  137. payload.update(finish_reason=None, response_model=None, usage=None, response=None,
  138. assistant_content_chars=0, assistant_tool_call_count=0)
  139. else:
  140. message, finish_reason, usage = normalized or (*_first_choice_message(response), _usage_summary(response))
  141. content = _field(message, "content") if message is not None else None
  142. tool_calls = (_field(message, "tool_calls") if message is not None else None) or []
  143. payload.update(
  144. finish_reason=finish_reason,
  145. response_model=_field(response, "model"),
  146. usage=usage,
  147. response=_Sanitize._sanitize_hook_payload({
  148. "model": _field(response, "model"),
  149. "finish_reason": finish_reason,
  150. "assistant_message": {
  151. "role": (_field(message, "role") if message is not None else None) or "assistant",
  152. "content": content,
  153. "tool_calls": tool_calls,
  154. },
  155. "usage": payload.get("usage"),
  156. }),
  157. assistant_content_chars=len(content) if isinstance(content, str) else 0,
  158. assistant_tool_call_count=len(tool_calls),
  159. )
  160. _fire(POST_AUXILIARY_CALL, **payload)
  161. def _has_hook(name: str) -> bool:
  162. try:
  163. from hermes_cli.lifecycle import has_hook
  164. return has_hook(name)
  165. except Exception:
  166. return False
  167. def _fire(name: str, **payload: Any) -> None:
  168. """Dispatch one event; a failing subscriber is logged, never propagated (the aux task's
  169. result must not depend on an observer)."""
  170. try:
  171. from hermes_cli.lifecycle import invoke_hook
  172. invoke_hook(name, **payload)
  173. except Exception:
  174. logger.warning("%s plugin hook failed for aux_task=%s; continuing",
  175. name, payload.get("aux_task"), exc_info=True)
  176. def start_aux_call_hooks(**kw: Any) -> Optional[_AuxCallHooks]:
  177. """Fire ``pre_auxiliary_call`` for one attempt; the handle for :func:`finish_aux_call_hooks`,
  178. or None when nothing subscribes. For callers that drive the provider call themselves."""
  179. if not (_has_hook(PRE_AUXILIARY_CALL) or _has_hook(POST_AUXILIARY_CALL)):
  180. return None
  181. try:
  182. hooks = _AuxCallHooks(**kw)
  183. hooks.pre()
  184. except Exception:
  185. logger.warning("pre_auxiliary_call payload build failed; continuing", exc_info=True)
  186. return None
  187. return hooks
  188. def finish_aux_call_hooks(
  189. hooks: Optional[_AuxCallHooks], response: Any = None, error: Any = None, *,
  190. normalized: Optional[tuple[Any, Any, Optional[dict[str, Any]]]] = None,
  191. ) -> None:
  192. """Fire the ``post_auxiliary_call`` paired with :func:`start_aux_call_hooks`."""
  193. if hooks is None:
  194. return
  195. try:
  196. hooks.post(response, error, normalized=normalized)
  197. except Exception:
  198. logger.warning("post_auxiliary_call payload build failed; continuing", exc_info=True)
  199. def run_with_aux_hooks(
  200. call: Callable[[], Any], *, aux_task: str, metadata: dict[str, Any], client: Any,
  201. kwargs: dict[str, Any], provider: str, model: str, api_mode: str, streaming: bool = False,
  202. ) -> Any:
  203. """Run one synchronous provider attempt between ``pre_auxiliary_call`` and
  204. ``post_auxiliary_call``; the exception (if any) is reported in ``post`` and re-raised."""
  205. hooks = start_aux_call_hooks(aux_task=aux_task, metadata=metadata, client=client, kwargs=kwargs,
  206. provider=provider, model=model, api_mode=api_mode, streaming=streaming)
  207. try:
  208. response = call()
  209. except BaseException as exc:
  210. finish_aux_call_hooks(hooks, error=exc)
  211. raise
  212. finish_aux_call_hooks(hooks, response=response)
  213. return response
  214. async def arun_with_aux_hooks(
  215. call: Callable[[], Awaitable[Any]], *, aux_task: str, metadata: dict[str, Any], client: Any,
  216. kwargs: dict[str, Any], provider: str, model: str, api_mode: str,
  217. ) -> Any:
  218. """Async twin of :func:`run_with_aux_hooks`."""
  219. hooks = start_aux_call_hooks(aux_task=aux_task, metadata=metadata, client=client, kwargs=kwargs,
  220. provider=provider, model=model, api_mode=api_mode, streaming=False)
  221. try:
  222. response = await call()
  223. except BaseException as exc:
  224. finish_aux_call_hooks(hooks, error=exc)
  225. raise
  226. finish_aux_call_hooks(hooks, response=response)
  227. return response