moa_loop.py 80 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564
  1. """Mixture-of-Agents runtime helpers for /moa turns.
  2. The slash command marks one user turn as MoA-enabled; the normal agent loop still
  3. owns tool calling and turn termination, while this module gathers reference-model
  4. context before each model iteration.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import functools
  9. import hashlib
  10. import json
  11. import logging
  12. import re
  13. import threading
  14. import time
  15. from concurrent.futures import ThreadPoolExecutor, wait as _futures_wait
  16. from dataclasses import KW_ONLY, dataclass, replace
  17. from types import SimpleNamespace
  18. from typing import Any
  19. from agent.auxiliary_client import call_llm
  20. from agent.message_content import flatten_message_text
  21. from agent.message_metadata import DATA_USE
  22. from agent.moa_alternation import destination_key, is_role_alternation_rejection, merge_same_role_messages
  23. from agent.transports import get_transport
  24. from agent.usage_pricing import CanonicalUsage
  25. logger = logging.getLogger(__name__)
  26. # Privacy filter (moa.privacy_filter: '' | display | full): PII classes agent.redact
  27. # leaves alone. The phone pattern requires explicit delimiters so line numbers,
  28. # dates, times, SHAs, IPs and versions never match.
  29. # Advisor (reference) outputs can echo PII from the conversation — emails, phone numbers, credentials pasted
  30. # by the user — into surfaces the user may not expect: the labelled reference blocks rendered in the UI,
  31. # saved MoA trace files, and (in `full` mode) the guidance block injected into the aggregator prompt (issue
  32. # #59959). Secret/credential shapes (API-key prefixes, JWTs, private keys, DB connection strings, E.164
  33. # phone numbers) are handled by the repo's central redactor, ``agent.redact .redact_sensitive_text`` — the
  34. # MoA filter never re-implements those. The two patterns below cover the PII classes the central redactor
  35. # deliberately leaves alone for log/tool output (emails and formatted phone numbers). Pattern safety:
  36. # advisory text is frequently code-review-shaped — line numbers, timestamps, git SHAs, IDs, IP addresses. A
  37. # bare 10-digit match would mangle all of those, so the phone pattern requires clearly delimited formatting:
  38. # a parenthesized area code and/or explicit `-`/`.` separators between groups ((555) 123-4567, 555-123-4567,
  39. # 555.123.4567, +1 555-123-4567). Undelimited digit runs (5551234567), dates (2026-07-12), times (12:34:56),
  40. # hex IDs, and dotted quads never match. International numbers in E.164 form (+14155551234) are already
  41. # masked by the central redactor.
  42. _MOA_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
  43. _MOA_PHONE_RE = re.compile(
  44. r"(?<![\w.+-])" # no leading word char / dot / + / - (kills IPs, IDs, versions)
  45. r"(?:\+?1[ .-])?" # optional NA country code
  46. r"(?:\(\d{3}\)[ .-]?|\d{3}[.-])" # delimited area code: (555) or 555- / 555.
  47. r"\d{3}[.-]\d{4}" # exchange-subscriber with explicit separator
  48. r"(?![\w-])" # no trailing word char / hyphen
  49. )
  50. def _redact_reference_text(text: Any) -> Any:
  51. """Redact secrets (central redactor) then MoA PII patterns.
  52. force=True: the privacy filter is its own opt-in, independent of the global
  53. log-redaction toggle. code_file=True: keeps the ENV/JSON assignment heuristics
  54. (which mangle source snippets) off advisory prose/code.
  55. """
  56. if not isinstance(text, str) or not text:
  57. return text
  58. from agent.redact import redact_sensitive_text
  59. text = redact_sensitive_text(text, force=True, code_file=True)
  60. text = _MOA_EMAIL_RE.sub("[redacted email]", text)
  61. return _MOA_PHONE_RE.sub("[redacted phone]", text)
  62. def _moa_privacy_mode(moa_raw: Any) -> str:
  63. """Normalized privacy-filter mode from a raw ``moa`` config."""
  64. from hermes_cli.moa_config import coerce_privacy_filter
  65. raw = moa_raw if isinstance(moa_raw, dict) else {}
  66. return coerce_privacy_filter(raw.get("privacy_filter"))
  67. def _redact_reference_outputs(reference_outputs: list[tuple[str, str, Any]]) -> list[tuple[str, str, Any]]:
  68. """Redact advisor text in reference-output tuples; accounting slot untouched."""
  69. return [(label, _redact_reference_text(text), acct) for label, text, acct in reference_outputs]
  70. def _redact_message_content(content: Any) -> Any:
  71. """Redact a message's content: a string, or the text parts of a content-part list."""
  72. if isinstance(content, str):
  73. return _redact_reference_text(content)
  74. if isinstance(content, list):
  75. return [
  76. {**p, "text": _redact_reference_text(p.get("text"))}
  77. if isinstance(p, dict) and isinstance(p.get("text"), str)
  78. else p
  79. for p in content
  80. ]
  81. return content
  82. def _redact_trace_messages(messages: Any) -> Any:
  83. """Redact message copies for trace persistence (string or content-part lists)."""
  84. if not isinstance(messages, list):
  85. return messages
  86. return [
  87. {**m, "content": _redact_message_content(m.get("content"))} if isinstance(m, dict) else m
  88. for m in messages
  89. ]
  90. def _redact_trace_accounting(acct: Any) -> Any:
  91. """Copy a ``_RefAccounting`` with its trace text (messages/output) redacted."""
  92. if not isinstance(acct, _RefAccounting):
  93. return acct
  94. return replace(acct, messages=_redact_trace_messages(acct.messages), output=_redact_reference_text(acct.output))
  95. # Cold-start caches: preset and per-(provider, model) runtime are immutable for a turn.
  96. # A MoA preset switch used to re-resolve the full config + preset + every slot's provider runtime on EACH
  97. # create() call (once per tool-loop iteration), serially before the parallel fan-out could start — adding
  98. # 5-30s of "frozen" latency on complex presets (#66793).
  99. _preset_cache_lock = threading.Lock()
  100. _preset_cache: dict[tuple, Any] = {}
  101. def _resolve_preset_cached(preset_name: str) -> tuple[dict[str, Any], Any]:
  102. """``(preset, raw moa config)``; the resolved preset is cached per config file signature
  103. (skips resolve_moa_preset's full validation of the moa block on every create())."""
  104. from hermes_cli.config import get_config_path, load_config
  105. from hermes_cli.moa_config import resolve_moa_preset
  106. from hermes_cli.config_backend import config_cache_version_or_none
  107. from agent.safe_worker_policy import worker_config_snapshot
  108. # A safe worker's config is its frozen snapshot: never open the profile file (uncached there).
  109. cfg_stamp = None if worker_config_snapshot() is not None else config_cache_version_or_none(get_config_path())
  110. moa_raw = load_config().get("moa") or {}
  111. key = (cfg_stamp, preset_name)
  112. with _preset_cache_lock:
  113. preset = _preset_cache.get(key) if cfg_stamp is not None else None
  114. if preset is None:
  115. preset = resolve_moa_preset(moa_raw, preset_name)
  116. if cfg_stamp is not None:
  117. with _preset_cache_lock:
  118. _preset_cache.clear() # one live config stamp at a time
  119. _preset_cache[key] = preset
  120. return preset, moa_raw
  121. _runtime_cache_lock = threading.Lock()
  122. _runtime_cache: dict[tuple[str, str, str], tuple[float, dict[str, Any]]] = {}
  123. # Short TTL so rotated keys / base_url edits are picked up within 5 minutes.
  124. _RUNTIME_CACHE_TTL_SECONDS = 300.0
  125. # Cap on concurrent reference calls (guards pathologically large presets).
  126. _MAX_REFERENCE_WORKERS = 8
  127. @dataclass(slots=True)
  128. class _RefAccounting:
  129. """Per-reference usage, cost and full trace (third slot of a reference-output tuple).
  130. Cost is priced at the advisor's OWN rate and summed in dollars (advisors may run
  131. on a different model than the aggregator). Trace fields are only populated when
  132. tracing is on.
  133. """
  134. usage: Any
  135. cost_usd: Any = None
  136. cost_status: str | None = None
  137. cost_source: str | None = None
  138. _: KW_ONLY
  139. messages: Any = None
  140. output: str | None = None
  141. model: str | None = None
  142. provider: str | None = None
  143. temperature: Any = None
  144. # Per-tool-result char budget for the advisory view: tool CALLS are kept in full,
  145. # tool RESULTS are head+tail previewed. The aggregator always gets the full transcript.
  146. _REFERENCE_TOOL_RESULT_BUDGET = 4000
  147. # Reference system prompt: without this framing a reference assumes it is the acting
  148. # agent and refuses ("I can't access repositories") or tries to call tools.
  149. _REFERENCE_SYSTEM_PROMPT = (
  150. "You are a reference advisor in a Mixture of Agents (MoA) process. You are "
  151. "NOT the acting agent and you do NOT execute anything: you cannot call "
  152. "tools, run commands, browse, or access files, repositories, or URLs, and "
  153. "you should not try to or apologize for being unable to. A separate "
  154. "aggregator/orchestrator model holds those capabilities and will take the "
  155. "actual actions.\n\n"
  156. "CRITICAL: You must NEVER claim or imply that you have executed a command, "
  157. "downloaded a file, accessed a URL, or performed any action. You can only "
  158. "analyze and advise based on the conversation context. Examples of what to "
  159. "avoid:\n"
  160. "- Bad: \"I ran curl and got 404.\"\n"
  161. "- Bad: \"I downloaded the file successfully.\"\n"
  162. "- Bad: \"I checked the repository and found...\"\n"
  163. "- Good: \"Based on the error pattern, a curl request to that URL would likely return 404.\"\n"
  164. "- Good: \"The conversation suggests downloading this file may help.\"\n"
  165. "- Good: \"From the context, checking the repository would reveal...\"\n\n"
  166. "The conversation below is the current state of a task handled by that "
  167. "acting agent. Your job is to give your most intelligent analysis of that "
  168. "state: understand the goal, reason about the problem, and advise on what "
  169. "to do next. Surface the best approach, concrete next steps and tool-use "
  170. "strategy, likely pitfalls and risks, and anything the acting agent may "
  171. "have missed or gotten wrong. Assume any referenced files, URLs, or "
  172. "systems exist and reason about them from the context given rather than "
  173. "asking for access.\n\n"
  174. "Respond with your advice directly — no preamble, no disclaimers about "
  175. "tools or access. Advise in prose: never emit a tool call or a JSON "
  176. "tool-call object, because the aggregator replays what looks like one. "
  177. "Your response is private guidance handed to the "
  178. "aggregator, not an answer shown to the user. NEVER claim to have executed "
  179. "anything."
  180. )
  181. def _slot_label(slot: dict[str, Any]) -> str:
  182. label = f"{(slot.get('provider') or '').strip()}:{(slot.get('model') or '').strip()}"
  183. effort = str(slot.get("reasoning_effort") or "").strip()
  184. return f"{label}[reasoning={effort}]" if effort else label
  185. def _slot_reasoning_config(slot: dict[str, Any]) -> dict[str, Any] | None:
  186. """Translate optional per-MoA-slot reasoning_effort into runtime config."""
  187. try:
  188. from hermes_constants import parse_reasoning_effort
  189. return parse_reasoning_effort(slot.get("reasoning_effort"))
  190. except Exception: # pragma: no cover - bad config must not break MoA
  191. return None
  192. def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | None:
  193. """Aggregator reasoning config: slot > per-model > global (shared chokepoint).
  194. References deliberately do NOT fall back: inheriting a global ``xhigh`` into
  195. every advisor would multiply cost.
  196. """
  197. cfg = _slot_reasoning_config(aggregator)
  198. if cfg is not None:
  199. return cfg
  200. try:
  201. from hermes_cli.config import load_config
  202. from hermes_constants import resolve_reasoning_config
  203. return resolve_reasoning_config(load_config() or {}, str(aggregator.get("model") or ""))
  204. except Exception: # pragma: no cover - bad config must not break MoA
  205. return None
  206. def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]:
  207. """Slot → ``call_llm`` kwargs with the provider's real api_mode/base_url/api_key.
  208. Cached per (profile home, provider, model) with a short TTL. Falls back to bare provider/model
  209. on error — never cached, or a transient error would pin bare kwargs for a TTL.
  210. """
  211. provider = str(slot.get("provider") or "").strip()
  212. model = str(slot.get("model") or "").strip()
  213. # hermes_home_key() in the key: the resolved api_key/base_url are per-profile, and under a
  214. # multiplex gateway two profiles can share (provider, model) with different accounts.
  215. from hermes_constants import hermes_home_key
  216. cache_key = (hermes_home_key(), provider, model)
  217. now = time.monotonic()
  218. with _runtime_cache_lock:
  219. entry = _runtime_cache.get(cache_key)
  220. if entry is not None and now - entry[0] < _RUNTIME_CACHE_TTL_SECONDS:
  221. return entry[1]
  222. out: dict[str, Any] = {"provider": provider, "model": model}
  223. try:
  224. from hermes_cli.runtime_provider import resolve_runtime_provider
  225. rt = resolve_runtime_provider(requested=provider, target_model=model)
  226. out.update({k: rt[k] for k in ("base_url", "api_key", "api_mode") if rt.get(k)})
  227. overrides = rt.get("request_overrides")
  228. extra_body = overrides.get("extra_body") if isinstance(overrides, dict) else None
  229. if isinstance(extra_body, dict) and extra_body:
  230. out["extra_body"] = dict(extra_body)
  231. except Exception as exc:
  232. logger.warning("MoA slot %s: provider '%s' could not be resolved (%s); calling with bare provider/model",
  233. _slot_label(slot), provider, exc)
  234. return out
  235. with _runtime_cache_lock:
  236. _runtime_cache[cache_key] = (now, out)
  237. return out
  238. def _merge_slot_extra_body(slot_extra_body: Any, caller_extra_body: Any) -> Any:
  239. """Merge slot defaults with a caller override (caller wins) for ``call_llm``."""
  240. if not (isinstance(slot_extra_body, dict) and slot_extra_body):
  241. return caller_extra_body
  242. if isinstance(caller_extra_body, dict):
  243. return {**slot_extra_body, **caller_extra_body}
  244. return caller_extra_body or dict(slot_extra_body)
  245. def _agent_cache_opts(agent: Any) -> tuple[Any, Any]:
  246. """The live agent's ``(_cache_disabled, _cache_ttl)``; ``(None, None)`` without an agent."""
  247. return (None, None) if agent is None else (getattr(agent, "_cache_disabled", None), getattr(agent, "_cache_ttl", None))
  248. def _with_cache_disabled(runtime: dict[str, Any], cache_disabled: Any) -> dict[str, Any]:
  249. """Pin the live agent's cache disable onto a runtime snapshot (None is a no-op)."""
  250. return runtime if cache_disabled is None else {**runtime, "_cache_disabled": cache_disabled}
  251. def _reference_liveness_hook(agent: Any, label: str) -> Any:
  252. """Thread-local aux progress hook bridging advisor stream chunks to the agent's
  253. activity clock.
  254. The MoA fan-out runs in worker threads outside the main agent loop, so without this
  255. bridge nothing stamps the turn's activity clock while advisors stream — the
  256. turn-liveness watchdog then reads the clock frozen at the last main-loop stamp and
  257. force-aborts healthy, still-streaming advisor turns at ``agent.turn_liveness.timeout_s``
  258. (default 600s) even though the aux stream layer deliberately permits
  259. ``max(600, 4 × timeout)``. ``call_llm`` preserves a thread-local hook across its
  260. dispatch/protected-daemon paths and ticks it per substantive stream chunk, so each
  261. chunk re-arms the watchdog. A failed touch must never break the advisor call.
  262. """
  263. touch = getattr(agent, "_touch_activity", None)
  264. def _touch_on_progress() -> None:
  265. if callable(touch):
  266. touch(f"MoA reference {label}: stream progress")
  267. return _touch_on_progress
  268. def _touch_fanout_progress(agent: Any, description: str) -> None:
  269. """Best-effort activity stamp for fan-out milestones; never raises (display of
  270. progress must not decide turn liveness on its own, it only supports it)."""
  271. touch = getattr(agent, "_touch_activity", None)
  272. if callable(touch):
  273. with contextlib.suppress(Exception):
  274. touch(description)
  275. def _maybe_apply_moa_cache_control(
  276. messages: list[dict[str, Any]], runtime: dict[str, Any], *, cache_disabled: bool | None = None,
  277. cache_ttl: str | None = None,
  278. ) -> list[dict[str, Any]]:
  279. """Apply cache_control to an advisor/aggregator request when its route honors it.
  280. Same policy/marker helpers as the main loop; MoA has no static prefix so the
  281. legacy system-and-3 fallback is used. ``cache_disabled`` is stamped onto the
  282. stub so ``cache_ttl: off`` is honored; ``cache_ttl`` is clamped per destination.
  283. Returns the messages unchanged on any error.
  284. ``cache_disabled`` (or the live config when omitted) is stamped onto the policy stub so
  285. ``prompt_caching.cache_ttl: off`` is not bypassed by the blank-agent pattern (#76085).
  286. """
  287. try:
  288. from agent.agent_runtime_helpers import anthropic_prompt_cache_policy, blank_cache_policy_stub
  289. from agent.prompt_caching import (
  290. apply_anthropic_cache_control, effective_cache_ttl, envelope_tool_part_cache_markers_supported,
  291. )
  292. # Explicit kwarg > runtime snapshot (threaded from the live agent) > config.
  293. if cache_disabled is None and "_cache_disabled" in runtime:
  294. cache_disabled = runtime.get("_cache_disabled")
  295. provider, model, base_url = runtime.get("provider") or "", runtime.get("model") or "", runtime.get("base_url") or ""
  296. # blank_cache_policy_stub is the only sanctioned stub (carries _cache_disabled).
  297. should_cache, native_layout = anthropic_prompt_cache_policy(
  298. blank_cache_policy_stub(cache_disabled), provider=provider, base_url=base_url,
  299. api_mode=runtime.get("api_mode") or "", model=model,
  300. )
  301. if not should_cache:
  302. return messages
  303. # None → "5m"; cache-disabled routes already returned above. Envelope routes
  304. # reject part-level markers in tool_result.content[].
  305. return apply_anthropic_cache_control(
  306. messages, cache_ttl=effective_cache_ttl(cache_ttl, provider=provider, model=model),
  307. native_anthropic=native_layout,
  308. tool_part_markers=envelope_tool_part_cache_markers_supported(provider, base_url),
  309. )
  310. except Exception as exc: # pragma: no cover - decoration must never break a call
  311. logger.debug("MoA cache_control decoration skipped: %s", exc)
  312. return messages
  313. def _price_reference_response(
  314. response: Any, slot: dict[str, Any], runtime: dict[str, Any]
  315. ) -> tuple[Any, Any, str | None, str | None]:
  316. """Normalize a reference's usage with the slot's OWN provider/api_mode and price it
  317. at its own rate (hence fan-out cost is summed in dollars). Never raises."""
  318. from agent.usage_pricing import estimate_usage_cost, normalize_usage
  319. usage = CanonicalUsage()
  320. raw_usage = getattr(response, "usage", None)
  321. if raw_usage:
  322. with contextlib.suppress(Exception): # pragma: no cover - defensive
  323. usage = normalize_usage(raw_usage, provider=runtime.get("provider"), api_mode=runtime.get("api_mode"))
  324. try:
  325. cost = estimate_usage_cost(
  326. slot.get("model") or "", usage, provider=runtime.get("provider"),
  327. base_url=runtime.get("base_url"), api_key=runtime.get("api_key"),
  328. )
  329. return usage, cost.amount_usd, cost.status, cost.source
  330. except Exception: # pragma: no cover - defensive
  331. return usage, None, None, None
  332. def _slot_request_headers(runtime: dict[str, Any], agent: Any = None) -> dict[str, str] | None:
  333. """User-initiated request headers of the slot's provider, attributed at that provider's own endpoint
  334. (MoA attributes by slot provider, whatever account host the slot resolved to). Asked like every
  335. per-request hook (``route_answer``: memoized, one failure policy for the agent's session)."""
  336. from agent.auxiliary_client import _normalize_aux_provider
  337. from agent.provider_route_hooks import route_answer
  338. from providers import get_provider_profile
  339. provider = _normalize_aux_provider(str(runtime.get("provider") or ""))
  340. profile = get_provider_profile(provider) if provider else None
  341. if profile is None:
  342. return None
  343. return route_answer("request_headers", provider=provider, base_url=profile.base_url or "",
  344. model=str(runtime.get("model") or ""), user_initiated=True, scope=agent)
  345. def _run_reference(
  346. slot: dict[str, Any], ref_messages: list[dict[str, Any]], *, temperature: float | None = None,
  347. max_tokens: int | None = None, reference_timeout: float | None = None, context_length_cache: Any = None,
  348. cache_disabled: bool | None = None, cache_ttl: str | None = None, agent: Any = None,
  349. ) -> tuple[str, str, Any]:
  350. """Call one reference model; return ``(label, text, accounting)``. Never raises:
  351. a failed reference becomes a labelled ``[failed: …]`` note. Runs in a thread pool."""
  352. label = _slot_label(slot)
  353. runtime = _slot_runtime(slot)
  354. trace_fields = {"model": slot.get("model"), "provider": runtime.get("provider") or slot.get("provider"), "temperature": temperature}
  355. # The advisory view already stripped the agent's system prompt; this is the only one.
  356. messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages]
  357. try:
  358. # Trim to THIS model's window (advisors may be smaller than the aggregator); the
  359. # advisory view is append-only across iterations, so cache_control lets
  360. # iteration N+1 replay N's cached prefix.
  361. # Reference models may have a smaller window than the aggregator (e.g. kimi-k2.7-code @ 262K
  362. # advising a glm-5.2 @ 1M conversation); without this trim the provider returns a hard HTTP 400
  363. # which the except below silently converts to a [failed: …] note (issue #60345). Estimated AFTER the
  364. # advisory system prompt is prepended so its tokens count against the budget too.
  365. trimmed = _trim_messages_for_reference(
  366. messages, slot, runtime, reserve_output_tokens=max_tokens, context_length_cache=context_length_cache,
  367. )
  368. trimmed = _maybe_apply_moa_cache_control(trimmed, _with_cache_disabled(runtime, cache_disabled), cache_ttl=cache_ttl)
  369. # Providers may gate premium models on request attribution; MoA fan-out serves the
  370. # user's current turn, so mirror the main agent's user-initiated request headers.
  371. initiator_headers = _slot_request_headers(runtime, agent)
  372. if agent is not None:
  373. from agent.auxiliary_client import aux_progress_hook
  374. with aux_progress_hook(_reference_liveness_hook(agent, label)):
  375. response = call_llm(
  376. task="moa_reference", messages=trimmed, temperature=temperature,
  377. max_tokens=max_tokens,
  378. timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot),
  379. extra_headers=initiator_headers, **runtime,
  380. )
  381. else:
  382. response = call_llm(
  383. task="moa_reference", messages=trimmed, temperature=temperature,
  384. max_tokens=max_tokens,
  385. timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot),
  386. extra_headers=initiator_headers, **runtime,
  387. )
  388. output_text = _extract_text(response) or "(empty response)"
  389. acct = _RefAccounting(*_price_reference_response(response, slot, runtime), messages=trimmed, output=output_text, **trace_fields)
  390. return label, output_text, acct
  391. except Exception as exc:
  392. logger.warning("MoA reference model %s failed: %s", label, exc)
  393. note = f"[failed: {exc}]"
  394. return label, note, _RefAccounting(CanonicalUsage(), messages=messages, output=note, **trace_fields)
  395. # Output headroom reserved in the reference window when reference_max_tokens is unset.
  396. _REFERENCE_DEFAULT_OUTPUT_RESERVE = 8192
  397. # Estimator slack: estimate_messages_tokens_rough is a rough chars/4 heuristic.
  398. _REFERENCE_TRIM_SAFETY_FRACTION = 0.10
  399. def _reference_context_length(slot: dict[str, Any], runtime: dict[str, Any], cache: Any) -> int | None:
  400. """Context window for a slot, memoized in ``cache`` per (provider, model) when given.
  401. Failures are cached too so a flaky metadata source is not re-probed per reference.
  402. """
  403. from agent.model_metadata import get_model_context_length
  404. model = str(slot.get("model") or "")
  405. provider = str(runtime.get("provider") or slot.get("provider") or "")
  406. key = (provider, model)
  407. has_cache = isinstance(cache, dict)
  408. if has_cache and key in cache:
  409. return cache[key]
  410. try:
  411. context_length = get_model_context_length(
  412. model=model, base_url=str(runtime.get("base_url") or ""), api_key=str(runtime.get("api_key") or ""), provider=provider,
  413. )
  414. except Exception:
  415. logger.debug("MoA reference context-length resolution failed for %s", _slot_label(slot))
  416. context_length = None
  417. if has_cache:
  418. cache[key] = context_length
  419. return context_length
  420. def _trim_messages_for_reference(
  421. messages: list[dict[str, Any]], slot: dict[str, str], runtime: dict[str, Any], *,
  422. reserve_output_tokens: int | None = None, context_length_cache: Any = None,
  423. ) -> list[dict[str, Any]]:
  424. """Trim an advisory request to fit a reference model's context window.
  425. Budget = window − ``reserve_output_tokens`` (or a default) − a safety fraction.
  426. Drops the OLDEST frames after the system prompt, always keeping a user-first
  427. body and the trailing user turn plus one preceding turn (even if still over
  428. budget). ``context_length_cache`` memoizes the window per (provider, model);
  429. unresolvable windows leave messages unchanged.
  430. Reference models may have a smaller context window than the aggregator or the main conversation. Without
  431. this trim, a reference whose window is exceeded gets a hard HTTP 400 from the provider, which
  432. ``_run_reference``'s try/except silently converts to a ``[failed: …]`` note — the MoA turn silently
  433. degrades to fewer references (issue #60345).
  434. """
  435. if not messages or not slot.get("model"):
  436. return messages
  437. from agent.model_metadata import estimate_messages_tokens_rough
  438. context_length = _reference_context_length(slot, runtime, context_length_cache)
  439. if not isinstance(context_length, int) or context_length <= 0:
  440. return messages
  441. reserve = reserve_output_tokens if isinstance(reserve_output_tokens, int) and reserve_output_tokens > 0 else _REFERENCE_DEFAULT_OUTPUT_RESERVE
  442. budget = int(context_length * (1.0 - _REFERENCE_TRIM_SAFETY_FRACTION)) - reserve
  443. estimated = estimate_messages_tokens_rough(messages)
  444. if budget <= 0 or estimated <= budget:
  445. return messages
  446. has_system = bool(messages) and messages[0].get("role") == "system"
  447. head_count = 1 if has_system else 0
  448. # estimate_messages_tokens_rough is a pure sum of per-message weights,
  449. # so weigh each message once and track a running total while popping
  450. # instead of re-estimating head + body after every pop. The naive form
  451. # paid one full memo walk per dropped frame (quadratic on long
  452. # histories). The pop sequence is unchanged, so the trim result is
  453. # identical to the naive loop.
  454. weights = [estimate_messages_tokens_rough([m]) for m in messages]
  455. total = sum(weights)
  456. # Keep the trailing user turn plus at least one preceding turn.
  457. start = head_count
  458. remaining = len(messages) - head_count
  459. while remaining > 2 and total > budget:
  460. total -= weights[start]
  461. start += 1
  462. remaining -= 1
  463. # Preserve the user-first invariant: never leave the advisory
  464. # conversation starting on an assistant turn after a pop.
  465. while remaining > 2 and messages[start].get("role") == "assistant":
  466. total -= weights[start]
  467. start += 1
  468. remaining -= 1
  469. # The loop can stop with two frames left where the first is an
  470. # assistant turn — enforce user-first even then (a lone trailing user
  471. # turn is a valid request; an assistant-first one is not).
  472. while remaining > 1 and messages[start].get("role") == "assistant":
  473. total -= weights[start]
  474. start += 1
  475. remaining -= 1
  476. trimmed = messages[:head_count] + messages[start:]
  477. dropped = len(messages) - len(trimmed)
  478. if dropped:
  479. logger.info(
  480. "MoA reference %s: estimated %d tokens exceeds budget %d "
  481. "(window %d, output reserve %d); dropped %d oldest message(s).",
  482. _slot_label(slot), estimated, budget, context_length, reserve, dropped,
  483. )
  484. return trimmed
  485. _REFERENCE_POLL_INTERVAL_S = 5.0
  486. _REFERENCE_INTERRUPT_SETTLE_S = 0.05
  487. # Sentinel for a reference aborted by user interrupt; the facade must never cache it.
  488. _INTERRUPTED_REFERENCE_NOTE = "[skipped: interrupted by user]"
  489. def _placeholder_output(slot: dict[str, Any], note: str) -> tuple[str, str, Any]:
  490. """A reference-output tuple for a slot that was not (fully) run: zero accounting."""
  491. return _slot_label(slot), note, _RefAccounting(CanonicalUsage())
  492. def _settle_interrupted(
  493. futures: dict[Any, int], results: list, reference_models: list[dict[str, Any]], late_accounting_sink: Any,
  494. ) -> None:
  495. """Fill every unfinished slot after a user interrupt: cancel never-dispatched
  496. futures (nothing billed), keep real output of ones that just finished, and hand
  497. running ones (cannot be killed, WILL bill) to ``late_accounting_sink``."""
  498. for future, idx in futures.items():
  499. if results[idx] is not None:
  500. continue
  501. # #38922: a slow confirmation does NOT necessarily mean the send failed — but we must distinguish
  502. # two cases via future.cancel()'s return value: cancel() == False -> the coroutine was already
  503. # running on the gateway loop when the timeout fired; the request is in flight on the wire and
  504. # cannot be un-sent. Re-sending via standalone would be a guaranteed DUPLICATE, so treat it as
  505. # delivered (assume-delivered). cancel() == True -> the scheduled callback never started executing
  506. # (loop wedged/backlogged for the full 60s), so nothing was sent. We MUST fall through to the
  507. # standalone path or the message is silently dropped (worse than a duplicate).
  508. cancelled = future.cancel()
  509. if not cancelled and future.done():
  510. results[idx] = future.result()
  511. continue
  512. results[idx] = _placeholder_output(reference_models[idx], _INTERRUPTED_REFERENCE_NOTE)
  513. if not cancelled and late_accounting_sink is not None:
  514. future.add_done_callback(functools.partial(_record_late, late_accounting_sink, results[idx][0]))
  515. def _record_late(sink: Any, label: str, future: Any) -> None:
  516. """Done-callback for an abandoned reference future: forward its real accounting."""
  517. try:
  518. sink(label, future.result()[2])
  519. except Exception: # pragma: no cover - defensive
  520. logger.debug("MoA: late accounting sink failed for %s", label)
  521. def _run_references_parallel(
  522. reference_models: list[dict[str, Any]], ref_messages: list[dict[str, Any]], *,
  523. temperature: float | None = None, max_tokens: int | None = None, progress_callback: Any = None,
  524. reference_timeout: float | None = None, agent: Any = None, late_accounting_sink: Any = None,
  525. ) -> list[tuple[str, str, Any]]:
  526. """Fan out all reference models in parallel; ``(label, text, _RefAccounting)`` per
  527. slot in ``reference_models`` order.
  528. ``provider == "moa"`` slots are skipped with a note (recursion guard).
  529. ``progress_callback(refs_done, refs_total, label)`` fires per completion. With
  530. *agent*, the wait polls every ``_REFERENCE_POLL_INTERVAL_S`` so a user interrupt
  531. can abort it; in-flight calls cannot be killed and bill via ``late_accounting_sink``.
  532. """
  533. if not reference_models:
  534. return []
  535. total = len(reference_models)
  536. results: list[tuple[str, str, Any] | None] = [None] * total
  537. futures: dict[Any, int] = {}
  538. # Propagate the turn's contextvars (approval callbacks, Nous conversation tag).
  539. from tools.thread_context import propagate_context_to_thread
  540. completed = 0
  541. executor = ThreadPoolExecutor(max_workers=min(_MAX_REFERENCE_WORKERS, total))
  542. interrupted = False
  543. # Shared per-fan-out context-length cache (dict get/set is GIL-atomic).
  544. ctx_len_cache: dict[tuple[str, str], int | None] = {}
  545. cache_disabled, cache_ttl = _agent_cache_opts(agent)
  546. def collect(done: set[Any]) -> None:
  547. nonlocal completed
  548. for future in done:
  549. idx = futures[future]
  550. results[idx] = future.result()
  551. completed += 1
  552. # Each completion is fan-out progress the main loop cannot see: the
  553. # fan-out blocks it, so stamp the turn's activity clock here (non-streaming
  554. # advisors otherwise tick nothing between dispatch and completion).
  555. _touch_fanout_progress(agent, f"MoA: {completed} of {total} references complete")
  556. if progress_callback is not None:
  557. try:
  558. progress_callback(completed, total, _slot_label(reference_models[idx]))
  559. except Exception as exc: # pragma: no cover - display must never break
  560. logger.debug("MoA progress_callback failed: %s", exc)
  561. try:
  562. for idx, slot in enumerate(reference_models):
  563. if slot.get("provider") == "moa":
  564. results[idx] = _placeholder_output(slot, "[skipped: MoA presets cannot recursively reference MoA]")
  565. continue
  566. futures[
  567. executor.submit(
  568. propagate_context_to_thread(_run_reference),
  569. slot,
  570. ref_messages,
  571. temperature=temperature,
  572. max_tokens=max_tokens,
  573. reference_timeout=reference_timeout,
  574. context_length_cache=ctx_len_cache,
  575. cache_disabled=cache_disabled,
  576. cache_ttl=cache_ttl,
  577. agent=agent,
  578. )
  579. ] = idx
  580. # Collect every reference (no early exit except a user interrupt).
  581. pending = set(futures)
  582. while pending:
  583. done, pending = _futures_wait(pending, timeout=_REFERENCE_POLL_INTERVAL_S)
  584. collect(done)
  585. if pending and agent is not None and getattr(agent, "_interrupt_requested", False):
  586. # A worker can raise the interrupt immediately before publishing
  587. # its own result. Give concurrently-finishing work one scheduler
  588. # turn so completed output is not replaced by an interrupt note.
  589. done, pending = _futures_wait(pending, timeout=_REFERENCE_INTERRUPT_SETTLE_S)
  590. collect(done)
  591. interrupted = True
  592. _settle_interrupted(futures, results, reference_models, late_accounting_sink)
  593. break
  594. finally:
  595. executor.shutdown(wait=not interrupted, cancel_futures=interrupted)
  596. return [r for r in results if r is not None]
  597. def _truncate_tool_result(text: str, budget: int = _REFERENCE_TOOL_RESULT_BUDGET) -> str:
  598. """Head+tail preview of a tool result with an ``[... N chars omitted ...]`` marker."""
  599. if not text or len(text) <= budget:
  600. return text
  601. half = budget // 2
  602. omitted = len(text) - 2 * half
  603. return f"{text[:half]}\n[... {omitted} chars omitted ...]\n{text[-half:]}"
  604. def _field(obj: Any, name: str) -> Any:
  605. """Read ``name`` from a dict or an attribute object (tool calls arrive as either)."""
  606. if obj is None:
  607. return None
  608. return obj.get(name) if isinstance(obj, dict) else getattr(obj, name, None)
  609. def _render_tool_calls(tool_calls: Any) -> str:
  610. """Render an assistant turn's tool_calls as ``[called tool: name(args)]`` lines.
  611. Tolerates dict- and SimpleNamespace-shaped entries (and nested ``function``).
  612. """
  613. lines: list[str] = []
  614. for tc in tool_calls or []:
  615. fn = _field(tc, "function")
  616. name = _field(fn, "name") or _field(tc, "name") or "tool"
  617. fn_args = _field(fn, "arguments")
  618. if isinstance(fn_args, str) or fn_args is None:
  619. args_text = fn_args or ""
  620. else:
  621. try:
  622. args_text = json.dumps(fn_args, ensure_ascii=False)
  623. except Exception:
  624. args_text = str(fn_args)
  625. lines.append(f"[called tool: {name}({args_text})]" if args_text else f"[called tool: {name}]")
  626. return "\n".join(lines)
  627. # Cached guidance (user_turn / off-cadence every_n fanout) is reused on later iterations of the
  628. # same turn, where it predates the tool results the acting model now sees. Without this line the
  629. # block reads as fresh instruction and an advisor's suggested tool call gets replayed after it
  630. # already ran.
  631. _STALE_GUIDANCE_NOTE = (
  632. "This guidance was produced earlier in this turn, before the tool results below it. "
  633. "Check the transcript before acting on it: a step it suggests may already have run, and "
  634. "repeating a completed tool call is never the next step.\n"
  635. )
  636. _ADVISORY_INSTRUCTION = (
  637. "[The conversation above is the current state of the task. Give your "
  638. "most intelligent judgement: what is going on, what should happen next, "
  639. "what risks or mistakes you see, and how the acting agent should "
  640. "proceed.]"
  641. )
  642. def _tool_activity_since_last_user(messages: list[dict[str, Any]]) -> bool:
  643. """Whether the acting model has already called tools since the last real user turn.
  644. Guidance cached from the start of the turn predates those results, so replaying a
  645. tool call it suggests can repeat work the transcript already shows as done."""
  646. for msg in reversed(messages):
  647. role = msg.get("role")
  648. if role == "user":
  649. return False
  650. if role == "tool" or (role == "assistant" and msg.get("tool_calls")):
  651. return True
  652. return False
  653. def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  654. """Build the advisory (reference-model) view of the conversation.
  655. Plain user/assistant TEXT turns only: system prompt dropped, tool_calls rendered
  656. inline, tool results folded into the preceding assistant turn as previews (no
  657. tool-role messages / tool_calls arrays, so strict providers do not 400). Always
  658. ends on a ``user`` turn (Anthropic treats a trailing assistant turn as prefill)
  659. by APPENDING a synthetic request. The aggregator always gets the full transcript.
  660. """
  661. rendered: list[dict[str, Any]] = []
  662. last_user_content: str | None = None
  663. for msg in messages:
  664. role = msg.get("role")
  665. content = msg.get("content")
  666. # Decorated (cache_control parts) and undecorated transcripts must yield a
  667. # byte-identical view so the advisory prefix stays cache-stable.
  668. text = flatten_message_text(content)
  669. if role == "user":
  670. if not text.strip() and isinstance(content, list) and content:
  671. # Image-only turn: strict providers reject empty user messages and
  672. # skipping would break alternation.
  673. text = "[user sent non-text content (e.g. an image attachment)]"
  674. if not text.strip():
  675. continue # genuinely empty user turn: strict providers 400 on it
  676. last_user_content = text
  677. rendered.append({"role": "user", "content": text})
  678. elif role == "assistant":
  679. parts = [text.strip()] if text.strip() else []
  680. calls_text = _render_tool_calls(msg.get("tool_calls"))
  681. if calls_text:
  682. parts.append(calls_text)
  683. if parts: # empty assistant turns carry nothing advisory
  684. rendered.append({"role": "assistant", "content": "\n".join(parts)})
  685. elif role == "tool":
  686. # Fold the tool result into the preceding assistant turn as text (a leading
  687. # tool result with no assistant turn opens one).
  688. block = f"[tool result: {_truncate_tool_result(text)}]"
  689. if rendered and rendered[-1].get("role") == "assistant":
  690. rendered[-1]["content"] = rendered[-1]["content"] + "\n" + block
  691. else:
  692. rendered.append({"role": "assistant", "content": block})
  693. # The preview is still the raw result, so the folded turn keeps its data-use requirement.
  694. if msg.get(DATA_USE):
  695. rendered[-1][DATA_USE] = sorted({*rendered[-1].get(DATA_USE, ()), *msg[DATA_USE]})
  696. # system and any other role are ignored.
  697. # Anthropic rejects trailing assistant prefill: end on a synthetic user request.
  698. if rendered and rendered[-1].get("role") == "assistant":
  699. rendered.append({"role": "user", "content": _ADVISORY_INSTRUCTION})
  700. if not rendered:
  701. # Nothing rendered: fall back to the latest user turn.
  702. if last_user_content is not None:
  703. return [{"role": "user", "content": last_user_content}]
  704. for msg in reversed(messages):
  705. fallback_text = flatten_message_text(msg.get("content")) if msg.get("role") == "user" else ""
  706. if fallback_text.strip():
  707. return [{"role": "user", "content": fallback_text}]
  708. return rendered
  709. def _extract_text(response: Any) -> str:
  710. """Assistant text of a completed response: transport-normalized, else ``choices[0]``."""
  711. with contextlib.suppress(Exception):
  712. text = (get_transport("chat_completions").normalize_response(response).content or "").strip()
  713. if text:
  714. return text
  715. try:
  716. message = response.choices[0].message
  717. content = message.get("content") if isinstance(message, dict) else getattr(message, "content", message)
  718. if not isinstance(content, str):
  719. content = str(content) if content else ""
  720. return content.strip()
  721. except Exception:
  722. return ""
  723. def _preset_temperature(preset: dict[str, Any], key: str) -> float | None:
  724. """Read an optional preset temperature; None (absent/empty/null) = provider default."""
  725. value = preset.get(key)
  726. if value is None or (isinstance(value, str) and not value.strip()):
  727. return None
  728. try:
  729. return float(value)
  730. except (TypeError, ValueError):
  731. logger.warning("ignoring non-numeric %s=%r in MoA preset", key, value)
  732. return None
  733. def _hash_messages(msgs: list[dict[str, Any]]) -> str:
  734. return hashlib.sha256("\u0000".join(f"{m.get('role')}:{m.get('content')}" for m in msgs).encode("utf-8", "replace")).hexdigest()
  735. def _is_failed_reference(text: str) -> bool:
  736. """Whether a reference output is a ``[failed: …]`` / ``[skipped: …]`` sentinel."""
  737. return text.lstrip().lower().startswith(("[failed:", "[skipped:"))
  738. def _join_reference_outputs(outputs: list[tuple[str, str, Any]], degraded: str = "") -> str:
  739. """Render numbered reference blocks for the aggregator, appending any degraded notice."""
  740. joined = "\n\n".join(f"Reference {idx} — {label}:\n{text}" for idx, (label, text, _acct) in enumerate(outputs, start=1))
  741. if degraded:
  742. joined = f"{joined}\n\n{degraded}" if joined else degraded
  743. return joined
  744. def _sum_reference_accounting(outputs: list[tuple[str, str, Any]]) -> tuple[Any, Any]:
  745. """Sum fan-out usage AND cost in dollars (each advisor priced at its OWN rate)."""
  746. usage = CanonicalUsage()
  747. cost: Any = None
  748. for _lbl, _txt, acct in outputs:
  749. if isinstance(acct, _RefAccounting):
  750. if isinstance(acct.usage, CanonicalUsage):
  751. usage = usage + acct.usage
  752. if acct.cost_usd is not None:
  753. cost = (cost or 0) + acct.cost_usd
  754. return usage, cost
  755. def _degraded_notice(failed_labels: list[str], policy: str) -> str:
  756. if not failed_labels or policy.strip().lower() == "silent":
  757. return ""
  758. return f"[Reference models unavailable: {', '.join(failed_labels)}]"
  759. def _slot_labels(slots: list[dict[str, Any]]) -> str:
  760. return ", ".join(_slot_label(slot) for slot in slots)
  761. def _guidance_inputs(
  762. reference_outputs: list[tuple[str, str, Any]], privacy_full: bool, policy: str,
  763. ) -> tuple[list[tuple[str, str, Any]], str, bool]:
  764. """``(advisor outputs for the aggregator, degraded notice, all_failed)``.
  765. 'full' privacy mode redacts advisor text reaching the aggregator (applied to a
  766. per-call copy — caches hold raw text). Failed refs are filtered out first.
  767. """
  768. successful = [o for o in reference_outputs if not _is_failed_reference(o[1])]
  769. failed_labels = [label for label, text, _acct in reference_outputs if _is_failed_reference(text)]
  770. agg_refs = _redact_reference_outputs(successful) if privacy_full else successful
  771. return agg_refs, _degraded_notice(failed_labels, policy), bool(reference_outputs) and not successful
  772. def aggregate_moa_context(
  773. *, user_prompt: str, api_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]],
  774. aggregator: dict[str, Any], temperature: float | None = None, aggregator_temperature: float | None = None,
  775. reference_max_tokens: int | None = None, reference_timeout: float | None = None,
  776. degraded_reference_policy: str = "loud", agent: Any = None,
  777. ) -> str:
  778. """Run configured reference models and synthesize their advice (one-shot /moa).
  779. Failures become model-specific notes instead of aborting the loop.
  780. ``reference_max_tokens`` caps ONLY the fan-out (capping the aggregator truncated
  781. long syntheses). ``agent`` makes the fan-out interruptible.
  782. ``reference_max_tokens`` applies ONLY to the reference fan-out — the aggregator's own synthesis call is
  783. not given an advisor budget. Omission uses provider-specific defaults; native protocols may
  784. still require an internal wire limit.
  785. A hardcoded cap on the aggregator call previously truncated long aggregator syntheses (#53580) — passing
  786. ``reference_max_tokens`` to both calls here would silently reintroduce that regression.
  787. """
  788. reference_models = [slot for slot in reference_models if slot.get("enabled", True)]
  789. reference_outputs = _run_references_parallel(
  790. reference_models, _reference_messages(api_messages), temperature=temperature,
  791. max_tokens=reference_max_tokens, reference_timeout=reference_timeout, agent=agent,
  792. )
  793. privacy_full = False
  794. try:
  795. from hermes_cli.config import load_config as _load_config
  796. privacy_full = _moa_privacy_mode((_load_config() or {}).get("moa")) == "full"
  797. except Exception: # pragma: no cover - privacy filter must never break a turn
  798. logger.debug("MoA privacy filter check failed", exc_info=True)
  799. agg_refs, degraded, all_failed = _guidance_inputs(reference_outputs, privacy_full, degraded_reference_policy)
  800. joined = _join_reference_outputs(agg_refs, degraded)
  801. if all_failed:
  802. # Skip the aggregator: synthesizing over nothing can block for the full provider timeout.
  803. logger.warning("MoA: all %d reference(s) failed — skipping aggregator synthesis", len(reference_outputs))
  804. return (
  805. "[Mixture of Agents context — all reference models failed. "
  806. "Proceeding without aggregated guidance.]\n"
  807. f"References: {_slot_labels(reference_models)}\n\n"
  808. f"{degraded or '[Reference models unavailable]'}"
  809. )
  810. synth_prompt = (
  811. "You are the aggregator in a Mixture of Agents process. Synthesize the "
  812. "reference responses into concise, actionable guidance for the main "
  813. "Hermes agent. Focus on next steps, tool-use strategy, risks, and any "
  814. "disagreements. Do not answer the user directly unless that is all that "
  815. "is needed; produce context the main agent should use in its normal loop.\n\n"
  816. f"Original user prompt:\n{user_prompt}\n\n"
  817. f"Reference responses:\n{joined}"
  818. )
  819. agg_label = _slot_label(aggregator)
  820. agg_runtime = _slot_runtime(aggregator)
  821. _touch_fanout_progress(
  822. agent, f"MoA: synthesizing aggregator guidance ({agg_label})"
  823. )
  824. try:
  825. # Undecorated: the prompt embeds this call's reference outputs, so no later call
  826. # reads it back and a marker would only pay a cache write (is_one_shot_request).
  827. synthesis = _extract_text(call_llm(
  828. task="moa_aggregator", messages=[{"role": "user", "content": synth_prompt}], temperature=aggregator_temperature,
  829. reasoning_config=_aggregator_reasoning_config(aggregator), **agg_runtime,
  830. ))
  831. except Exception as exc:
  832. logger.warning("MoA aggregator model %s failed: %s", agg_label, exc)
  833. synthesis = ""
  834. return (
  835. "[Mixture of Agents context — use this as private guidance for the "
  836. "normal Hermes agent loop. You may call tools, continue reasoning, or "
  837. "finish normally.]\n"
  838. f"Aggregator: {agg_label}\n"
  839. f"References: {_slot_labels(reference_models)}\n\n"
  840. f"{(synthesis or joined).strip()}"
  841. )
  842. def _completed_response_as_stream_chunk(response: Any) -> Any:
  843. """Adapt a completed response into one delta stream chunk (facade boundary only)."""
  844. choices = getattr(response, "choices", None)
  845. first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None
  846. message = getattr(first_choice, "message", None)
  847. raw_tool_calls = getattr(message, "tool_calls", None)
  848. tool_call_deltas = None
  849. if isinstance(raw_tool_calls, (list, tuple)) and raw_tool_calls:
  850. tool_call_deltas = [
  851. SimpleNamespace(
  852. index=getattr(tc, "index", index), id=getattr(tc, "id", None), type=getattr(tc, "type", None) or "function",
  853. function=SimpleNamespace(
  854. name=getattr(getattr(tc, "function", None), "name", None),
  855. arguments=getattr(getattr(tc, "function", None), "arguments", None),
  856. ),
  857. )
  858. for index, tc in enumerate(raw_tool_calls)
  859. ]
  860. delta = SimpleNamespace(
  861. content=getattr(message, "content", None), tool_calls=tool_call_deltas,
  862. **{k: getattr(message, k, None) for k in ("reasoning_content", "reasoning", "reasoning_details")},
  863. )
  864. choice = SimpleNamespace(
  865. index=getattr(first_choice, "index", 0), delta=delta, finish_reason=getattr(first_choice, "finish_reason", None) or "stop",
  866. )
  867. return SimpleNamespace(
  868. id=getattr(response, "id", None), model=getattr(response, "model", None), choices=[choice], usage=getattr(response, "usage", None),
  869. )
  870. def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str) -> None:
  871. """Attach the per-turn reference block as its OWN trailing user message.
  872. The block varies per turn; appending keeps ``[system][task][tool-history]``
  873. cache-stable. It is never merged into a trailing user turn: iteration 1 of a
  874. tool loop ends on ``user(task)``, and a merged ``user(task + guidance)`` byte-differs
  875. from the ``user(task)`` every later iteration replays, so the provider prefix cache
  876. collapsed to the system prompt on iteration 2 of every turn (#112358). Converters
  877. that require strict alternation (Anthropic Messages, Converse, native Gemini) merge
  878. the two user turns as SEPARATE content blocks, so the task block stays byte-stable
  879. there too; on the OpenAI-compatible wire the request ends ``user(task), user(guidance)``,
  880. which a chat template that enforces strict user/assistant alternation rejects.
  881. """
  882. agg_messages.append({"role": "user", "content": guidance})
  883. def peel_reference_guidance(messages: list[dict[str, Any]], guidance: Any) -> list[dict[str, Any]]:
  884. """Exact inverse of ``_attach_reference_guidance`` (plain string, or its cache-decorated
  885. single-text-part form), so a cache breakpoint never lands on the turn-varying guidance.
  886. Inputs are not mutated."""
  887. if not guidance or not messages:
  888. return messages
  889. guidance_text = str(guidance)
  890. last = messages[-1]
  891. if not isinstance(last, dict) or last.get("role") != "user":
  892. return messages
  893. content = last.get("content")
  894. if content == guidance_text:
  895. return list(messages[:-1])
  896. if isinstance(content, list) and len(content) == 1:
  897. part = content[0]
  898. if isinstance(part, dict) and part.get("type", "text") == "text" and (part.get("text") or "") == guidance_text:
  899. return list(messages[:-1])
  900. return messages
  901. class MoAChatCompletions:
  902. """OpenAI-chat-compatible facade where the aggregator is the acting model.
  903. ``reference_callback(event, **kwargs)`` is an optional best-effort display hook
  904. (events: ``moa.reference``, ``moa.progress``, ``moa.phase``, ``moa.aggregating``;
  905. kwargs per ``_RELAY_EVENTS``). ``agent`` is the owning AIAgent; it lets the
  906. fan-out check ``_interrupt_requested``.
  907. """
  908. def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None):
  909. self.preset_name = preset_name or "default"
  910. self.reference_callback = reference_callback
  911. self._agent = agent
  912. # Reference cache keyed on the advisory-view signature (HIT = no re-run, no re-emit).
  913. self._ref_cache_key: tuple | None = None
  914. self._ref_cache_outputs: list[tuple[str, str, Any]] = []
  915. # Fan-out spend awaiting consume_reference_usage (nothing deposited on a HIT so
  916. # spend counts once); the lock guards late-accounting callbacks on worker threads.
  917. self._pending_reference_usage: Any = CanonicalUsage()
  918. self._pending_reference_cost: Any = None
  919. self._accounting_lock = threading.Lock()
  920. # Real aggregator slot so cost accounting prices the acting turn at its model.
  921. self.last_aggregator_slot: Any = None
  922. # Full-turn trace parts from a cache-MISS create(), flushed by consume_and_save_trace.
  923. self._pending_trace: Any = None
  924. # Per-advisor metrics for observability hooks; NOT consumed (post_api_request
  925. # fires on a different branch than consume_and_save_trace).
  926. self._last_reference_metrics: Any = None
  927. # every_n cadence state, scoped to one USER TURN so iteration 1 is on-cadence.
  928. self._fanout_iteration_count = 0
  929. self._fanout_turn_sig: str | None = None
  930. self._fanout_last_state_sig: str | None = None
  931. self._privacy_mode: str = "" # normalized moa.privacy_filter, refreshed per create()
  932. # Destinations (route, model) that 400'd on adjacent same-role messages this session:
  933. # their aggregator requests are pre-merged; every other destination keeps the split,
  934. # cache-stable shape (agent/moa_alternation.py).
  935. self._merge_same_role_destinations: set[tuple[str, str]] = set()
  936. def consume_reference_usage(self) -> tuple[Any, Any]:
  937. """Pop pending fan-out ``(CanonicalUsage, cost_usd_or_None)`` and reset both
  938. (so a streaming retry re-entering accounting cannot double-count)."""
  939. with self._accounting_lock:
  940. usage = self._pending_reference_usage or CanonicalUsage()
  941. cost = self._pending_reference_cost
  942. self._pending_reference_usage = CanonicalUsage()
  943. self._pending_reference_cost = None
  944. return usage, cost
  945. def last_reference_metrics(self) -> Any:
  946. """Per-advisor metrics from the most recent fan-out, or None (read-only)."""
  947. return self._last_reference_metrics
  948. def _record_late_reference_accounting(self, label: str, accounting: Any) -> None:
  949. """Done-callback for abandoned (still billing) futures: fold their real spend in."""
  950. if not isinstance(accounting, _RefAccounting):
  951. return
  952. self._fold_pending_accounting(*_sum_reference_accounting([(label, "", accounting)]))
  953. logger.debug("MoA: recorded late accounting for interrupted reference %s", label)
  954. def _fold_pending_accounting(self, usage: Any, cost: Any) -> None:
  955. """Add (never overwrite) fan-out spend so late interrupted-reference deposits survive."""
  956. with self._accounting_lock:
  957. self._pending_reference_usage = (self._pending_reference_usage or CanonicalUsage()) + usage
  958. if cost is not None:
  959. self._pending_reference_cost = (self._pending_reference_cost or 0) + cost
  960. def consume_and_save_trace(self, session_id: Any = None, aggregator_output_fallback: Any = None) -> None:
  961. """Flush the pending full-turn trace to disk (no-op when nothing is pending).
  962. ``aggregator_output_fallback`` is the caller's resolved acting text for the
  963. streaming path (not capturable at ``create()`` time). Never raises.
  964. """
  965. pending = self._pending_trace
  966. self._pending_trace = None
  967. if not pending or "aggregator_input_messages" not in pending:
  968. return
  969. try:
  970. from agent.moa_trace import save_moa_turn
  971. agg_slot = pending.get("aggregator_slot") or {}
  972. # Inline capture (non-streaming) beats the caller's streamed text.
  973. agg_output = pending.get("aggregator_output")
  974. if agg_output is None and aggregator_output_fallback:
  975. agg_output = aggregator_output_fallback
  976. save_moa_turn(
  977. session_id=session_id, preset_name=pending.get("preset", ""),
  978. reference_outputs=pending.get("reference_outputs", []),
  979. aggregator_label=pending.get("aggregator_label", ""), aggregator_model=agg_slot.get("model"),
  980. aggregator_provider=agg_slot.get("provider"), aggregator_temperature=pending.get("aggregator_temperature"),
  981. aggregator_input_messages=pending.get("aggregator_input_messages"),
  982. aggregator_output=agg_output, aggregator_streamed=bool(pending.get("aggregator_streamed")),
  983. )
  984. except Exception as exc: # pragma: no cover - tracing must never break a turn
  985. logger.debug("MoA trace flush failed: %s", exc)
  986. def _emit(self, event: str, **kwargs: Any) -> None:
  987. if self.reference_callback is None:
  988. return
  989. try:
  990. self.reference_callback(event, **kwargs)
  991. except Exception as exc: # pragma: no cover - display must never break the turn
  992. logger.debug("MoA reference_callback failed for %s: %s", event, exc)
  993. def prepare(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
  994. """Run the advisor fan-out and return the exact aggregator request, which the
  995. loop measures before its compression gate and hands back to ``create()``."""
  996. return self.create(messages=messages, _moa_prepare_only=True)
  997. def rebase_prepared_request(self, prepared: dict[str, Any], messages: list[dict[str, Any]]) -> dict[str, Any]:
  998. """Re-attach already-generated guidance to a rebuilt (compressed) transcript."""
  999. guidance = prepared.get("guidance")
  1000. agg_messages = [dict(message) for message in messages]
  1001. if guidance:
  1002. _attach_reference_guidance(agg_messages, str(guidance))
  1003. return {**prepared, "messages": agg_messages}
  1004. def _plan_aggregator_cache(
  1005. self, agg_messages: list[dict[str, Any]], tools: Any, guidance: Any, agg_runtime: dict[str, Any],
  1006. ) -> tuple[list[dict[str, Any]], Any]:
  1007. """Cache-breakpoint the aggregator request for its destination.
  1008. Guidance is peeled before planning and re-attached after so a breakpoint never
  1009. lands on the turn-varying block. Any error → undecorated request (warning, not
  1010. debug: this is the aggregator's ONLY decoration path).
  1011. """
  1012. try:
  1013. from agent.agent_runtime_helpers import plan_cache_sections_for_destination
  1014. planning_messages = peel_reference_guidance(agg_messages, str(guidance)) if guidance else agg_messages
  1015. # Tri-state cache_disabled: facades built via __new__ have no _agent; forcing
  1016. # False would suppress the planner's config fallback.
  1017. # plan_cache_sections_for_destination never mutates its inputs and always returns request-local
  1018. # copies, so the prepared state stays canonical. Tri-state: only pass a bool when a live agent
  1019. # snapshot exists. See #76085.
  1020. _agent = getattr(self, "_agent", None)
  1021. cache_disabled, cache_ttl = _agent_cache_opts(_agent)
  1022. # Agent TTL + stable system prefix so MoA does not regress 1h → 5m.
  1023. agg_messages, tools = plan_cache_sections_for_destination(
  1024. planning_messages, tools, provider=agg_runtime.get("provider") or "",
  1025. base_url=agg_runtime.get("base_url") or "", api_mode=agg_runtime.get("api_mode") or "",
  1026. model=agg_runtime.get("model") or "", cache_disabled=cache_disabled, cache_ttl=cache_ttl,
  1027. static_system_prefix=getattr(_agent, "_cached_system_prompt_static", None),
  1028. )
  1029. if guidance:
  1030. _attach_reference_guidance(agg_messages, str(guidance))
  1031. except Exception as exc: # pragma: no cover - cache planning must not block MoA
  1032. logger.warning(
  1033. "MoA aggregator cache plan failed — sending undecorated "
  1034. "request (cache misses expected): %s", exc,
  1035. )
  1036. return agg_messages, tools
  1037. def _call_prepared_aggregator(self, prepared: dict[str, Any], api_kwargs: dict[str, Any]) -> Any:
  1038. """Send an already prepared MoA aggregator request exactly once."""
  1039. aggregator = prepared["aggregator"]
  1040. if aggregator.get("provider") == "moa":
  1041. raise RuntimeError("MoA aggregator cannot be another MoA preset")
  1042. agg_runtime = _slot_runtime(aggregator)
  1043. agg_messages, tools = self._plan_aggregator_cache(
  1044. prepared["messages"], api_kwargs.get("tools"), prepared.get("guidance"), agg_runtime
  1045. )
  1046. trace = self._pending_trace
  1047. if trace is not None:
  1048. trace["aggregator_label"] = _slot_label(aggregator)
  1049. # stream=True returns the RAW token stream (consumer reassembles + retries);
  1050. # the non-streaming path forwards no stream/stream_options/timeout. The
  1051. # consumer's stream-read timeout must govern the aggregator stream.
  1052. stream = bool(api_kwargs.get("stream"))
  1053. stream_kwargs: dict[str, Any] = {}
  1054. if stream:
  1055. stream_kwargs = {"stream": True, "stream_options": api_kwargs.get("stream_options") or {"include_usage": True}}
  1056. if api_kwargs.get("timeout") is not None:
  1057. stream_kwargs["timeout"] = api_kwargs["timeout"]
  1058. # Pop the runtime's extra_body so the explicit kwarg never collides with **agg_runtime.
  1059. agg_extra_body = _merge_slot_extra_body(agg_runtime.pop("extra_body", None), api_kwargs.get("extra_body"))
  1060. destination = destination_key(agg_runtime)
  1061. # Facades built via __new__ (tests, swaps) have no __init__ state.
  1062. remembered = getattr(self, "_merge_same_role_destinations", None)
  1063. if remembered is None:
  1064. remembered = self._merge_same_role_destinations = set()
  1065. merged = destination in remembered
  1066. if merged:
  1067. agg_messages = merge_same_role_messages(agg_messages)
  1068. send = functools.partial(
  1069. call_llm, task="moa_aggregator", temperature=prepared["aggregator_temperature"],
  1070. max_tokens=api_kwargs.get("max_tokens"), tools=tools, extra_body=agg_extra_body,
  1071. reasoning_config=_aggregator_reasoning_config(aggregator), # same policy as direct create()
  1072. **stream_kwargs, **agg_runtime,
  1073. )
  1074. try:
  1075. agg_response = send(messages=agg_messages)
  1076. except Exception as exc:
  1077. # Strict-alternation template rejected ``user(task), user(guidance)``: merge the pair for
  1078. # THIS destination only and retry once; remember it so later iterations pre-merge.
  1079. retry_messages = None if merged else merge_same_role_messages(agg_messages)
  1080. if retry_messages is None or retry_messages is agg_messages or not is_role_alternation_rejection(exc, agg_runtime):
  1081. raise
  1082. remembered.add(destination)
  1083. logger.warning(
  1084. "MoA aggregator %s rejected adjacent same-role messages — merging them for this "
  1085. "destination for the rest of the session and retrying once: %.200s", _slot_label(aggregator), exc,
  1086. )
  1087. agg_messages = retry_messages
  1088. agg_response = send(messages=agg_messages)
  1089. if trace is not None:
  1090. # Trace the exact aggregator INPUT as sent (persisted copy redacted; live input raw).
  1091. trace["aggregator_input_messages"] = (
  1092. _redact_trace_messages([dict(m) for m in agg_messages]) if getattr(self, "_privacy_mode", "") else agg_messages
  1093. )
  1094. # Streaming output lands as the turn's assistant message; the trace marks it.
  1095. trace["aggregator_streamed"] = stream
  1096. output = None
  1097. if not stream:
  1098. with contextlib.suppress(Exception): # pragma: no cover - defensive
  1099. output = _extract_text(agg_response)
  1100. trace["aggregator_output"] = output
  1101. if stream and hasattr(agg_response, "choices"):
  1102. # Some adapters (openai-codex Responses) return a completed response even
  1103. # when streaming was requested; hand the loop a one-chunk iterator.
  1104. return iter((_completed_response_as_stream_chunk(agg_response),))
  1105. return agg_response
  1106. def _fanout_cache_key(
  1107. self, preset: dict[str, Any], ref_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]],
  1108. ) -> tuple:
  1109. """Turn-scoped reference cache key per the preset's fan-out cadence.
  1110. "user_turn" (default) hashes only the prefix up to the LAST USER message, so
  1111. later tool iterations are HITs. "per_iteration" re-runs whenever the advisory
  1112. view changes. "every_n:<N>": iteration 1 of a turn, then every Nth; in-between
  1113. iterations return the pinned last on-cadence key (HIT: no calls, no re-emit).
  1114. """
  1115. # "user_turn" (default — cheapest cadence, #67199): advisors run ONCE per user turn; subsequent tool
  1116. # iterations reuse that turn's advice and the aggregator acts alone (the original MoA shape:
  1117. # synthesize at the start, then let the acting model work). Implemented by hashing only the prefix
  1118. # up to the LAST USER message so mid-turn growth doesn't change the signature — iteration 2+ becomes
  1119. # a cache HIT. "per_iteration": advisors re-run whenever the advisory view changes — i.e. every tool
  1120. # iteration, since the view grows with each tool result; advice tracks live task state at the cost
  1121. # of multiplying advisor latency/spend by tool-loop depth. "every_n:<N>" (N >= 2): the middle ground
  1122. # (issue #63393 — advisor fan-out multiplies latency/cost by the tool-iteration count). Advisors run
  1123. # on iteration 1 of a user turn and then every Nth tool iteration; the iterations in between REUSE
  1124. # the cached guidance from the last on-cadence run (same mechanism as user_turn's cache HIT — the
  1125. # aggregator still gets advice every iteration, it's just not refreshed against the very latest tool
  1126. # results). The iteration counter is scoped per user turn and resets on a new user message, so every
  1127. # turn starts with fresh advice.
  1128. fanout_mode = str(preset.get("fanout") or "user_turn").strip().lower()
  1129. every_n = 0
  1130. if fanout_mode.startswith("every_n:"):
  1131. with contextlib.suppress(TypeError, ValueError):
  1132. every_n = int(fanout_mode.split(":", 1)[1])
  1133. if every_n < 2:
  1134. fanout_mode = "per_iteration" # every_n:1 IS per-iteration (mirrors _coerce_fanout)
  1135. sig_messages = turn_prefix = ref_messages
  1136. if fanout_mode == "user_turn" or every_n >= 2:
  1137. # Last REAL user message: the synthetic _ADVISORY_INSTRUCTION marker must not
  1138. # count or the prefix would grow (and re-sign) every iteration.
  1139. last_user = next(
  1140. (i for i in range(len(ref_messages) - 1, -1, -1)
  1141. if ref_messages[i].get("role") == "user" and ref_messages[i].get("content") != _ADVISORY_INSTRUCTION),
  1142. None,
  1143. )
  1144. if last_user is not None:
  1145. turn_prefix = ref_messages[: last_user + 1]
  1146. if fanout_mode == "user_turn":
  1147. sig_messages = turn_prefix
  1148. # every_n bookkeeping: advance the counter only when the advisory STATE changed
  1149. # (a streaming retry must not consume a cadence slot); reset on a new turn prefix.
  1150. if every_n >= 2:
  1151. turn_sig = _hash_messages(turn_prefix)
  1152. if turn_sig != self._fanout_turn_sig:
  1153. self._fanout_turn_sig = turn_sig
  1154. self._fanout_iteration_count = 0
  1155. self._fanout_last_state_sig = None
  1156. state_sig = _hash_messages(ref_messages)
  1157. if state_sig != self._fanout_last_state_sig:
  1158. self._fanout_last_state_sig = state_sig
  1159. self._fanout_iteration_count += 1
  1160. # Iteration 1 is on-cadence; then every Nth iteration after it.
  1161. if (self._fanout_iteration_count - 1) % every_n != 0 and self._ref_cache_outputs:
  1162. return self._ref_cache_key
  1163. return (self.preset_name, _hash_messages(sig_messages), tuple(_slot_label(s) for s in reference_models))
  1164. def _run_fanout(
  1165. self, preset: dict[str, Any], ref_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]],
  1166. aggregator: dict[str, Any], aggregator_temperature: Any, cache_key: tuple,
  1167. ) -> list[tuple[str, str, Any]]:
  1168. """Cache-MISS path of ``create``: run the advisors, account, trace and emit.
  1169. A preset MAY cap ADVISOR output (dominant MoA latency); the acting aggregator
  1170. is never capped. None timeout = inherit auxiliary.moa_reference.timeout.
  1171. """
  1172. raw_reference_timeout = preset.get("reference_timeout")
  1173. reference_outputs = _run_references_parallel(
  1174. reference_models, ref_messages, temperature=_preset_temperature(preset, "reference_temperature"),
  1175. progress_callback=lambda done, total, label: self._emit("moa.progress", refs_done=done, refs_total=total, label=label),
  1176. reference_timeout=float(raw_reference_timeout) if raw_reference_timeout else None,
  1177. agent=self._agent, late_accounting_sink=self._record_late_reference_accounting,
  1178. )
  1179. # An interrupted fan-out is a partial snapshot: never cache it (a HIT would
  1180. # replay placeholder notes every iteration).
  1181. interrupted = any(text == _INTERRUPTED_REFERENCE_NOTE for _lbl, text, _acct in reference_outputs)
  1182. self._ref_cache_key = None if interrupted else cache_key
  1183. self._ref_cache_outputs = [] if interrupted else list(reference_outputs)
  1184. # Fold advisor spend into accounting exactly once per turn.
  1185. self._fold_pending_accounting(*_sum_reference_accounting(reference_outputs))
  1186. # Stash the fan-out for trace persistence (aggregator parts filled in later).
  1187. # Traces are persisted, so ANY active privacy mode redacts them.
  1188. privacy_mode = self._privacy_mode
  1189. trace_refs = [
  1190. (label, _redact_reference_text(text), _redact_trace_accounting(acct)) for label, text, acct in reference_outputs
  1191. ] if privacy_mode else list(reference_outputs)
  1192. self._pending_trace = {
  1193. "preset": self.preset_name, "reference_outputs": trace_refs,
  1194. "aggregator_slot": aggregator, "aggregator_temperature": aggregator_temperature,
  1195. }
  1196. # Derived from the privacy-redacted trace_refs.
  1197. try:
  1198. from agent.moa_trace import slot_metrics
  1199. self._last_reference_metrics = [slot_metrics(acct, label, output=text) for label, text, acct in trace_refs]
  1200. except Exception as exc: # pragma: no cover - never break a turn
  1201. logger.debug("MoA reference metrics render failed: %s", exc)
  1202. self._last_reference_metrics = None
  1203. # Surface each answer BEFORE the aggregator acts; the cache keeps RAW text.
  1204. ref_count = len(reference_outputs)
  1205. for idx, (label, text, _accounting) in enumerate(reference_outputs, start=1):
  1206. self._emit(
  1207. "moa.reference", index=idx, count=ref_count, label=label,
  1208. text=_redact_reference_text(text) if privacy_mode else text,
  1209. )
  1210. if ref_count:
  1211. # Phase transition: fan-out complete, aggregator about to act.
  1212. agg_label = _slot_label(aggregator)
  1213. self._emit("moa.phase", phase="aggregator", refs_done=ref_count, refs_total=ref_count, aggregator=agg_label)
  1214. self._emit("moa.aggregating", aggregator=agg_label, ref_count=ref_count)
  1215. return reference_outputs
  1216. def _build_guidance(
  1217. self, reference_outputs: list[tuple[str, str, Any]], aggregator: dict[str, Any], degraded_reference_policy: str,
  1218. stale: bool = False,
  1219. ) -> str | None:
  1220. """Render the reference block attached to the aggregator prompt (None = nothing)."""
  1221. agg_refs, degraded, all_failed = _guidance_inputs(
  1222. reference_outputs, self._privacy_mode == "full", degraded_reference_policy
  1223. )
  1224. header = (
  1225. "[Mixture of Agents reference context]\n"
  1226. f"Preset: {self.preset_name}\n"
  1227. f"Aggregator/acting model: {_slot_label(aggregator)}\n"
  1228. )
  1229. if all_failed:
  1230. # Every reference failed: the aggregator acts alone (loud policy → notice).
  1231. logger.warning(
  1232. "MoA: all %d reference(s) failed — acting aggregator-alone "
  1233. "without reference guidance",
  1234. len(reference_outputs),
  1235. )
  1236. if degraded:
  1237. return (
  1238. f"{header}\n"
  1239. "All reference models failed this turn — no advisory "
  1240. "guidance is available. Act on your own judgment.\n\n"
  1241. f"{degraded}"
  1242. )
  1243. return None
  1244. if agg_refs or degraded:
  1245. return (
  1246. f"{header}"
  1247. f"References: {', '.join(label for label, _, _ in agg_refs)}\n\n"
  1248. "Use the reference responses below as private context. You are the aggregator and acting model: "
  1249. "answer the user directly or call tools as needed.\n"
  1250. f"{_STALE_GUIDANCE_NOTE if stale else ''}\n"
  1251. f"{_join_reference_outputs(agg_refs, degraded)}"
  1252. )
  1253. return None
  1254. def create(self, **api_kwargs: Any) -> Any:
  1255. prepared_request = api_kwargs.pop("_moa_prepared_request", None)
  1256. if prepared_request is not None:
  1257. if not isinstance(prepared_request, dict):
  1258. raise TypeError("_moa_prepared_request must be a dict")
  1259. return self._call_prepared_aggregator(prepared_request, api_kwargs)
  1260. preset, moa_raw = _resolve_preset_cached(self.preset_name)
  1261. # Remembered on self so _call_prepared_aggregator redacts the trace consistently.
  1262. self._privacy_mode = _moa_privacy_mode(moa_raw)
  1263. messages = list(api_kwargs.get("messages") or [])
  1264. # A disabled preset = "use the aggregator directly".
  1265. reference_models = [
  1266. slot for slot in (preset.get("reference_models") or []) if slot.get("enabled", True)
  1267. ] if preset.get("enabled", True) else []
  1268. aggregator = preset.get("aggregator") or {}
  1269. # The MoA path's virtual model/provider have no pricing entry; expose the real slot.
  1270. self.last_aggregator_slot = dict(aggregator) if aggregator else None
  1271. # None = provider default (see _preset_temperature); the acting agent's own
  1272. # temperature applies to the aggregator (the acting model).
  1273. aggregator_temperature = _preset_temperature(preset, "aggregator_temperature")
  1274. if aggregator_temperature is None and api_kwargs.get("temperature") is not None:
  1275. aggregator_temperature = api_kwargs.get("temperature")
  1276. ref_messages = _reference_messages(messages)
  1277. cache_key = self._fanout_cache_key(preset, ref_messages, reference_models)
  1278. cache_hit = bool(cache_key == self._ref_cache_key and self._ref_cache_outputs)
  1279. if cache_hit:
  1280. # HIT: already ran and accounted. Do NOT zero pending totals (a late
  1281. # interrupted reference may have deposited) and no trace (not a new turn).
  1282. reference_outputs = list(self._ref_cache_outputs)
  1283. self._pending_trace = None
  1284. else:
  1285. reference_outputs = self._run_fanout(preset, ref_messages, reference_models, aggregator, aggregator_temperature, cache_key)
  1286. agg_messages = [dict(m) for m in messages]
  1287. guidance = self._build_guidance(
  1288. reference_outputs, aggregator, str(preset.get("degraded_reference_policy") or "loud"),
  1289. stale=cache_hit and _tool_activity_since_last_user(messages),
  1290. )
  1291. if guidance:
  1292. _attach_reference_guidance(agg_messages, guidance)
  1293. prepared_request = {
  1294. "messages": agg_messages, "guidance": guidance, "aggregator": aggregator,
  1295. "aggregator_temperature": aggregator_temperature,
  1296. }
  1297. if api_kwargs.pop("_moa_prepare_only", False):
  1298. return prepared_request
  1299. return self._call_prepared_aggregator(prepared_request, api_kwargs)
  1300. class MoAClient:
  1301. """OpenAI-client-shaped wrapper: ``client.chat.completions`` is a ``MoAChatCompletions``;
  1302. the accounting/trace surface below is delegated to that facade."""
  1303. def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None):
  1304. self.chat = type("_MoAChat", (), {})()
  1305. self.chat.completions = MoAChatCompletions(preset_name, reference_callback=reference_callback, agent=agent)
  1306. def consume_reference_usage(self) -> Any:
  1307. return self.chat.completions.consume_reference_usage()
  1308. @property
  1309. def last_aggregator_slot(self) -> Any:
  1310. return getattr(self.chat.completions, "last_aggregator_slot", None)
  1311. def consume_and_save_trace(self, session_id: Any = None, aggregator_output_fallback: Any = None) -> None:
  1312. return self.chat.completions.consume_and_save_trace(session_id, aggregator_output_fallback=aggregator_output_fallback)
  1313. def last_reference_metrics(self) -> Any:
  1314. return self.chat.completions.last_reference_metrics()
  1315. # Relay table: event -> (primary kwarg, secondary kwarg or None, {cb kwarg: emit kwarg}).
  1316. # The callback signature is ``cb(event, label, text, None, **moa_*)``.
  1317. _RELAY_EVENTS: dict[str, tuple[str, str | None, dict[str, str]]] = {
  1318. "moa.reference": ("label", "text", {"moa_index": "index", "moa_count": "count"}),
  1319. "moa.progress": ("label", None, {"moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}),
  1320. "moa.phase": ("aggregator", None, {"moa_phase": "phase", "moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}),
  1321. "moa.aggregating": ("aggregator", None, {"moa_ref_count": "ref_count"}),
  1322. }
  1323. def build_moa_facade(agent, preset_name: Any = None) -> MoAClient:
  1324. """Single construction point for ``MoAClient``: a bare ``MoAClient(preset)`` would
  1325. drop the ``reference_callback`` relay and silence display events for the session.
  1326. The relay reads ``agent.tool_progress_callback`` at emit time."""
  1327. def _moa_reference_relay(event: str, **kwargs: Any) -> None:
  1328. cb = getattr(agent, "tool_progress_callback", None)
  1329. spec = _RELAY_EVENTS.get(event)
  1330. if cb is None or spec is None:
  1331. return
  1332. primary, secondary, extra_map = spec
  1333. with contextlib.suppress(Exception):
  1334. cb(
  1335. event, str(kwargs.get(primary) or ""), str(kwargs.get(secondary) or "") if secondary else None, None,
  1336. **{out: kwargs.get(src) for out, src in extra_map.items()},
  1337. )
  1338. resolved_preset = preset_name
  1339. if resolved_preset is None and getattr(agent, "provider", None) == "moa":
  1340. resolved_preset = getattr(agent, "model", None)
  1341. resolved_preset = str(resolved_preset or "default")
  1342. try:
  1343. from hermes_cli.config import load_config
  1344. from hermes_cli.moa_config import normalize_moa_config
  1345. moa_cfg = normalize_moa_config(load_config().get("moa") or {})
  1346. if resolved_preset not in (moa_cfg.get("presets") or {}):
  1347. resolved_preset = moa_cfg.get("default_preset") or "default"
  1348. except Exception:
  1349. resolved_preset = "default"
  1350. # ``agent`` lets the fan-out wait be aborted on a user interrupt.
  1351. return MoAClient(resolved_preset, reference_callback=_moa_reference_relay, agent=agent)
  1352. def bind_moa_runtime(agent, preset_name: Any, api_key: Any = None) -> None:
  1353. """Make ``agent`` act as the MoA preset: pin the virtual runtime fields and install the facade.
  1354. Every site that puts an agent onto ``provider: moa`` (init, ``/model`` switch, fallback
  1355. activation) must pin the same fields — the facade speaks only chat.completions, has no HTTP
  1356. endpoint and no OpenAI client kwargs — or the next dispatch/rebuild reaches a real wire with a
  1357. virtual identity (``moa://local`` 404, or the preset name sent as a model id).
  1358. """
  1359. agent.model = str(preset_name or "default")
  1360. agent.provider = agent.requested_provider = "moa"
  1361. agent.api_mode = "chat_completions"
  1362. agent.api_key = api_key or "moa-virtual-provider"
  1363. agent.base_url = "moa://local"
  1364. agent._client_kwargs = {}
  1365. agent.client = build_moa_facade(agent, agent.model)