chat_completions.py 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804
  1. """OpenAI Chat Completions transport (default api_mode for OpenAI-compatible providers).
  2. Messages/tools are already OpenAI-shaped, so convert_* are near-identity; the
  3. provider-specific work lives in build_kwargs (max_tokens, reasoning, extra_body).
  4. """
  5. import hashlib
  6. import json
  7. import re
  8. import string
  9. from typing import Any
  10. from urllib.parse import urlparse
  11. from agent.reasoning_effort import (
  12. GEMINI4_THINKING_LEVELS, KIMI_K3_EFFORTS, KIMI_K3_OVERRIDES, OPENAI_COMPAT_WIRE_EFFORTS, clamp_effort,
  13. clamp_reasoning_config, kimi_supported_efforts, requested_effort, tokenhub_effort,
  14. )
  15. from agent.message_metadata import DATA_USE, MESSAGE_UID, data_use_requirement
  16. from agent.message_sanitization import coalesce_tool_call_id, normalize_finish_reason as _normalize_finish_reason
  17. from agent.moonshot_schema import is_moonshot_model, sanitize_moonshot_tools
  18. from agent.prompt_builder import DEVELOPER_ROLE_MODELS
  19. from agent.transports.base import ProviderTransport
  20. from agent.transports.types import NormalizedResponse, ToolCall, Usage
  21. # xAI reserves ``tool_search`` for its server-side tool (HTTP 400 on client
  22. # declarations); aliased on the wire, mapped back in normalize_response.
  23. # xAI's chat-completions API reserves the function name ``tool_search`` for its own server-side tool and
  24. # rejects any request declaring a client function with that name (HTTP 400 "The function name tool_search is
  25. # reserved for the tool_search tool", #95003). The Tool Search bridge (tools/tool_search.py) assembles its
  26. # client-side discovery tool under the same literal name for every provider, so Grok providers are unusable
  27. # whenever the bridge is active. Mirror the web_search treatment in transports/codex.py
  28. # (_rename_client_web_search_for_xai): alias the wire declaration and map the alias back in
  29. # normalize_response. The alias value matches _CODEX_TOOL_SEARCH_ALIAS from the Codex-side fix for the same
  30. # reserved-name class (#83122) so the two transports stay consistent.
  31. _XAI_TOOL_SEARCH_ALIAS = "hermes_tool_search"
  32. # Persistence-only / cross-transport message keys that strict OpenAI-compatible
  33. # providers reject with HTTP 400 ("Extra inputs are not permitted").
  34. _STRIP_MSG_KEYS = (
  35. "codex_reasoning_items", "codex_message_items", "tool_name", "effect_disposition", "timestamp",
  36. # ``platform_message_id`` is persisted as-is and comes back from replay as ``message_id``
  37. # (JSONL transcript compat); both are delivery bookkeeping, never model input.
  38. "platform_message_id", "message_id", "api_content", "anthropic_content_blocks", "bedrock_content_blocks",
  39. MESSAGE_UID, DATA_USE,
  40. )
  41. _STRIP_TC_KEYS = ("call_id", "response_item_id")
  42. _HIGH_EFFORTS = {"high", "xhigh", "max", "ultra"}
  43. def _rename_tool_search_bridge_for_xai(tools: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, str]]:
  44. """Alias the client ``tool_search`` declaration for xAI; returns ``(tools, {alias: "tool_search"})``.
  45. If a real tool already holds ``hermes_tool_search``, the bridge takes a ``_2``/``_3`` suffix.
  46. """
  47. from agent.transports.codex import _alias_reserved_tools
  48. return _alias_reserved_tools(
  49. tools, ("tool_search",), name_of=lambda t: (t.get("function") or {}).get("name"),
  50. rename=lambda t, alias: {**t, "function": {**t["function"], "name": alias}},
  51. )
  52. def _static_prompt_instructions(messages: list[dict[str, Any]]) -> str:
  53. """Stable leading system/developer prefix used for cache routing (later messages are conversation state)."""
  54. first = messages[0] if messages and isinstance(messages[0], dict) else {}
  55. if first.get("role") not in {"system", "developer"}:
  56. return ""
  57. content = first.get("content")
  58. if isinstance(content, str):
  59. return content
  60. try:
  61. return json.dumps(content, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
  62. except (TypeError, ValueError):
  63. return str(content or "")
  64. def _add_prompt_cache_key(
  65. api_kwargs: dict[str, Any], *, messages: list[dict[str, Any]],
  66. tools: list[dict[str, Any]] | None, supports_prompt_cache_key: bool,
  67. session_id: str | None = None, cache_scope_id: str | None = None,
  68. ) -> None:
  69. """Add a content-addressed ``prompt_cache_key`` only for a capable endpoint.
  70. ``cache_scope_id`` (compression-lineage root) beats ``session_id`` so the key
  71. survives compression rotation. A caller-supplied key is authoritative but is
  72. bounded to OpenAI's 64-char cap in place. Shares the Responses transport's hash
  73. so equivalent prefixes hit one bucket across modes.
  74. ``cache_scope_id``, when provided, is the rotation-stable logical scope (compression-lineage root —
  75. agent/prompt_cache_scope.py) and takes precedence over the physical ``session_id`` so the key survives
  76. context-compression session rotation (#79017).
  77. """
  78. # Stable prompt-cache routing for the Codex/Responses aux path, mirroring the main transport
  79. # (agent/transports/codex.py::build_kwargs, which sets prompt_cache_key =
  80. # _content_cache_key(instructions, tools)). Without this, MoA acting-aggregator and other auxiliary
  81. # Responses calls stay cache-cold while the main Responses transport is warm (issue #53735). The key is
  82. # content-addressed from the static prefix (instructions + tool schemas) so it stays warm across
  83. # turns/fires. Guard the top-level field the same way the main transport does: xAI Responses takes the
  84. # key in extra_body (not top-level) and endpoints declaring ``responses_prompt_cache_key=False`` opt out
  85. # of cache-key routing entirely — for those hosts, skip it here.
  86. from agent.transports.codex import (
  87. _bound_prompt_cache_key_field, _cache_scope_from_session_id, _content_cache_key
  88. )
  89. containers = [c for c in (api_kwargs, api_kwargs.get("extra_body")) if isinstance(c, dict) and "prompt_cache_key" in c]
  90. if containers:
  91. for c in containers:
  92. _bound_prompt_cache_key_field(c)
  93. return
  94. if not supports_prompt_cache_key:
  95. return
  96. cache_key = _content_cache_key(
  97. _static_prompt_instructions(messages), tools, _cache_scope_from_session_id(cache_scope_id or session_id),
  98. )
  99. if cache_key:
  100. api_kwargs["prompt_cache_key"] = cache_key
  101. _ROUTER_TIMEOUT_SHIM = "Connect timeout, please try again later."
  102. def _has_positive_completion_tokens(usage: Any) -> bool:
  103. """Return whether a response usage object proves text was generated."""
  104. for field in ("completion_tokens", "output_tokens"):
  105. value = usage.get(field) if isinstance(usage, dict) else getattr(usage, field, None)
  106. if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0:
  107. return True
  108. return False
  109. def router_timeout_shim_may_follow(text: str) -> bool:
  110. """True while streamed text is still a prefix of the shim sentinel (hold it back until judged)."""
  111. return bool(text) and _ROUTER_TIMEOUT_SHIM.startswith(text.lstrip())
  112. def is_router_timeout_shim(response: Any) -> bool:
  113. """Recognize a router failure encoded as a successful ChatCompletion (#68396).
  114. Some OpenAI-compatible routers answer an upstream connect timeout with HTTP 200 and the
  115. sentinel as the sole assistant message. Only the exact sentinel, with no tool calls and no
  116. positive ``completion_tokens``/``output_tokens`` proof of generation, is a shim — a model
  117. that really produced those words keeps its usage evidence. Shared by every consumer of an
  118. OpenAI-compatible response: ``validate_response``, the stream assembler, the
  119. iteration-limit summary and the auxiliary ``_validate_llm_response``.
  120. """
  121. choices = getattr(response, "choices", None)
  122. if not isinstance(choices, list) or len(choices) != 1:
  123. return False
  124. message = getattr(choices[0], "message", None)
  125. content = getattr(message, "content", None)
  126. if not isinstance(content, str) or content.strip() != _ROUTER_TIMEOUT_SHIM:
  127. return False
  128. if getattr(message, "tool_calls", None):
  129. return False
  130. return not _has_positive_completion_tokens(getattr(response, "usage", None))
  131. def _reasoning_config_for_model(model: str, reasoning_config: dict | None) -> dict | None:
  132. """Clamp Hermes' extended effort set (``ultra``) to the OpenAI-compat wire vocabulary.
  133. Hermes' internal effort set extends the wire vocabulary with ``ultra`` (the /reasoning command documents
  134. none..xhigh|max|ultra). OpenAI- compatible wires — OpenRouter chief among them — accept exactly
  135. max|xhigh|high|medium|low|minimal|none and reject the extension with HTTP 400 (#89503). Clamp against
  136. the declared wire vocabulary via the shared policy in ``agent.reasoning_effort``; provider profiles with
  137. narrower sets clamp again downstream.
  138. """
  139. return clamp_reasoning_config(reasoning_config, OPENAI_COMPAT_WIRE_EFFORTS)
  140. def _build_gemini_thinking_config(model: str, reasoning_config: dict | None) -> dict | None:
  141. """Translate Hermes/OpenRouter-style reasoning config to Gemini thinkingConfig."""
  142. from agent.gemini_native_adapter import gemini_uses_thinking_level
  143. if not isinstance(reasoning_config, dict):
  144. return None
  145. normalized_model = (model or "").strip().lower().removeprefix("google/")
  146. # Gemini-only; Gemma/PaLM on the same provider 400 on the field even as ``{"includeThoughts": False}``.
  147. # ``thinking_config`` is a Gemini-only request parameter. The same ``gemini`` provider also serves Gemma
  148. # (and historically PaLM/Bard); those reject the field with HTTP 400 "Unknown name 'thinking_config':
  149. # Cannot find field" — including the polite ``{"includeThoughts": False}`` form. Omit the field entirely
  150. # on non-Gemini models. (#17426)
  151. if not normalized_model.startswith("gemini"):
  152. return None
  153. effort = str(reasoning_config.get("effort", "medium") or "medium").strip().lower()
  154. if normalized_model.startswith("gemini-4"):
  155. # Gemini 4 always thinks: a disable request gets the floor level with thoughts hidden.
  156. disabled = reasoning_config.get("enabled") is False or effort == "none"
  157. level = "low" if disabled else clamp_effort(effort, GEMINI4_THINKING_LEVELS)
  158. return {"includeThoughts": not disabled, "thinkingLevel": level if level in GEMINI4_THINKING_LEVELS else "medium"}
  159. if reasoning_config.get("enabled") is False or effort == "none":
  160. # ``includeThoughts: False`` only omits thought parts from the returned
  161. # response; the model may still reason internally and bill thought
  162. # tokens against maxOutputTokens, starving small budgets (title
  163. # generation's 64 tokens). Bound thinking as low as each family allows:
  164. # Gemini 2.5 documents ``thinkingBudget: 0`` as the off switch, while
  165. # Gemini 3+ deprecated that field in favour of the ``thinkingLevel``
  166. # string enum (which has no "none"), so use its lowest universally
  167. # supported level — "low" (3.6's "minimal" is not accepted by 3.7+ /
  168. # 3.1 Pro). ``_thinking_requests_output_headroom`` keeps the small
  169. # budget intact for this hidden-thoughts path. (#91927)
  170. config: dict[str, Any] = {"includeThoughts": False}
  171. if normalized_model.startswith("gemini-2.5-"):
  172. config["thinkingBudget"] = 0
  173. elif gemini_uses_thinking_level(normalized_model):
  174. config["thinkingLevel"] = "low"
  175. return config
  176. thinking_config: dict[str, Any] = {"includeThoughts": True}
  177. # Gemini 2.5 takes thinkingBudget; don't guess one from coarse effort levels.
  178. if normalized_model.startswith("gemini-2.5-"):
  179. return thinking_config
  180. if effort not in {"minimal", "low", "medium", "high", "xhigh", "max", "ultra"}:
  181. effort = "medium"
  182. # Gemini 3 Pro is strict (low/high); every other thinking-level model (3.x Flash, Gemini 4 Argon)
  183. # documents low/medium/high, and "minimal" is not universal (Argon 400s on it). Clamp Hermes' wider
  184. # effort set to what each family accepts so we never forward an undocumented level verbatim.
  185. if gemini_uses_thinking_level(normalized_model):
  186. if "pro" in normalized_model:
  187. thinking_config["thinkingLevel"] = "high" if effort in _HIGH_EFFORTS else "low"
  188. else:
  189. thinking_config["thinkingLevel"] = (
  190. "low" if effort in {"minimal", "low"} else "high" if effort in _HIGH_EFFORTS else "medium"
  191. )
  192. return thinking_config
  193. def _snake_case_gemini_thinking_config(config: dict | None) -> dict | None:
  194. """Convert Gemini thinking config keys to the OpenAI-compat field names."""
  195. if not isinstance(config, dict) or not config:
  196. return None
  197. translated: dict[str, Any] = {}
  198. include, level, budget = config.get("includeThoughts"), config.get("thinkingLevel"), config.get("thinkingBudget")
  199. if isinstance(include, bool):
  200. translated["include_thoughts"] = include
  201. if isinstance(level, str) and level.strip():
  202. translated["thinking_level"] = level.strip().lower()
  203. if isinstance(budget, (int, float)):
  204. translated["thinking_budget"] = int(budget)
  205. return translated or None
  206. def _raise_gemini_thinking_max_tokens(model: str, reasoning_config: dict | None, requested: Any) -> Any:
  207. """Raise Gemini output caps that thinking tokens (billed against max_tokens) would otherwise exhaust."""
  208. thinking_config = _build_gemini_thinking_config(model, reasoning_config)
  209. if not thinking_config:
  210. return requested
  211. from agent.gemini_native_adapter import _effective_gemini_max_output_tokens
  212. return _effective_gemini_max_output_tokens(requested, thinking_config)
  213. def _is_gemini_openai_compat_base_url(base_url: Any) -> bool:
  214. normalized = str(base_url or "").strip().rstrip("/").lower()
  215. return bool(normalized) and "generativelanguage.googleapis.com" in normalized and normalized.endswith("/openai")
  216. def _is_openai_api_base_url(base_url: Any) -> bool:
  217. """True only for the exact api.openai.com host (implies ``prompt_cache_key`` support).
  218. Not a substring match: Azure / strict compat endpoints stay opt-in via ``supports_prompt_cache_key``.
  219. """
  220. try:
  221. return (urlparse(str(base_url or "").strip()).hostname or "").lower() == "api.openai.com"
  222. except Exception:
  223. return False
  224. _GEMINI_SIGNATURE_HOSTS = ("generativelanguage.googleapis.com", "aiplatform.googleapis.com")
  225. _GEMINI_SIGNATURE_PROVIDERS = frozenset({"gemini", "vertex"})
  226. def _profile_replays_tool_call_signatures(provider: Any) -> bool:
  227. """``ProviderProfile.replays_tool_call_signatures`` for the provider named exactly (canonical name only)."""
  228. name = (provider or "").strip().lower()
  229. if not name:
  230. return False
  231. from providers import get_provider_profile
  232. profile = get_provider_profile(name)
  233. return profile is not None and profile.name == name and profile.replays_tool_call_signatures
  234. def _model_consumes_thought_signature(model: Any, base_url: Any = None, provider: Any = None) -> bool:
  235. """True for Gemini routes, which require tool-call ``extra_content`` (thought_signature) replay.
  236. A Gemini host or provider counts even when the model id is an alias without "gemini" in it;
  237. every other strict provider (Fireworks, Cerebras) rejects the key, so it is stripped there.
  238. """
  239. from utils import base_url_host_matches
  240. m = str(model or "").lower()
  241. return (
  242. "gemini" in m or "gemma" in m
  243. or (provider or "").strip().lower() in _GEMINI_SIGNATURE_PROVIDERS
  244. or _profile_replays_tool_call_signatures(provider)
  245. or any(base_url_host_matches(base_url, host) for host in _GEMINI_SIGNATURE_HOSTS)
  246. )
  247. def _wire_reasoning_carriers(provider: Any, base_url: Any) -> frozenset:
  248. """Reasoning keys an assistant message may carry on this route; the rest are stripped.
  249. ``message_sanitization.route_reasoning_carriers`` owns the table; keys the route rejected
  250. this session were already removed upstream by ``reapply_reasoning_echo``. The table replays
  251. ``reasoning_details`` to OpenRouter-format gateways (OpenRouter, Kilo, Vercel AI Gateway,
  252. Nous Portal) and to vendors documenting it (Novita, MiniMax /v1, TokenHub), never to strict
  253. schemas (Groq, Mistral, Cerebras, opencode relays: ``property 'reasoning_details' is
  254. unsupported`` / ``Extra inputs are not permitted`` / ``no such field``, #70233).
  255. The Portal is back on the list: the "cumulative replayed-reasoning budget" blamed for #118182
  256. does not exist in the Portal code and ~440 live replay requests (up to 32 KB of reasoning)
  257. returned no 400; the observed failures were an upstream content filter surfaced as
  258. "Provider returned error". Portal lanes that reject a carrier are handled by the
  259. per-(provider, host, model) reactive strip. Stored history always keeps every carrier.
  260. """
  261. from agent.message_sanitization import route_reasoning_carriers
  262. return route_reasoning_carriers(provider, base_url)
  263. def _has_replayable_thought_signature(extra_content: Any) -> bool:
  264. """Whether OpenRouter's Gemini sidecar contains a usable thought signature.
  265. Gemini accepts the signature either directly or under its ``google``
  266. namespace. Replaying an empty or non-string value makes a multimodal
  267. request fail with ``Corrupted thought signature``; omit that sidecar while
  268. leaving the stored history untouched.
  269. """
  270. if not isinstance(extra_content, dict):
  271. return False
  272. candidate = extra_content.get("thought_signature")
  273. google = extra_content.get("google")
  274. if candidate is None and isinstance(google, dict):
  275. candidate = google.get("thought_signature")
  276. return isinstance(candidate, str) and bool(candidate.strip())
  277. def _attr_or_model_extra(obj: Any, name: str) -> Any:
  278. """``obj.<name>``, else the same key from pydantic ``model_extra`` (some SDKs park fields there)."""
  279. value = getattr(obj, name, None)
  280. if value is None and hasattr(obj, "model_extra"):
  281. value = (obj.model_extra if isinstance(obj.model_extra, dict) else {}).get(name)
  282. return value
  283. def _dump_extra_content(extra: Any) -> Any:
  284. """Plain-dict form of a pydantic ``extra_content``; older pydantic lacks ``warnings=``, so retry without it."""
  285. if hasattr(extra, "model_dump"):
  286. for dump_kwargs in ({"warnings": False}, {}):
  287. try:
  288. return extra.model_dump(**dump_kwargs)
  289. except TypeError:
  290. continue
  291. except Exception:
  292. break
  293. return extra
  294. def _pareto_score(raw: Any) -> float | None:
  295. """Coding-score floor for the Pareto router as a float in [0, 1], else None."""
  296. try:
  297. score = float(raw) if raw not in (None, "") else None
  298. except (TypeError, ValueError):
  299. return None
  300. return score if score is not None and 0.0 <= score <= 1.0 else None
  301. def _swap_developer_role(sanitized: list, model_lower: str) -> list:
  302. """GPT-5/Codex models take a ``developer`` role instead of ``system``."""
  303. if (
  304. sanitized and isinstance(sanitized[0], dict) and sanitized[0].get("role") == "system"
  305. and any(p in model_lower for p in DEVELOPER_ROLE_MODELS)
  306. ):
  307. return [{**sanitized[0], "role": "developer"}, *sanitized[1:]]
  308. return sanitized
  309. def _apply_max_tokens(api_kwargs: dict, model: str, reasoning_config: Any, params: dict, profile_max: Any = None) -> None:
  310. """Preserve internal task/recovery budgets and provider protocol exceptions."""
  311. max_tokens_fn = params.get("max_tokens_param_fn")
  312. for candidate in (params.get("ephemeral_max_output_tokens"), params.get("max_tokens")):
  313. if candidate is not None and max_tokens_fn:
  314. api_kwargs.update(max_tokens_fn(_raise_gemini_thinking_max_tokens(model, reasoning_config, candidate)))
  315. return
  316. if profile_max and max_tokens_fn:
  317. api_kwargs.update(max_tokens_fn(_raise_gemini_thinking_max_tokens(model, reasoning_config, profile_max)))
  318. def _base_kwargs(model: str, sanitized: list, tools: Any, params: dict, profile: Any = None) -> dict[str, Any]:
  319. """Shared ``{model, messages[, temperature][, timeout][, tools]}`` scaffold for both build paths.
  320. ``temperature`` is profile-path only: ``fixed_temperature`` beats the caller's; ``OMIT_TEMPERATURE`` sends none.
  321. """
  322. api_kwargs: dict[str, Any] = {"model": model, "messages": sanitized}
  323. if profile is not None:
  324. from providers.base import OMIT_TEMPERATURE
  325. if profile.fixed_temperature is OMIT_TEMPERATURE:
  326. pass
  327. elif profile.fixed_temperature is not None:
  328. api_kwargs["temperature"] = profile.fixed_temperature
  329. elif params.get("temperature") is not None:
  330. api_kwargs["temperature"] = params["temperature"]
  331. if params.get("timeout") is not None:
  332. api_kwargs["timeout"] = params["timeout"]
  333. if tools:
  334. # Moonshot/Kimi uses a stricter JSON Schema flavor; rewriting here also covers aggregator routes.
  335. api_kwargs["tools"] = sanitize_moonshot_tools(tools) if is_moonshot_model(model) else tools
  336. return api_kwargs
  337. def _finish_kwargs(api_kwargs: dict[str, Any], sanitized: list, params: dict, *, supports_prompt_cache_key: bool) -> dict[str, Any]:
  338. """Tail shared by both build paths: content-addressed prompt_cache_key, then return."""
  339. _add_prompt_cache_key(
  340. api_kwargs, messages=sanitized, tools=api_kwargs.get("tools"), supports_prompt_cache_key=supports_prompt_cache_key,
  341. session_id=params.get("session_id"), cache_scope_id=params.get("cache_scope_id"),
  342. )
  343. return api_kwargs
  344. # Mistral's validator accepts only ``^[A-Za-z0-9]{9}$`` ("Tool call id was ... but must be a-z,
  345. # A-Z, 0-9, with a length of 9"); ``call_`` + 12 hex from a split tool_call batch or another
  346. # provider's id is rejected on replay. Ids are mapped on the wire copy only: stored history stays
  347. # byte-stable, and one source id always becomes the same 9 characters so the cache prefix holds.
  348. _MISTRAL_TOOL_ID_RE = re.compile(r"^[A-Za-z0-9]{9}$")
  349. _MISTRAL_ID_ALPHABET = string.digits + string.ascii_uppercase + string.ascii_lowercase
  350. # Hosts/providers mirror the Mistral row of message_sanitization._REASONING_CARRIER_ROUTES; the
  351. # model markers catch Mistral models served elsewhere (Ollama Cloud, vLLM, NIM, LM Studio).
  352. _MISTRAL_MODEL_MARKERS = (
  353. "mistral", "magistral", "mixtral", "devstral", "codestral", "ministral", "pixtral", "voxtral",
  354. )
  355. def _route_requires_mistral_tool_ids(provider: str | None, model: str | None, base_url: str | None) -> bool:
  356. from utils import base_url_host_matches
  357. if (provider or "").strip().lower() == "mistral" or base_url_host_matches(str(base_url or ""), "mistral.ai"):
  358. return True
  359. model_lower = (model or "").lower()
  360. return any(marker in model_lower for marker in _MISTRAL_MODEL_MARKERS)
  361. def _mistral_tool_id(key: str) -> str:
  362. """``key`` when already 9 alphanumeric characters, else a stable base62 digest of it."""
  363. if _MISTRAL_TOOL_ID_RE.fullmatch(key):
  364. return key
  365. number = int.from_bytes(hashlib.sha256(key.encode("utf-8", "surrogatepass")).digest()[:8], "big")
  366. chars = []
  367. for _ in range(9):
  368. number, rem = divmod(number, 62)
  369. chars.append(_MISTRAL_ID_ALPHABET[rem])
  370. return "".join(chars)
  371. def _sanitize_message(
  372. msg: Any, strip_extra_content: bool, strip_reasoning: frozenset = frozenset(),
  373. native_reasoning_details_type: str | None = None, *, mistral_tool_ids: bool = False,
  374. ) -> dict | None:
  375. """Sanitized copy of ``msg``, or None when nothing needs stripping.
  376. Drops persistence sidecars, ``_``-prefixed scaffolding markers, tool-call ``call_id`` /
  377. ``response_item_id`` (and ``extra_content`` unless Gemini), an assistant
  378. ``tool_calls: []`` / ``null`` (strict providers reject both), ``name``
  379. on tool results (schema-valid only on user/assistant messages; strict
  380. providers reject it with ``contains item with unknown key name``), and the reasoning
  381. keys in ``strip_reasoning`` (carriers the route does not read, ``_wire_reasoning_carriers``).
  382. On a route that reads ``reasoning_details``, private ``<provider>.native_assistant`` carriers still go only to the
  383. profile that declared that exact type: another provider's signed replay is meaningless (or
  384. rejected) elsewhere, and stored history keeps it for a return to the original provider.
  385. ``mistral_tool_ids`` maps the call ``id`` and the result ``tool_call_id`` through
  386. ``_mistral_tool_id`` from the pairing key the result writers use (``coalesce_tool_call_id``:
  387. ``call_id`` first, call half of a composite ``call|item`` id), so each pair stays matched.
  388. """
  389. if not isinstance(msg, dict):
  390. return None
  391. strip_keys = [k for k in msg if k in _STRIP_MSG_KEYS or (isinstance(k, str) and k.startswith("_"))]
  392. if msg.get("role") == "assistant":
  393. strip_keys += [k for k in strip_reasoning if k in msg]
  394. kept_details = None
  395. if "reasoning_details" not in strip_keys and isinstance(msg.get("reasoning_details"), list):
  396. details = msg["reasoning_details"]
  397. kept = [d for d in details if not (
  398. isinstance(d, dict) and isinstance(d.get("type"), str)
  399. and d["type"].endswith(".native_assistant") and d["type"] != native_reasoning_details_type)]
  400. if len(kept) != len(details):
  401. strip_keys.append("reasoning_details")
  402. kept_details = kept
  403. # ``name`` is schema-valid on user/assistant messages, so the removal is
  404. # role-qualified: only tool results carry it illegally (strict providers
  405. # reject with "contains item with unknown key name").
  406. if msg.get("role") == "tool" and "name" in msg:
  407. strip_keys.append("name")
  408. out_msg = {k: v for k, v in msg.items() if k not in strip_keys}
  409. remapped = False
  410. if mistral_tool_ids and msg.get("role") == "tool":
  411. result_key = coalesce_tool_call_id({"id": msg.get("tool_call_id")})
  412. if result_key and (mapped := _mistral_tool_id(result_key)) != msg["tool_call_id"]:
  413. out_msg["tool_call_id"] = mapped
  414. remapped = True
  415. if kept_details:
  416. out_msg["reasoning_details"] = kept_details
  417. tool_calls = msg.get("tool_calls")
  418. copied_tool_calls = None
  419. if msg.get("role") == "assistant" and "tool_calls" in msg and (tool_calls is None or (isinstance(tool_calls, list) and not tool_calls)):
  420. out_msg.pop("tool_calls", None)
  421. strip_keys.append("tool_calls")
  422. elif isinstance(tool_calls, list):
  423. for tc_idx, tc in enumerate(tool_calls):
  424. if not isinstance(tc, dict):
  425. continue
  426. keys = [k for k in _STRIP_TC_KEYS if k in tc]
  427. if "extra_content" in tc and (
  428. strip_extra_content or not _has_replayable_thought_signature(tc["extra_content"])
  429. ):
  430. keys.append("extra_content")
  431. key = coalesce_tool_call_id(tc) if mistral_tool_ids else ""
  432. mapped = _mistral_tool_id(key) if key else None
  433. if keys or (mapped is not None and mapped != tc.get("id")):
  434. if copied_tool_calls is None:
  435. copied_tool_calls = list(tool_calls)
  436. copied_tool_calls[tc_idx] = {k: v for k, v in tc.items() if k not in keys}
  437. if mapped is not None:
  438. copied_tool_calls[tc_idx]["id"] = mapped
  439. if copied_tool_calls is not None:
  440. out_msg["tool_calls"] = copied_tool_calls
  441. return out_msg if strip_keys or remapped or copied_tool_calls is not None else None
  442. class ChatCompletionsTransport(ProviderTransport):
  443. """Transport for api_mode='chat_completions'."""
  444. # ``{alias: original}`` of the most recent request. ``None`` = no request recorded
  445. # (normalize-only call sites) -> static alias; ``{}`` = no aliases emitted.
  446. _last_wire_aliases: dict[str, str] | None = None
  447. @property
  448. def api_mode(self) -> str:
  449. return "chat_completions"
  450. def convert_messages(self, messages: list[dict[str, Any]], **kwargs) -> list[dict[str, Any]]:
  451. """Strip internal fields that strict chat-completions providers reject (HTTP 400/422).
  452. Returns the input list unchanged when nothing needs sanitizing.
  453. """
  454. profile = kwargs.get("provider_profile")
  455. provider = getattr(profile, "name", None) or kwargs.get("provider")
  456. strip_extra_content = not _model_consumes_thought_signature(kwargs.get("model"), kwargs.get("base_url"), provider)
  457. native_type = getattr(profile, "native_reasoning_details_type", None) or None
  458. carriers = _wire_reasoning_carriers(provider, kwargs.get("base_url"))
  459. if native_type and carriers:
  460. # A profile declaring a native carrier type consumes replayed details by contract.
  461. carriers = carriers | {"reasoning_details"}
  462. strip_reasoning = frozenset({"reasoning_content", "reasoning", "reasoning_details"}) - carriers
  463. mistral_ids = _route_requires_mistral_tool_ids(provider, kwargs.get("model"), kwargs.get("base_url"))
  464. sanitized_pairs = [(m, _sanitize_message(m, strip_extra_content, strip_reasoning, native_type,
  465. mistral_tool_ids=mistral_ids))
  466. for m in messages]
  467. if all(s is None for _, s in sanitized_pairs):
  468. return messages
  469. return [m if s is None else s for m, s in sanitized_pairs]
  470. def convert_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
  471. """Tools are already in OpenAI format — identity."""
  472. return tools
  473. def build_kwargs(
  474. self, model: str, messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, **params,
  475. ) -> dict[str, Any]:
  476. """Build chat.completions.create() kwargs.
  477. With ``provider_profile`` every quirk comes from the profile; the legacy flag
  478. path below (is_kimi, is_openrouter, ...) is only reached for unregistered providers.
  479. """
  480. _profile = params.get("provider_profile")
  481. # Read before convert_messages strips the key; only a profile that asks for it consumes it.
  482. params["data_use"] = data_use_requirement(messages)
  483. sanitized = self.convert_messages(
  484. messages, model=model, base_url=params.get("base_url"), provider_profile=_profile,
  485. provider=params.get("provider_name"),
  486. )
  487. if _profile:
  488. return self._build_kwargs_from_profile(_profile, model, sanitized, tools, params)
  489. sanitized = _swap_developer_role(sanitized, params.get("model_lower", (model or "").lower()))
  490. api_kwargs = _base_kwargs(model, sanitized, tools, params)
  491. is_kimi = params.get("is_kimi", False)
  492. supports_reasoning = params.get("supports_reasoning", False)
  493. reasoning_config = _reasoning_config_for_model(model, params.get("reasoning_config"))
  494. _apply_max_tokens(api_kwargs, model, reasoning_config, params)
  495. # Kimi / TokenHub by host (agents with no registered profile): top-level reasoning_effort (unless thinking disabled).
  496. thinking_off = isinstance(reasoning_config, dict) and reasoning_config.get("enabled") is False
  497. _e = requested_effort(reasoning_config)
  498. if is_kimi and not thinking_off:
  499. # K3 = low/high/max (server default high), K2-era = low/medium/high (default medium).
  500. _supported = kimi_supported_efforts(model)
  501. is_k3 = _supported is KIMI_K3_EFFORTS
  502. api_kwargs["reasoning_effort"] = (
  503. ("high" if is_k3 else "medium") if _e is None
  504. else clamp_effort(_e, _supported, KIMI_K3_OVERRIDES if is_k3 else None)
  505. )
  506. if params.get("is_tokenhub", False) and not thinking_off:
  507. api_kwargs["reasoning_effort"] = tokenhub_effort(_e)
  508. extra_body: dict[str, Any] = {}
  509. is_openrouter = params.get("is_openrouter", False)
  510. base_url = params.get("base_url")
  511. if is_openrouter and params.get("provider_preferences"):
  512. extra_body["provider"] = params["provider_preferences"]
  513. # Pareto Code router plugin (same shape as the OpenRouter profile path).
  514. if is_openrouter and model == "openrouter/pareto-code":
  515. _pareto_score_f = _pareto_score(params.get("openrouter_min_coding_score"))
  516. if _pareto_score_f is not None:
  517. extra_body["plugins"] = [{"id": "pareto-router", "min_coding_score": _pareto_score_f}]
  518. if is_kimi:
  519. extra_body["thinking"] = {"type": "disabled" if thinking_off else "enabled"}
  520. if supports_reasoning:
  521. # An endpoint claimed by a profile (providers.profile_for_host) takes its verbatim payload.
  522. if params.get("profile_endpoint", False):
  523. if params.get("endpoint_reasoning") is not None:
  524. extra_body["reasoning"] = params["endpoint_reasoning"]
  525. else:
  526. _effort = (reasoning_config.get("effort", "medium") or "medium") if reasoning_config and isinstance(reasoning_config, dict) else "medium"
  527. # Honor explicit "thinking off" like the profile path — never re-enable it.
  528. off = thinking_off or _effort == "none"
  529. extra_body["reasoning"] = {"enabled": not off, "effort": "none" if off else _effort}
  530. if str(params.get("provider_name") or "").strip().lower() == "gemini":
  531. raw_thinking_config = _build_gemini_thinking_config(model, reasoning_config)
  532. if _is_gemini_openai_compat_base_url(base_url):
  533. thinking_config = _snake_case_gemini_thinking_config(raw_thinking_config)
  534. if thinking_config:
  535. openai_compat_extra = extra_body.get("extra_body", {})
  536. openai_compat_extra["google"] = {**openai_compat_extra.get("google", {}), "thinking_config": thinking_config}
  537. extra_body["extra_body"] = openai_compat_extra
  538. elif raw_thinking_config:
  539. extra_body["thinking_config"] = raw_thinking_config
  540. if params.get("extra_body_additions"):
  541. extra_body.update(params["extra_body_additions"])
  542. if extra_body:
  543. api_kwargs["extra_body"] = extra_body
  544. if params.get("request_overrides"):
  545. api_kwargs.update(params["request_overrides"])
  546. return _finish_kwargs(
  547. api_kwargs, sanitized, params,
  548. supports_prompt_cache_key=bool(params.get("supports_prompt_cache_key")) or _is_openai_api_base_url(base_url),
  549. )
  550. def _build_kwargs_from_profile(self, profile, model, sanitized, tools, params):
  551. """Build API kwargs from a ProviderProfile — every quirk comes from the profile object."""
  552. sanitized = _swap_developer_role(profile.prepare_messages(sanitized), (model or "").lower())
  553. api_kwargs = _base_kwargs(model, sanitized, tools, params, profile=profile)
  554. reasoning_config = _reasoning_config_for_model(model, params.get("reasoning_config"))
  555. # Profiles fronting several backends override get_max_tokens() per model.
  556. _apply_max_tokens(api_kwargs, model, reasoning_config, params, profile_max=profile.get_max_tokens(model))
  557. # LM Studio's probed allowed_options ride only when present, so plugin hooks written
  558. # against the older keyword set (no ``**context``) keep working.
  559. _lm_options = params.get("lmstudio_reasoning_options")
  560. extra_body_from_profile, top_level_from_profile = profile.build_api_kwargs_extras(
  561. reasoning_config=reasoning_config, supports_reasoning=params.get("supports_reasoning", False),
  562. qwen_session_metadata=params.get("qwen_session_metadata"), model=model,
  563. base_url=params.get("base_url"), ollama_num_ctx=params.get("ollama_num_ctx"),
  564. session_id=params.get("session_id"), cache_scope_id=params.get("cache_scope_id"),
  565. **({"lmstudio_reasoning_options": _lm_options} if _lm_options is not None else {}),
  566. )
  567. api_kwargs.update(top_level_from_profile)
  568. extra_body: dict[str, Any] = {}
  569. profile_body = profile.build_extra_body(
  570. session_id=params.get("session_id"), provider_preferences=params.get("provider_preferences"), model=model,
  571. base_url=params.get("base_url"), reasoning_config=reasoning_config,
  572. openrouter_min_coding_score=params.get("openrouter_min_coding_score"), data_use=params.get("data_use"),
  573. )
  574. for part in (profile_body, extra_body_from_profile, params.get("extra_body_additions")):
  575. if part:
  576. extra_body.update(part)
  577. for k, v in (params.get("request_overrides") or {}).items():
  578. if k == "extra_body" and isinstance(v, dict):
  579. extra_body.update(v)
  580. else:
  581. api_kwargs[k] = v
  582. if extra_body:
  583. # Native Gemini speaks Google's REST schema: OpenAI-style extra_body
  584. # keys (tags, reasoning, provider, ...) are unknown fields -> HTTP 400.
  585. # The native client reads only thinking_config and the long-decode budget.
  586. try:
  587. from agent.gemini_native_adapter import is_native_gemini_base_url
  588. _native_gemini = is_native_gemini_base_url(params.get("base_url"))
  589. except Exception:
  590. _native_gemini = False
  591. if _native_gemini:
  592. extra_body = {k: v for k, v in extra_body.items() if k in ("thinking_config", "thinkingConfig", "max_output_tokens")}
  593. if extra_body:
  594. api_kwargs["extra_body"] = extra_body
  595. return _finish_kwargs(
  596. api_kwargs, sanitized, params, supports_prompt_cache_key=bool(getattr(profile, "supports_prompt_cache_key", False)),
  597. )
  598. def normalize_response(self, response: Any, **kwargs) -> NormalizedResponse:
  599. """Normalize an OpenAI ChatCompletion.
  600. Gemini ``extra_content`` rides on ToolCall.provider_data; ``reasoning_content`` and
  601. ``reasoning_details`` stay distinct in provider_data because downstream reads them so.
  602. """
  603. choice = response.choices[0]
  604. msg = getattr(choice, "message", None)
  605. _fr = getattr(choice, "finish_reason", None)
  606. # Poolside returns int finish_reason; Gemini-fronting gateways return
  607. # uppercase STOP / MAX_TOKENS — fold to the OpenAI contract here.
  608. finish_reason = _normalize_finish_reason(str(_fr) if isinstance(_fr, int) else _fr) or "stop"
  609. tool_calls = None
  610. if getattr(msg, "tool_calls", None):
  611. tool_calls = [tc for tc in (self._normalize_tool_call(tc) for tc in msg.tool_calls) if tc is not None]
  612. usage = Usage.from_openai(response.usage) if hasattr(response, "usage") and response.usage else None
  613. # Fields some SDKs park in pydantic ``model_extra`` rather than as attributes.
  614. reasoning_content = _attr_or_model_extra(msg, "reasoning_content")
  615. provider_data: dict[str, Any] = {}
  616. if reasoning_content is not None:
  617. provider_data["reasoning_content"] = reasoning_content
  618. if getattr(msg, "reasoning_details", None):
  619. provider_data["reasoning_details"] = msg.reasoning_details
  620. # Message-level carriers (Gemini text-turn signature, profile-declared reasoning keys):
  621. # agent/reasoning_carriers.py.
  622. from agent.reasoning_carriers import declared_carrier_keys
  623. opaque_keys, text_keys = declared_carrier_keys()
  624. for key in dict.fromkeys(("extra_content", *opaque_keys, *text_keys)):
  625. if (value := _attr_or_model_extra(msg, key)) is not None:
  626. provider_data[key] = _dump_extra_content(value)
  627. # OpenAI structured refusal (``message.refusal`` set, ``content`` empty); without
  628. # promotion the loop retries a deterministic refusal as an empty response.
  629. content = getattr(msg, "content", None)
  630. refusal = _attr_or_model_extra(msg, "refusal")
  631. if isinstance(refusal, str) and refusal.strip():
  632. provider_data["refusal"] = refusal
  633. # Terminal ``content_filter`` only when the refusal is the sole payload.
  634. if not (isinstance(content, str) and content.strip()) and not tool_calls:
  635. content = refusal
  636. if finish_reason in (None, "stop"):
  637. finish_reason = "content_filter"
  638. return NormalizedResponse(
  639. content=content, tool_calls=tool_calls, finish_reason=finish_reason,
  640. # A profile-declared text carrier doubles as the readable reasoning (first declared wins).
  641. reasoning=getattr(msg, "reasoning", None) or next(
  642. (provider_data[k] for k in text_keys if provider_data.get(k)), None), usage=usage,
  643. provider_data=provider_data or None,
  644. )
  645. def _normalize_tool_call(self, tc: Any) -> ToolCall | None:
  646. """One SDK tool call -> ToolCall; None when it lacks a function/name (matches Relay's codec)."""
  647. tc_function = getattr(tc, "function", None)
  648. name = getattr(tc_function, "name", None)
  649. if tc_function is None or name is None:
  650. return None
  651. # Reverse only aliases THIS request emitted; a real ``hermes_tool_search`` tool stays itself.
  652. alias_map = self._last_wire_aliases
  653. if alias_map is None:
  654. name = "tool_search" if name == _XAI_TOOL_SEARCH_ALIAS else name
  655. else:
  656. name = alias_map.get(name, name)
  657. arguments = getattr(tc_function, "arguments", None)
  658. extra = _attr_or_model_extra(tc, "extra_content")
  659. fn_sig = _attr_or_model_extra(tc_function, "thought_signature") # gateway-signed Gemini 3 calls
  660. provider_data = {k: v for k, v in (("extra_content", None if extra is None else _dump_extra_content(extra)),
  661. ("thought_signature", fn_sig)) if v is not None}
  662. call = ToolCall(
  663. id=getattr(tc, "id", None), name=name, arguments="{}" if arguments is None else arguments,
  664. provider_data=provider_data or None,
  665. )
  666. if getattr(tc_function, "args_repaired", False) is True:
  667. call.args_repaired = True # stream assembly fixed the JSON; read by tool-call quality metrics
  668. return call
  669. def validate_response(self, response: Any) -> bool:
  670. """Check that response has valid choices and is not a router failure shim."""
  671. if response is None or not getattr(response, "choices", None):
  672. return False
  673. return not is_router_timeout_shim(response)
  674. def extract_cache_stats(self, response: Any) -> dict[str, int] | None:
  675. """Cache stats from prompt_tokens_details (OpenRouter/OpenAI) or DeepSeek's top-level prompt_cache_hit_tokens."""
  676. usage = getattr(response, "usage", None)
  677. if usage is None:
  678. return None
  679. details = getattr(usage, "prompt_tokens_details", None)
  680. cached = getattr(details, "cached_tokens", 0) or 0 if details else 0
  681. written = getattr(details, "cache_write_tokens", 0) or 0 if details else 0
  682. cached = cached or getattr(usage, "prompt_cache_hit_tokens", 0) or 0 # DeepSeek native
  683. return {"cached_tokens": cached, "creation_tokens": written} if cached or written else None
  684. from agent.transports import register_transport
  685. register_transport("chat_completions", ChatCompletionsTransport)