api_server_openai_routes.py 94 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633
  1. """OpenAI-compatible routes for the API server adapter.
  2. ``OpenAICompatRoutesMixin`` (inherited by ``APIServerAdapter``) carries ``/v1/chat/completions``,
  3. ``/v1/responses`` (+ GET/DELETE), their SSE writers and the Responses-transcript helpers.
  4. api_server-internal helpers are imported lazily inside each method: the origin imports this
  5. module (top-level import = cycle), and lazy lookup keeps ``patch("...api_server.X")`` effective.
  6. """
  7. import asyncio
  8. import copy
  9. import json
  10. import logging
  11. import re
  12. import time
  13. import uuid
  14. from contextlib import suppress
  15. from typing import Any, Dict, List, Optional
  16. from agent.i18n import t
  17. try:
  18. from aiohttp import web
  19. except ImportError: # pragma: no cover - mirrors api_server's optional import
  20. web = None # type: ignore[assignment]
  21. # Logger parity with the origin module (moved log records keep their name).
  22. logger = logging.getLogger("gateway.platforms.api_server")
  23. async def _iter_stream_items(stream_q, agent_task, response):
  24. """Yield agent stream items until EOS, writing SSE keepalives while idle.
  25. Yields the ``None`` sentinel once so callers can run EOS-only work; when ``agent_task``
  26. is already done the remaining queue is drained and the sentinel swallowed.
  27. """
  28. from gateway.platforms.api_server import CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS
  29. last_activity = time.monotonic()
  30. while True:
  31. try:
  32. item = await asyncio.wait_for(stream_q.get(), timeout=0.5)
  33. except TimeoutError:
  34. if agent_task.done():
  35. while True:
  36. try:
  37. item = stream_q.get_nowait()
  38. except asyncio.QueueEmpty:
  39. return
  40. if item is None:
  41. return
  42. yield item
  43. last_activity = time.monotonic()
  44. if time.monotonic() - last_activity >= CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS:
  45. await response.write(b": keepalive\n\n")
  46. last_activity = time.monotonic()
  47. continue
  48. if item is None:
  49. yield None
  50. return
  51. yield item
  52. last_activity = time.monotonic()
  53. def _result_flags(result: Any) -> tuple:
  54. """``(completed, partial, failed, error)`` from an agent result dict (defaults if not a dict)."""
  55. if not isinstance(result, dict):
  56. return True, False, False, None
  57. return (bool(result.get("completed", True)) and not result.get('interrupted'), bool(result.get("partial")),
  58. bool(result.get("failed")), result.get("error"))
  59. def _budget_exhausted(result: Any) -> bool:
  60. """The turn ran out of its iteration budget; its final text is the agent's own wrap-up."""
  61. reason = str(result.get("turn_exit_reason") or "") if isinstance(result, dict) else ""
  62. return reason.startswith(("budget_exhausted", "max_iterations_reached("))
  63. def _finish_reason(completed, is_partial, is_failed, err_msg, agent_error=None, *, budget_exhausted=False) -> str:
  64. """OpenAI ``finish_reason``: "length" for truncation, "error" for failure, else "stop"."""
  65. # OpenAI uses "length" for truncation, "stop" for normal completion, and downstream SDKs accept "error"
  66. # / custom codes. See issue #22496.
  67. if is_partial and err_msg and "truncat" in err_msg.lower():
  68. return "length"
  69. if agent_error is not None or is_failed or (not completed and err_msg):
  70. return "error"
  71. # "cancelled" means the user interrupted; a budget-capped turn stopped short of the task instead,
  72. # and a broker reading "cancelled" discards a usable wrap-up answer.
  73. if budget_exhausted:
  74. return "length"
  75. return 'cancelled' if not completed else 'stop'
  76. _RESPONSES_FINGERPRINT_KEYS = (
  77. "input", "instructions", "previous_response_id", "conversation", "conversation_history", "model",
  78. "provider", "model_options", "tools",
  79. )
  80. def _settled(record):
  81. """A stored idempotency record that carries its answer. A claim written without one (an
  82. older release recorded the claim before the run) has an unknown outcome and never replays."""
  83. return record is not None and record.get('response') is not None
  84. def _responses_fingerprint_keys(body):
  85. """``truncation`` joins the fingerprint only when sent, so an exact retry of a request
  86. recorded before it was fingerprinted keeps matching its stored key."""
  87. return _RESPONSES_FINGERPRINT_KEYS + (("truncation",) if body.get("truncation") is not None else ())
  88. async def _response_identity(store, durable_key):
  89. if durable_key is not None:
  90. from gateway.platforms.api_server_response_identity import durable_response_id
  91. return durable_response_id(durable_key[0]), await store.off_loop(store.request_created_at, durable_key[0])
  92. return f'resp_{uuid.uuid4().hex[:28]}', int(time.time())
  93. def _matching_replay(store, key, binding):
  94. """-> ``(binding matches, stored replay record or None)``, read as one store unit."""
  95. if not store.request_key_matches(key, binding):
  96. return False, None
  97. return True, store.get(key)
  98. def _store_nonstream_response(store, snapshot, replay_record, conversation, terminal_replay):
  99. """Non-streaming /v1/responses persistence as one store unit: body, conversation pointer, replay record."""
  100. if snapshot is not None:
  101. response_id, data = snapshot
  102. advance = conversation and (not terminal_replay or store.get_conversation(conversation) in (None, response_id))
  103. store.put(response_id, data)
  104. if advance:
  105. store.set_conversation(conversation, response_id)
  106. if replay_record is not None:
  107. store.put(*replay_record)
  108. def _response_status(result):
  109. if result.get('interrupted'):
  110. return 'cancelled'
  111. if result.get('failed') or result.get('error'):
  112. return 'failed'
  113. if result.get('completed') is False:
  114. return 'incomplete'
  115. return 'completed'
  116. def _hermes_extras(completed, is_partial, is_failed, err_msg, finish_reason: str, *,
  117. budget_exhausted: bool = False) -> dict[str, Any]:
  118. error_code = ("iteration_budget_exhausted" if budget_exhausted and finish_reason == "length"
  119. else "output_truncated" if finish_reason == "length" else "agent_error")
  120. return {"completed": completed, "partial": is_partial, "failed": is_failed, "error": err_msg,
  121. "error_code": error_code}
  122. def _transformed_notice() -> str:
  123. return t("platform.api_server.transformed_notice")
  124. def _post_stream_transform(result: Any) -> tuple:
  125. """``(text, appended)`` a stream still owes after ``transform_llm_output`` rewrote the final
  126. (the deltas already carried the raw reply): the appended suffix, or the whole rewrite with
  127. ``appended=False`` when it is not a pure append. ``("", False)`` when nothing was transformed."""
  128. if not isinstance(result, dict) or not result.get("response_transformed"):
  129. return "", False
  130. final = result.get("final_response") or ""
  131. original = result.get("pre_transform_response") or ""
  132. if original and final.startswith(original):
  133. return final[len(original):], True
  134. return final, False
  135. def _message_item(text: Any) -> dict[str, Any]:
  136. """Responses ``message`` output item carrying one ``output_text`` part."""
  137. return {"type": "message", "role": "assistant",
  138. "content": [{"type": "output_text", "text": text}]}
  139. def _cap_text(text: str, keep: int) -> str:
  140. """Head of ``text`` plus a marker saying how much was cut (the Responses truncation rule)."""
  141. return text[:keep] + "...[" + str(len(text) - keep) + " more chars]"
  142. def _cap_history_tool_outputs(history: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]:
  143. """Copy of ``history`` with tool outputs and string tool-call arguments longer than
  144. ``max_chars`` cut down. Only tool rows and ``tool_calls`` blobs change; user/assistant text
  145. is left alone, and the agent's own transcript rows are never mutated (rows are copied).
  146. Opt-in via gateway.api_server.history_tool_output_max_chars: a single stored snapshot
  147. embeds the full cumulative history, so a few large tool outputs pushed one
  148. response_store.db write to ~677 KB (#82513)."""
  149. if max_chars <= 0:
  150. return history
  151. out: list[dict[str, Any]] = []
  152. for msg in history:
  153. if not isinstance(msg, dict):
  154. out.append(msg)
  155. continue
  156. content = msg.get("content")
  157. if msg.get("role") == "tool" and isinstance(content, str) and len(content) > max_chars:
  158. msg = {**msg, "content": _cap_text(content, max_chars)}
  159. tool_calls = msg.get("tool_calls")
  160. if msg.get("role") == "assistant" and isinstance(tool_calls, list):
  161. capped_calls = []
  162. for call in tool_calls:
  163. fn = call.get("function") if isinstance(call, dict) else None
  164. raw = fn.get("arguments") if isinstance(fn, dict) else None
  165. if isinstance(raw, str) and len(raw) > max_chars:
  166. try:
  167. args = json.loads(raw)
  168. except ValueError:
  169. args = None
  170. if isinstance(args, dict):
  171. for k, v in args.items():
  172. if isinstance(v, str) and len(v) > max_chars:
  173. args[k] = _cap_text(v, max_chars)
  174. call = {**call, "function": {**fn, "arguments": json.dumps(args)}}
  175. capped_calls.append(call)
  176. msg = {**msg, "tool_calls": capped_calls}
  177. out.append(msg)
  178. return out
  179. def _reasoning_item(text: str) -> dict[str, Any]:
  180. """Completed Responses ``reasoning`` output item (same shape the SSE writer closes with)."""
  181. return {"id": f"rs_{uuid.uuid4().hex[:24]}", "type": "reasoning", "status": "completed",
  182. "summary": [{"type": "summary_text", "text": text}]}
  183. def _is_reasoning_input_item(item: Any) -> bool:
  184. """Echoed-back ``reasoning`` output item: Responses SDK clients replay a prior response's
  185. ``output`` list as the next ``input``. It carries no message content, so it must be
  186. skipped rather than parsed into an empty ``user`` turn (#99552)."""
  187. return isinstance(item, dict) and item.get("type") == "reasoning"
  188. def _turn_reasoning_text(
  189. conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any]) -> str:
  190. """Reasoning the model produced on this turn, joined for a non-streaming
  191. ``message.reasoning_content``. Read from the assistant messages the agent already
  192. persisted (``build_assistant_message`` stores the structured reasoning under
  193. ``reasoning``) rather than re-accumulating callback deltas, so it is exactly what the
  194. stream would have carried and cannot double-count the post-response fallback."""
  195. messages = result.get("messages") if isinstance(result, dict) else None
  196. if not isinstance(messages, list):
  197. return ""
  198. start = OpenAICompatRoutesMixin._response_messages_turn_start_index(
  199. conversation_history, user_message, result)
  200. parts = [m["reasoning"] for m in messages[start:]
  201. if isinstance(m, dict) and m.get("role") == "assistant"
  202. and isinstance(m.get("reasoning"), str) and m["reasoning"].strip()]
  203. return "\n\n".join(parts)
  204. def _trim_tool_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
  205. """Trim large tool payloads in place so response.completed stays under ~100KB (clients
  206. already received the full details via the incremental events)."""
  207. for item in items:
  208. if item.get("type") == "function_call":
  209. try:
  210. raw = item.get("arguments")
  211. args = json.loads(raw) if isinstance(raw, str) else item.get("arguments", {})
  212. if isinstance(args, dict):
  213. for k in ("content", "query", "pattern", "old_string", "new_string"):
  214. if isinstance(args.get(k), str) and len(args[k]) > 500:
  215. args[k] = f"[{len(args[k])} chars — truncated for response.completed]"
  216. item["arguments"] = json.dumps(args)
  217. except Exception:
  218. pass
  219. elif item.get("type") == "function_call_output":
  220. output = item.get("output", [])
  221. if isinstance(output, list) and output:
  222. first = output[0]
  223. if isinstance(first, dict) and first.get("type") == "input_text":
  224. text = first.get("text", "")
  225. if len(text) > 1000:
  226. first["text"] = _cap_text(text, 500)
  227. item["output"] = [first]
  228. return items
  229. class _ResponsesStream:
  230. """Per-request state and event emitters for the POST /v1/responses SSE writer.
  231. Every event carries a monotonic ``sequence_number`` (canonical Responses SSE schema). Text
  232. deltas are batched (50ms) against Open WebUI re-render storms; tool events flush first.
  233. """
  234. def __init__(self, adapter, response, *, response_id: str, model: str, created_at: int,
  235. conversation_history: list[dict[str, str]], user_message: str,
  236. instructions: Optional[str], conversation: Optional[str], store: bool, session_id: str,
  237. durable_key=None, terminal_replay=False):
  238. from gateway.platforms import api_server as api
  239. self._api = api
  240. self.adapter, self.response, self.response_id = adapter, response, response_id
  241. self.model, self.created_at, self.conversation_history = model, created_at, conversation_history
  242. self.user_message, self.instructions = user_message, instructions
  243. self.conversation, self.store, self.session_id = conversation, store, session_id
  244. self.durable_key = durable_key
  245. self.terminal_replay = terminal_replay
  246. # Resolved in the request's profile scope: a snapshot written after it (disconnect) must not follow another.
  247. self.response_store = adapter._current_response_store()
  248. self.final_text_parts: list[str] = []
  249. self.pending_tool_calls: list[dict[str, Any]] = [] # open function_call items, in order
  250. self.emitted_items: list[dict[str, Any]] = [] # output items so far (terminal payload)
  251. self.output_index = 0
  252. self.call_counter = 0 # call_id fallback when the agent supplies no tool_call_id
  253. self.sequence_number = 0
  254. self._identity_counts = {}
  255. self._published_output_ids = []
  256. self._suppress_incremental = False
  257. self.message_item_id = self._item_id('message')
  258. self.message_output_index: Optional[int] = None
  259. self.message_opened = False
  260. self.reasoning_item: Optional[dict[str, Any]] = None # open ``reasoning`` output item
  261. self.final_response_text = ""
  262. self.transformed_final = "" # non-append transform_llm_output rewrite; replaces the deltas
  263. self.agent_error: Optional[str] = None
  264. self.usage: dict[str, int] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
  265. self.terminal_snapshot_persisted = False
  266. self.recorded_events: list[tuple] = [] # (event_type, data) as written, for idempotent replay
  267. self.result: Any = None
  268. self._batch_buf: list[str] = []
  269. self._batch_timer: Optional[asyncio.Task] = None
  270. self._batch_lock = asyncio.Lock()
  271. async def write_event(self, event_type: str, data: dict[str, Any]) -> None:
  272. if self.durable_key and 'output_index' in data:
  273. if self._suppress_incremental:
  274. return
  275. index = data['output_index']
  276. if event_type == 'response.output_item.added':
  277. valid = index == len(self._published_output_ids)
  278. if valid:
  279. self._published_output_ids.append(data['item']['id'])
  280. else:
  281. identity = data.get('item_id') or data.get('item', {}).get('id')
  282. valid = 0 <= index < len(self._published_output_ids) and (
  283. identity is None or self._published_output_ids[index] == identity)
  284. if not valid:
  285. # A late/backpressured observer may miss a globally allocated prefix. Keep the
  286. # prefix it already published; complete from the canonical terminal snapshot.
  287. self._suppress_incremental = True
  288. return
  289. if "sequence_number" not in data:
  290. data["sequence_number"] = self.sequence_number
  291. self.sequence_number += 1
  292. if self.durable_key: # the only reader is the keyed replay record (emit_completed)
  293. # Snapshot: terminal trimming later edits the nested tool output these frames share.
  294. self.recorded_events.append((event_type, copy.deepcopy(data)))
  295. await self.response.write(self._api._sse_frame(data, event=event_type))
  296. def _item_id(self, kind, *, call_id=None, commentary=False):
  297. from gateway.platforms.api_server_response_identity import next_item_id
  298. return next_item_id(self.response_id, self._identity_counts, kind,
  299. call_id=call_id, commentary=commentary)
  300. async def _output_index(self, item_id):
  301. index = (await self.response_store.off_loop(self.response_store.output_index, self.durable_key[0], item_id)
  302. if self.durable_key else self.output_index)
  303. self.output_index = max(self.output_index, index + 1)
  304. return index
  305. def envelope(self, status: str) -> dict[str, Any]:
  306. return {"id": self.response_id, "object": "response", "status": status,
  307. "created_at": self.created_at, "model": self.model}
  308. def terminal_envelope(self, status: str, output: list[dict[str, Any]], *, error=None) -> dict:
  309. """``envelope`` + ``output`` (+ ``error`` when given) + ``usage``, in wire key order."""
  310. env = self.envelope(status)
  311. env["output"] = output
  312. if error is not None:
  313. env["error"] = ({"code": "agent_error", "message": error} if self.durable_key
  314. else {"message": error, "type": "server_error"})
  315. env["usage"] = self._api._responses_usage_payload(self.usage)
  316. return env
  317. def _history_with_user(self) -> list[dict[str, Any]]:
  318. return list(self.conversation_history) + [{"role": "user", "content": self.user_message}]
  319. async def persist_snapshot(self, response_env: dict[str, Any], *, history=None, session_id=None):
  320. if self.store:
  321. await self.response_store.off_loop(self._persist_snapshot, response_env, history, session_id)
  322. def _persist_snapshot(self, response_env, history, session_id):
  323. """One store unit (worker thread, store lock held): the settled check, the body and the
  324. conversation pointer can not interleave with another observer's."""
  325. if self.durable_key and _settled(self.response_store.get(self.durable_key[0])):
  326. return # A concurrent observer already committed the exact terminal envelope.
  327. if response_env.get('status') == 'incomplete' and self.terminal_snapshot_persisted:
  328. # A cancel that interrupted the await of a terminal commit: its worker still finished it.
  329. return
  330. advance_conversation = self.conversation and (not self.terminal_replay
  331. or self.response_store.get_conversation(self.conversation) in (None, self.response_id))
  332. self.response_store.put(self.response_id, {
  333. "response": response_env,
  334. "conversation_history": self._history_with_user() if history is None else history,
  335. "instructions": self.instructions,
  336. "session_id": session_id or self.session_id})
  337. if advance_conversation:
  338. self.response_store.set_conversation(self.conversation, self.response_id)
  339. if response_env.get('status') != 'in_progress':
  340. # Set by the worker, under the store lock: a cancel of the awaiting handler cannot lose it.
  341. self.terminal_snapshot_persisted = True
  342. async def persist_incomplete_if_needed(self) -> None:
  343. """Persist an ``incomplete`` snapshot when no terminal one was written (disconnect /
  344. cancel paths), so GET /v1/responses/{id} and ``previous_response_id`` chaining survive."""
  345. if not self.store or self.terminal_snapshot_persisted:
  346. return
  347. text = "".join(self.final_text_parts) or self.final_response_text
  348. items = list(self.emitted_items)
  349. history = self._history_with_user()
  350. if text:
  351. items.append(_message_item(text))
  352. history.append({"role": "assistant", "content": text})
  353. await self.persist_snapshot(self.terminal_envelope("incomplete", items), history=history)
  354. async def emit_created(self) -> None:
  355. env = self.envelope("in_progress")
  356. env["output"] = []
  357. if not self.terminal_replay:
  358. # Stored before the id goes out: a client that reads it back at once must find it.
  359. await self.persist_snapshot(env)
  360. await self.write_event("response.created", {"type": "response.created", "response": env})
  361. async def _open_message_item(self) -> None:
  362. """Emit output_item.added for the assistant message on the first text delta."""
  363. if self.message_opened:
  364. return
  365. self.message_opened = True
  366. self.message_output_index = await self._output_index(self.message_item_id)
  367. await self.write_event("response.output_item.added", {
  368. "type": "response.output_item.added", "output_index": self.message_output_index,
  369. "item": {"id": self.message_item_id, "type": "message", "status": "in_progress",
  370. "role": "assistant", "content": []}})
  371. async def emit_text_delta(self, delta_text: str) -> None:
  372. if self.terminal_replay:
  373. self.final_text_parts.append(delta_text)
  374. return
  375. await self.close_reasoning_item()
  376. await self._open_message_item()
  377. self.final_text_parts.append(delta_text)
  378. await self.write_event("response.output_text.delta", {
  379. "type": "response.output_text.delta", "item_id": self.message_item_id,
  380. "output_index": self.message_output_index, "content_index": 0, "delta": delta_text,
  381. "logprobs": []})
  382. async def emit_reasoning_delta(self, delta_text: str) -> None:
  383. """Responses reasoning-summary family (#99552): one ``reasoning`` output item per
  384. thinking burst, closed before the next message/tool item opens."""
  385. if self.reasoning_item is None:
  386. item = {"id": self._item_id('reasoning'), "type": "reasoning", "status": "in_progress",
  387. "summary": []}
  388. self.reasoning_item = {"item": item, "output_index": await self._output_index(item['id']), "parts": []}
  389. await self.write_event("response.output_item.added", {
  390. "type": "response.output_item.added",
  391. "output_index": self.reasoning_item["output_index"], "item": item})
  392. await self.write_event("response.reasoning_summary_part.added", {
  393. "type": "response.reasoning_summary_part.added", "item_id": item["id"],
  394. "output_index": self.reasoning_item["output_index"], "summary_index": 0,
  395. "part": {"type": "summary_text", "text": ""}})
  396. rs = self.reasoning_item
  397. rs["parts"].append(delta_text)
  398. await self.write_event("response.reasoning_summary_text.delta", {
  399. "type": "response.reasoning_summary_text.delta", "item_id": rs["item"]["id"],
  400. "output_index": rs["output_index"], "summary_index": 0, "delta": delta_text})
  401. async def close_reasoning_item(self) -> None:
  402. rs, self.reasoning_item = self.reasoning_item, None
  403. if rs is None:
  404. return
  405. text = "".join(rs["parts"])
  406. item = dict(rs["item"], status="completed", summary=[{"type": "summary_text", "text": text}])
  407. base = {"item_id": item["id"], "output_index": rs["output_index"], "summary_index": 0}
  408. await self.write_event("response.reasoning_summary_text.done", {
  409. "type": "response.reasoning_summary_text.done", **base, "text": text})
  410. await self.write_event("response.reasoning_summary_part.done", {
  411. "type": "response.reasoning_summary_part.done", **base,
  412. "part": {"type": "summary_text", "text": text}})
  413. self.emitted_items.append(item)
  414. await self.write_event("response.output_item.done", {
  415. "type": "response.output_item.done", "output_index": rs["output_index"], "item": item})
  416. async def emit_commentary(self, text: str) -> None:
  417. """Mid-turn assistant commentary as its own completed ``message`` item carrying
  418. ``"phase": "commentary"`` — never appended to ``final_text_parts``, so the final answer
  419. item stays clean (#67580). Closes any open reasoning item first so a reasoning item
  420. never straddles a message item."""
  421. await self.close_reasoning_item()
  422. item = {"id": self._item_id('message', commentary=True), "status": "completed", "phase": "commentary",
  423. **_message_item(text)}
  424. idx = await self._output_index(item['id'])
  425. self.emitted_items.append({"phase": "commentary", **_message_item(text)})
  426. for event in ("response.output_item.added", "response.output_item.done"):
  427. await self.write_event(event, {"type": event, "output_index": idx, "item": item})
  428. async def emit_tool_started(self, payload: dict[str, Any]) -> None:
  429. """function_call ``output_item.added``; the agent's tool_call_id beats a generated call id."""
  430. await self.close_reasoning_item()
  431. self.call_counter += 1
  432. call_id = payload.get("tool_call_id") or f"call_{self.response_id[5:]}_{self.call_counter}"
  433. args = payload.get("arguments", {})
  434. arguments_str = json.dumps(args) if isinstance(args, dict) else str(args)
  435. name = payload.get("name", "")
  436. item = {"id": self._item_id('function_call', call_id=call_id), "type": "function_call",
  437. "status": "in_progress", "name": name, "call_id": call_id, "arguments": arguments_str}
  438. idx = await self._output_index(item['id'])
  439. self.pending_tool_calls.append({
  440. "call_id": call_id, "name": name, "arguments": arguments_str, "item_id": item["id"],
  441. "output_index": idx})
  442. self.emitted_items.append(
  443. {"type": "function_call", "name": name, "arguments": arguments_str, "call_id": call_id})
  444. await self.write_event("response.output_item.added", {
  445. "type": "response.output_item.added", "output_index": idx, "item": item})
  446. async def emit_tool_completed(self, payload: dict[str, Any]) -> None:
  447. """function_call ``output_item.done`` + function_call_output added/done; orphans skipped."""
  448. call_id = payload.get("tool_call_id")
  449. pending = next((p for p in self.pending_tool_calls if p["call_id"] == call_id), None)
  450. if not call_id or pending is None:
  451. return
  452. self.pending_tool_calls.remove(pending)
  453. done_item = {"id": pending["item_id"], "type": "function_call", "status": "completed",
  454. "name": pending["name"], "call_id": pending["call_id"],
  455. "arguments": pending["arguments"]}
  456. await self.write_event("response.output_item.done", {
  457. "type": "response.output_item.done", "output_index": pending["output_index"],
  458. "item": done_item})
  459. result = payload.get("result", "")
  460. result_str = result if isinstance(result, str) else json.dumps(result)
  461. output_parts = [{"type": "input_text", "text": result_str}]
  462. output_item = {"id": self._item_id('function_call_output', call_id=call_id), "type": "function_call_output",
  463. "call_id": pending["call_id"], "output": output_parts, "status": "completed"}
  464. idx = await self._output_index(output_item['id'])
  465. self.emitted_items.append(
  466. {"type": "function_call_output", "call_id": pending["call_id"], "output": output_parts})
  467. for event in ("response.output_item.added", "response.output_item.done"):
  468. await self.write_event(event, {"type": event, "output_index": idx, "item": output_item})
  469. async def emit_status(self, payload: dict[str, Any]) -> None:
  470. """Lifecycle/warning status (provider wait, auto-recovery countdown, fallback switch) as a
  471. ``hermes.status`` custom event; not a Responses output item."""
  472. await self.response.write(self._api._sse_frame(payload, event="hermes.status"))
  473. # queue tag -> (method name, payload adapter)
  474. _TAG_HANDLERS = {
  475. "__tool_started__": ("emit_tool_started", lambda p: p),
  476. "__tool_completed__": ("emit_tool_completed", lambda p: p),
  477. "__commentary__": ("emit_commentary", lambda p: p["text"]),
  478. "__reasoning__": ("emit_reasoning_delta", lambda p: p),
  479. "__status__": ("emit_status", lambda p: p),
  480. }
  481. async def dispatch(self, item: Any) -> None:
  482. """Route one queue item: tagged tuples emit immediately, strings are batched, others dropped."""
  483. if self.terminal_replay:
  484. return
  485. if isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str):
  486. tag, payload = item
  487. await self.flush_batch()
  488. handler = self._TAG_HANDLERS.get(tag)
  489. if handler is not None:
  490. method, adapt = handler
  491. await getattr(self, method)(adapt(payload))
  492. elif isinstance(item, str):
  493. self._batch_buf.append(item)
  494. if self._batch_timer is None:
  495. self._batch_timer = asyncio.create_task(self._batch_flush_after(0.05))
  496. async def _batch_flush_after(self, delay: float) -> None:
  497. try:
  498. await asyncio.sleep(delay)
  499. except asyncio.CancelledError:
  500. return
  501. # Clear the timer BEFORE flushing so new deltas can start a fresh timer while we emit.
  502. self._batch_timer = None
  503. await self.flush_batch()
  504. def cancel_batch_timer(self) -> None:
  505. if self._batch_timer and not self._batch_timer.done():
  506. self._batch_timer.cancel()
  507. self._batch_timer = None
  508. async def flush_batch(self) -> None:
  509. """Emit a single delta for all buffered text. Always takes the lock, even on an empty buffer: a
  510. timer flush may still be opening the message item (its keyed output index is read off the loop),
  511. and a caller that closes the item next must wait for that index."""
  512. async with self._batch_lock:
  513. if self._batch_buf:
  514. combined = "".join(self._batch_buf)
  515. self._batch_buf = []
  516. await self.emit_text_delta(combined)
  517. async def collect_result(self, agent_task) -> None:
  518. """Await the agent; when it produced a final_response but streamed no deltas
  519. (some providers only emit the full text at the end), emit one fallback delta."""
  520. try:
  521. result, agent_usage = await agent_task
  522. self.result = result
  523. self.usage = agent_usage or self.usage
  524. agent_final = result.get("final_response", "") if isinstance(result, dict) else ""
  525. if self.terminal_replay:
  526. # The receipt already contains the transformed final; it is not a raw provider
  527. # stream onto which a verifier/footer transform should be applied a second time.
  528. self.final_response_text = agent_final or self._api._redact_api_error_text(
  529. result.get('error', '(No response generated)'))
  530. self.final_text_parts = [self.final_response_text]
  531. return
  532. tail, appended = _post_stream_transform(result)
  533. if tail and self.final_text_parts:
  534. if appended:
  535. await self.emit_text_delta(tail)
  536. else:
  537. self.transformed_final = agent_final
  538. if agent_final and not self.final_text_parts:
  539. await self.emit_text_delta(agent_final)
  540. if agent_final and not self.final_response_text:
  541. self.final_response_text = agent_final
  542. if isinstance(result, dict) and result.get("error") and not self.final_response_text:
  543. self.agent_error = self._api._redact_api_error_text(result["error"])
  544. except Exception as e:
  545. logger.error("Error running agent for streaming responses: %s", e, exc_info=True)
  546. self.agent_error = self._api._redact_api_error_text(e)
  547. async def close_message_item(self) -> None:
  548. await self.close_reasoning_item()
  549. self.final_response_text = (
  550. self.transformed_final or "".join(self.final_text_parts) or self.final_response_text)
  551. if not self.message_opened:
  552. return
  553. await self.write_event("response.output_text.done", {
  554. "type": "response.output_text.done", "item_id": self.message_item_id,
  555. "output_index": self.message_output_index, "content_index": 0,
  556. "text": self.final_response_text, "logprobs": []})
  557. await self.write_event("response.output_item.done", {
  558. "type": "response.output_item.done", "output_index": self.message_output_index,
  559. "item": {"id": self.message_item_id, "type": "message", "status": "completed",
  560. "role": "assistant",
  561. "content": [{"type": "output_text", "text": self.final_response_text}]}})
  562. def _final_items(self) -> list[dict[str, Any]]:
  563. """Emitted items (trimmed) plus a final message item, so clients that only parse
  564. the terminal payload still see the assistant text (mirrors _extract_output_items)."""
  565. items = _trim_tool_items(list(self.emitted_items))
  566. redact = self._api._redact_api_error_text
  567. text = self.final_response_text or (redact(self.agent_error) if self.agent_error else "")
  568. items.append(_message_item(text))
  569. return items
  570. async def emit_failed(self) -> None:
  571. redact = self._api._redact_api_error_text
  572. env = self.terminal_envelope("failed", self._final_items(), error=redact(self.agent_error))
  573. history = self._history_with_user()
  574. history.append(
  575. {"role": "assistant", "content": self.final_response_text or redact(self.agent_error)})
  576. await self.persist_snapshot(env, history=history)
  577. await self.write_event("response.failed", {"type": "response.failed", "response": env})
  578. async def emit_completed(self) -> dict[str, Any]:
  579. result = self.result
  580. status = _response_status(result)
  581. output = self._final_items()
  582. if self.durable_key:
  583. start = self.adapter._response_messages_turn_start_index(self.conversation_history, self.user_message, result)
  584. output = self.adapter._extract_output_items(result, start_index=start, response_id=self.response_id)
  585. output = await self.response_store.off_loop(self.response_store.order_output, self.durable_key[0], output)
  586. env = self.terminal_envelope(status, output,
  587. error=self._api._redact_api_error_text(result.get('error') or 'The admitted turn failed.')
  588. if status == 'failed' else None)
  589. full_history = self.adapter._build_response_conversation_history(
  590. self.conversation_history, self.user_message, result, self.final_response_text,
  591. tool_output_max_chars=self.adapter._history_tool_output_max_chars)
  592. # Transcript substitution for result["_compressed"] happens in the history builder; only
  593. # a compression-rotated session_id is propagated so chaining resumes the child session.
  594. sid = result.get("session_id") if isinstance(result, dict) else None
  595. record = None
  596. if self.durable_key and not self.terminal_replay:
  597. # Only the publisher recorded the deltas; a replay observer winning the
  598. # INSERT OR IGNORE would leave every later replay without them. Built here, on the
  599. # loop, so the worker never reads headers or events the loop may still append to.
  600. headers = {k: v for k, v in self.response.headers.items() if k.startswith('X-Hermes-')}
  601. terminal_event = {'type': 'response.' + status, 'response': env, 'sequence_number': self.sequence_number}
  602. record = {'fingerprint': self.durable_key[1], 'response': env, 'headers': headers,
  603. 'events': [*self.recorded_events, ('response.' + status, terminal_event)]}
  604. stored = await self.response_store.off_loop(
  605. self._commit_terminal, env, full_history, sid if isinstance(sid, str) and sid else None, record)
  606. if self.durable_key:
  607. if _settled(stored):
  608. env = stored['response']
  609. status = env['status']
  610. await self.write_event(
  611. 'response.' + status, {'type': 'response.' + status, 'response': env})
  612. return env
  613. def _commit_terminal(self, env, full_history, session_id, record):
  614. """The terminal snapshot plus, when keyed, the replay record and its read-back: one store
  615. unit, so no other observer of the same key interleaves between them. -> stored record.
  616. ``store: false`` skips only the snapshot; the keyed replay record is idempotency state."""
  617. if self.store:
  618. self._persist_snapshot(env, full_history, session_id)
  619. if not self.durable_key:
  620. return None
  621. if record is not None:
  622. self.response_store.put(self.durable_key[0], record)
  623. return self.response_store.get(self.durable_key[0])
  624. async def emit_crash(self, exc: BaseException) -> None:
  625. error = self._api._redact_api_error_text(exc, limit=500)
  626. env = self.terminal_envelope("failed", list(self.emitted_items), error=error)
  627. await self.write_event("response.failed", {"type": "response.failed", "response": env})
  628. def _parse_chat_messages(messages: list[Any]) -> tuple:
  629. """Chat ``messages`` -> ``(system_prompt, conversation_messages, error_response_or_None)``.
  630. System messages -> ephemeral system prompt layered ON TOP of core, flattened to text
  631. (Anthropic rejects images there, OpenAI text models ignore them)."""
  632. from gateway.platforms.api_server import (
  633. _multimodal_validation_error, _normalize_chat_content, _normalize_multimodal_content)
  634. system_prompt = None
  635. conversation_messages: list[dict[str, str]] = []
  636. for idx, msg in enumerate(messages):
  637. role = msg.get("role", "")
  638. raw_content = msg.get("content", "")
  639. if role == "system":
  640. content = _normalize_chat_content(raw_content)
  641. system_prompt = content if system_prompt is None else system_prompt + "\n" + content
  642. elif role in {"user", "assistant"}:
  643. try:
  644. content = _normalize_multimodal_content(raw_content)
  645. except ValueError as exc:
  646. return None, None, _multimodal_validation_error(exc, param=f"messages[{idx}].content")
  647. conversation_messages.append({"role": role, "content": content})
  648. return system_prompt, conversation_messages, None
  649. def _parse_responses_input(raw_input: Any) -> tuple:
  650. """Responses ``input`` (string or item array) -> ``(input_messages, error_response_or_None)``."""
  651. from gateway.platforms.api_server import (
  652. _error_response, _multimodal_validation_error, _normalize_multimodal_content)
  653. input_messages: list[dict[str, Any]] = []
  654. if isinstance(raw_input, str):
  655. input_messages = [{"role": "user", "content": raw_input}]
  656. elif isinstance(raw_input, list):
  657. for idx, item in enumerate(raw_input):
  658. if isinstance(item, str):
  659. input_messages.append({"role": "user", "content": item})
  660. elif _is_reasoning_input_item(item):
  661. continue
  662. elif isinstance(item, dict):
  663. try:
  664. content = _normalize_multimodal_content(item.get("content", ""))
  665. except ValueError as exc:
  666. return None, _multimodal_validation_error(exc, param=f"input[{idx}].content")
  667. input_messages.append({"role": item.get("role", "user"), "content": content})
  668. else:
  669. return None, _error_response("'input' must be a string or array", 400)
  670. return input_messages, None
  671. def _parse_conversation_history(raw_history: Any) -> tuple:
  672. """Explicit Responses ``conversation_history`` -> ``(history, error_response_or_None)``."""
  673. from gateway.platforms.api_server import (
  674. _error_response, _multimodal_validation_error, _normalize_multimodal_content)
  675. conversation_history: list[dict[str, Any]] = []
  676. if not isinstance(raw_history, list):
  677. return None, _error_response("'conversation_history' must be an array of message objects", 400)
  678. for i, entry in enumerate(raw_history):
  679. if _is_reasoning_input_item(entry):
  680. continue
  681. if not isinstance(entry, dict) or "role" not in entry or "content" not in entry:
  682. return None, _error_response(f"conversation_history[{i}] must have 'role' and 'content' fields", 400)
  683. try:
  684. entry_content = _normalize_multimodal_content(entry["content"])
  685. except ValueError as exc:
  686. return None, _multimodal_validation_error(exc, param=f"conversation_history[{i}].content")
  687. conversation_history.append({"role": str(entry["role"]), "content": entry_content})
  688. return conversation_history, None
  689. class OpenAICompatRoutesMixin:
  690. """/v1/chat/completions and /v1/responses handlers + SSE writers."""
  691. def _select_request_route(
  692. self, body: dict[str, Any], *, session_id, gateway_session_key, model_alias) -> tuple:
  693. """Resolve the model_routes alias + per-request overrides ->
  694. ``(route, agent_overrides, error_response_or_None)``."""
  695. from gateway.platforms.api_server import _error_response, _request_agent_overrides
  696. route = self._resolve_route(model_alias)
  697. overrides = _request_agent_overrides(
  698. body, virtual_model=self._model_name, allow_bare_model=self._direct_model_requests)
  699. err = self._request_route_conflict_error(
  700. session_id=session_id, gateway_session_key=gateway_session_key,
  701. requested_model=overrides.get("requested_model"),
  702. requested_provider=overrides.get("requested_provider"), route=route)
  703. return route, overrides, (_error_response(err, 400) if err else None)
  704. def _register_stream_approval(self, request, completion_id, stream_q, session_id) -> tuple:
  705. """Expose a streaming completion as a run for ``POST /v1/runs/{completion_id}/approval``
  706. (#51871) -> ``(notify, on_done)``. Owner + status are stamped like the session stream, so
  707. ``_load_owned_run`` finds it; keyed by the completion id (never the shared session key) so
  708. concurrent turns can't cross-resolve. ``on_done`` retires it when the agent task ends."""
  709. from gateway.platforms.api_server import _approval_request_event
  710. self._run_owners[completion_id] = self._run_idempotency_scope(request)
  711. self._set_run_status(completion_id, "running", session_id=session_id or "")
  712. self._run_approval_sessions[completion_id] = completion_id
  713. def _approval_notify(approval_data):
  714. event = _approval_request_event(completion_id, approval_data, session_id=session_id or "")
  715. self._set_run_status(completion_id, "waiting_for_approval", last_event="approval.request",
  716. approval=event)
  717. stream_q.put_threadsafe(("__approval__", event))
  718. def _on_done(fut):
  719. from gateway.platforms.api_server_runs import terminal_run_status
  720. self._run_approval_sessions.pop(completion_id, None)
  721. if fut.cancelled():
  722. status = "cancelled"
  723. elif fut.exception() is not None:
  724. status = "failed"
  725. else:
  726. result = (fut.result() or (None,))[0]
  727. status = terminal_run_status(result)[0] if isinstance(result, dict) else "completed"
  728. self._set_run_status(completion_id, status)
  729. self._release_run_owner_if_forgotten(completion_id)
  730. return _approval_notify, _on_done
  731. def _spawn_stream_agent(self, stream_q, *, on_done=None, run_agent=None, **run_kwargs) -> tuple:
  732. """Start ``_run_agent`` for an SSE writer -> ``(agent_task, agent_ref)``. ``agent_ref[0]``
  733. lets the writer interrupt on disconnect; the EOS sentinel is enqueued from the task's done
  734. callback so drain loops never race a polled ``agent_task.done()``. ``on_done(task)`` runs
  735. in that same callback, before EOS."""
  736. def _on_delta(delta):
  737. # None from the agent is a CLI box-close signal, not EOS — forwarding it would end
  738. # the stream early. Called from the run_conversation worker thread: put_threadsafe.
  739. if delta is not None:
  740. stream_q.put_threadsafe(delta)
  741. def _on_reasoning(text):
  742. # Structured reasoning deltas (#99552): the agent's reasoning_callback, not the
  743. # lossy 500-char ``reasoning.available`` progress preview. Tagged so the writers
  744. # keep them distinct from answer text.
  745. if text:
  746. stream_q.put_threadsafe(("__reasoning__", text))
  747. def _on_status(kind, message=None):
  748. # Lifecycle/warning status (provider wait, auto-recovery countdown, fallback switch) as a
  749. # ``hermes.status`` event, so a client sees why the stream is silent instead of a dead socket.
  750. from gateway.platforms.api_server import _redact_api_error_text
  751. text = _redact_api_error_text(message if message is not None else kind or "").strip()
  752. if text:
  753. stream_q.put_threadsafe(("__status__", {"kind": str(kind), "text": text}))
  754. agent_ref = [None]
  755. agent_task = asyncio.ensure_future((run_agent or self._run_agent)(
  756. stream_delta_callback=_on_delta, reasoning_callback=_on_reasoning, status_callback=_on_status,
  757. agent_ref=agent_ref, **run_kwargs))
  758. def _done(fut):
  759. try:
  760. if on_done is not None:
  761. on_done(fut)
  762. except Exception:
  763. logger.debug("[api_server] stream on_done hook failed", exc_info=True)
  764. finally:
  765. stream_q.put_nowait(None)
  766. agent_task.add_done_callback(_done)
  767. return agent_task, agent_ref
  768. async def _handle_chat_completions(self, request: web.Request) -> web.Response:
  769. """POST /v1/chat/completions — OpenAI Chat Completions format."""
  770. from gateway.platforms.api_server import (
  771. ThreadSafeAsyncQueue, _api_request_profile, _chat_usage_payload, _coerce_request_bool,
  772. _content_has_visible_payload, _derive_chat_session_id, _error_response, _invalid_request,
  773. _openai_error, _redact_api_error_text, _resolve_media_to_data_urls)
  774. # Bound total in-flight agent runs (configurable; #7483).
  775. limited = self._concurrency_limited_response()
  776. if limited is not None:
  777. return limited
  778. try:
  779. body = await request.json()
  780. except Exception:
  781. return _error_response("Invalid JSON in request body", 400)
  782. from gateway.platforms.api_server import _request_relay_metadata
  783. relay_metadata = _request_relay_metadata(body)
  784. messages = body.get("messages")
  785. if not messages or not isinstance(messages, list):
  786. return _invalid_request("Missing or invalid 'messages' field")
  787. stream = _coerce_request_bool(body.get("stream"), default=False)
  788. system_prompt, conversation_messages, messages_err = _parse_chat_messages(messages)
  789. if messages_err is not None:
  790. return messages_err
  791. user_message: Any = (conversation_messages[-1].get("content", "") if conversation_messages else "")
  792. history = conversation_messages[:-1]
  793. if not _content_has_visible_payload(user_message):
  794. return _invalid_request("No user message found in messages")
  795. # X-Hermes-Session-Key scopes long-term memory per channel; independent of
  796. # X-Hermes-Session-Id (the key persists across transcripts, the id rotates on /new).
  797. gateway_session_key, key_err = self._parse_session_key_header(request)
  798. if key_err is not None:
  799. return key_err
  800. # X-Hermes-Session-Id continues an existing session (history from state.db, not the body);
  801. # requires a configured API key or any client could read history by guessing ids.
  802. provided_session_id = request.headers.get("X-Hermes-Session-Id", "").strip()
  803. if provided_session_id:
  804. if not self._api_key:
  805. logger.warning(
  806. "Session continuation via X-Hermes-Session-Id rejected: "
  807. "no API key configured. Set API_SERVER_KEY to enable "
  808. "session continuity.")
  809. return _error_response("Session continuation requires API key authentication. "
  810. "Configure API_SERVER_KEY to enable this feature.", 403)
  811. # Same guard as the native gateway: ids are interpolated into on-disk filenames.
  812. from gateway.session import _is_path_unsafe
  813. if re.search(r'[\r\n\x00]', provided_session_id) or _is_path_unsafe(provided_session_id):
  814. return _invalid_request("Invalid session ID")
  815. if len(provided_session_id) > self._MAX_SESSION_HEADER_LEN:
  816. return _invalid_request("Session ID too long")
  817. session_id = provided_session_id
  818. try:
  819. db = await self._ensure_session_db_async()
  820. if db is not None:
  821. # #98619/#13437: a client-addressed id from before a compression rotation
  822. # must adopt the live continuation tip — history loads from it, the turn and
  823. # the wake target bind it, and a detached delegation delivery row persisted
  824. # on the tip is what this continuation consumes. Same canonical resolution
  825. # the delivery writer (gateway/wake.py) and /v1/runs use; fails open.
  826. from gateway.platforms.api_server_runs import _resolve_live_session_id
  827. session_id = await _resolve_live_session_id(self, provided_session_id)
  828. history = await asyncio.to_thread(db.get_messages_as_conversation, session_id)
  829. except Exception as e:
  830. logger.warning("Failed to load session history for %s: %s", session_id, e)
  831. history = []
  832. else:
  833. # Stable id from the conversation fingerprint so Open WebUI-style clients map onto
  834. # one Hermes session; namespaced by the routed profile (#123989).
  835. first_user = next(
  836. (cm.get("content", "") for cm in conversation_messages if cm.get("role") == "user"), "")
  837. session_id = _derive_chat_session_id(system_prompt, first_user, _api_request_profile.get())
  838. session_id, lookup_error = await self._implicit_chat_session(request, session_id)
  839. if lookup_error is not None:
  840. return lookup_error
  841. completion_id = f"chatcmpl-{uuid.uuid4().hex[:29]}"
  842. model_name = body.get("model", self._model_name)
  843. created = int(time.time())
  844. route, agent_overrides, selection_error = self._select_request_route(
  845. body, session_id=session_id, gateway_session_key=gateway_session_key,
  846. model_alias=model_name)
  847. if selection_error is not None:
  848. return selection_error
  849. run_kwargs = dict(
  850. user_message=user_message, conversation_history=history,
  851. ephemeral_system_prompt=system_prompt, session_id=session_id,
  852. gateway_session_key=gateway_session_key, **agent_overrides, route=route,
  853. relay_metadata=relay_metadata,
  854. # #98619: only an explicitly provided X-Hermes-Session-Id is wake-capable (the
  855. # header is 403-gated on API_SERVER_KEY, so the wake self-post can authenticate
  856. # and the client can resume the session by sending it again). A fingerprint-derived
  857. # id from a header-less client is NOT: delegate_task keeps its forced-sync fallback
  858. # there — the wake would hard-fail or land in history that client never reloads.
  859. session_history_delivery=("1" if provided_session_id else ""))
  860. run_kwargs.update(self._authority_chat_kwargs(request, provided_session_id))
  861. # This is presentation only. The ordinary API-key/session authorization
  862. # above still applies; it grants no internal ingress or control authority.
  863. if provided_session_id and body.get("hermes_notification_category") == "diagnostic":
  864. run_kwargs["notification_category"] = "diagnostic"
  865. if stream:
  866. _stream_q = ThreadSafeAsyncQueue()
  867. # tool_call_ids with an emitted "running": a "completed" without one (internal/
  868. # filtered tools) is dropped rather than orphaned on the wire.
  869. _started_tool_call_ids: set[str] = set()
  870. def _on_tool_start(tool_call_id, function_name, function_args):
  871. """``hermes.tool.progress`` status=running; ``_``-prefixed tools stay off the wire."""
  872. if not tool_call_id or function_name.startswith("_"):
  873. return
  874. _started_tool_call_ids.add(tool_call_id)
  875. from agent.display import build_tool_preview, get_tool_emoji
  876. label = build_tool_preview(function_name, function_args) or function_name
  877. _stream_q.put_threadsafe(("__tool_progress__", {
  878. "tool": function_name, "emoji": get_tool_emoji(function_name), "label": label,
  879. "toolCallId": tool_call_id, "status": "running"}))
  880. def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
  881. if not tool_call_id or tool_call_id not in _started_tool_call_ids:
  882. return
  883. _started_tool_call_ids.discard(tool_call_id)
  884. _stream_q.put_threadsafe(("__tool_progress__", {
  885. "tool": function_name, "toolCallId": tool_call_id, "status": "completed"}))
  886. # tool_progress_callback deliberately NOT wired: it would duplicate the structured
  887. # start/complete callbacks (which carry the tool_call id).
  888. # The completion id doubles as the run id; on_done drops the approval mapping once the
  889. # turn ends (POST /v1/runs/{id}/approval then answers 409) and sets the terminal status.
  890. approval_notify, end_stream_run = self._register_stream_approval(
  891. request, completion_id, _stream_q, session_id)
  892. agent_task, agent_ref = self._spawn_stream_agent(
  893. _stream_q, on_done=end_stream_run, tool_start_callback=_on_tool_start,
  894. tool_complete_callback=_on_tool_complete, approval_notify_callback=approval_notify,
  895. approval_session_key=completion_id, **run_kwargs)
  896. # #13437 identity contract: an explicit-header client keeps addressing the id it
  897. # sent; the response echoes that stable id while reads/writes adopt the live tip,
  898. # so a rotation mid-turn (after these headers are prepared) never changes what the
  899. # client should send next — it re-sends the same id and the tip resolution above
  900. # finds whatever session is live by then.
  901. return await self._write_sse_chat_completion(
  902. request, completion_id, model_name, created, _stream_q,
  903. agent_task, agent_ref, session_id=(provided_session_id or session_id),
  904. gateway_session_key=gateway_session_key)
  905. async def _compute_completion():
  906. return await self._run_agent(**run_kwargs)
  907. outcome, err = await self._run_idempotent(
  908. request, body, _compute_completion, log_label="chat completions",
  909. fingerprint_keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream",
  910. "hermes_notification_category"],
  911. route="chat_completions",
  912. )
  913. if err is not None:
  914. return err
  915. result, usage = outcome
  916. presentation_muted = result.get("_notification_presentation_suppressed") is True
  917. final_response = _resolve_media_to_data_urls(result.get("final_response") or "")
  918. completed, is_partial, is_failed, err_msg = _result_flags(result)
  919. if err_msg:
  920. err_msg = _redact_api_error_text(err_msg)
  921. budget_exhausted = _budget_exhausted(result)
  922. finish_reason = _finish_reason(completed, is_partial, is_failed, err_msg, budget_exhausted=budget_exhausted)
  923. # Same #13437 identity contract as the SSE path: an explicit-header client is echoed
  924. # the stable id it sent; a fingerprint-derived (header-less) turn keeps reporting the
  925. # id the agent actually resolved, so headerless clients still learn where the turn went.
  926. response_headers = {"X-Hermes-Session-Id": (provided_session_id or result.get("session_id", session_id))}
  927. if gateway_session_key:
  928. response_headers["X-Hermes-Session-Key"] = gateway_session_key
  929. # Hard fail (no usable text AND a real failure) -> 502 OpenAI error envelope so SDK
  930. # clients raise instead of rendering the failure string as message.content.
  931. if not final_response and (is_failed or is_partial):
  932. err_body = _openai_error(
  933. "" if presentation_muted else (err_msg or "Agent run did not produce a response."), err_type="server_error",
  934. code="agent_incomplete")
  935. err_body["error"]["hermes"] = {
  936. "completed": completed, "partial": is_partial, "failed": is_failed}
  937. response_headers["X-Hermes-Completed"] = "false"
  938. response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
  939. return web.json_response(err_body, status=502, headers=response_headers)
  940. # Soft partial (some text, run incomplete): 200 + finish_reason="length"/Hermes extras.
  941. response_data = {
  942. "id": completion_id, "object": "chat.completion", "created": created,
  943. "model": model_name,
  944. "choices": [{"index": 0, "message": {"role": "assistant", "content": "" if presentation_muted else final_response},
  945. "finish_reason": finish_reason}],
  946. "usage": _chat_usage_payload(usage)}
  947. # Non-streaming twin of ``delta.reasoning_content`` (#99552).
  948. reasoning_text = _turn_reasoning_text(history, user_message, result)
  949. if reasoning_text and not presentation_muted:
  950. response_data["choices"][0]["message"]["reasoning_content"] = reasoning_text
  951. if is_partial or is_failed or not completed:
  952. response_data["hermes"] = _hermes_extras(
  953. completed, is_partial, is_failed, "" if presentation_muted else err_msg, finish_reason,
  954. budget_exhausted=budget_exhausted)
  955. response_headers["X-Hermes-Completed"] = "false"
  956. response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
  957. if err_msg and not presentation_muted:
  958. response_headers["X-Hermes-Error"] = _redact_api_error_text(err_msg, limit=200)
  959. return web.json_response(response_data, headers=response_headers)
  960. def _chat_request_id(self, request: web.Request) -> Optional[str]:
  961. """Durable admission identity of a Chat Completions Idempotency-Key. It carries the
  962. authenticated namespace: a rotated API key (or another profile) reusing a client key is a
  963. new principal, not a retry."""
  964. key = request.headers.get('Idempotency-Key')
  965. return f'chat:{self._run_idempotency_scope(request)}:{key}' if key else None
  966. def _authority_chat_kwargs(self, request: web.Request, provided_session_id) -> dict:
  967. """Admission identity of a Chat Completions turn under session authority; none before it."""
  968. if getattr(self.gateway_runner, 'session_authority', None) is None:
  969. return {}
  970. return dict(request_id=self._chat_request_id(request), history_from_session=bool(provided_session_id),
  971. run_owner_scope=self._run_idempotency_scope(request))
  972. async def _implicit_chat_session(self, request: web.Request, session_id: str) -> tuple:
  973. """A derived (header-less) id whose session was deleted names a new conversation with the
  974. same opening: the owning profile's store picks its generation (``implicit_api_session``).
  975. -> ``(session_id, None)`` or ``(session_id, error response)``, before either response starts:
  976. a refusal keeps its reason and a busy or unreadable store is a retryable 503
  977. ``storage_unavailable`` (``store_error_response``); anything else is the OpenAI 500."""
  978. if getattr(self.gateway_runner, 'session_authority', None) is None:
  979. return session_id, None
  980. from gateway.session_api import implicit_api_session
  981. from gateway.session_authorities import active_authority
  982. authority = active_authority(self.gateway_runner)
  983. if authority is None:
  984. return session_id, None # admission refuses the unserved profile with its own verdict
  985. import sqlite3
  986. from hermes_state_runtime import RuntimeStoreError
  987. try:
  988. return await asyncio.to_thread(implicit_api_session, authority.db, session_id,
  989. self._chat_request_id(request)), None
  990. except (RuntimeStoreError, sqlite3.Error, OSError) as exc:
  991. from gateway.platforms.api_server import _openai_error
  992. from gateway.platforms.api_server_authority_runs import store_error_response
  993. return session_id, store_error_response(_openai_error, exc)
  994. except Exception as e:
  995. from gateway.platforms.api_server import _error_response
  996. logger.error("Error resolving the chat completions session: %s", e, exc_info=True)
  997. return session_id, _error_response(f"Internal server error: {e}", 500, err_type="server_error")
  998. async def _run_idempotent(
  999. self, request: web.Request, body: dict[str, Any], compute, *,
  1000. log_label: str, fingerprint_keys: list[str], route: str) -> tuple:
  1001. """Run ``compute()`` once per (principal scope, logical route, Idempotency-Key) + body fingerprint
  1002. -> ``((result, usage), None)`` or ``(None, 500 response)``.
  1003. ``_idem_cache`` is process-global: under ``gateway.multiplex_profiles`` every profile's
  1004. ``/p/<profile>/v1/...`` mirror shares it, so the key carries ``_run_idempotency_scope`` (the same
  1005. ``sha256(profile, expected API key)`` namespace the durable ``/v1/runs`` API uses) — a client key
  1006. colliding across profiles, or a rotated API_SERVER_KEY, never replays another principal's response.
  1007. ``route`` is the logical endpoint (``/v1/...`` and its ``/p/<profile>/v1/...`` alias are the same
  1008. route), folded into the key because the store keeps the fingerprint only as the slot's value.
  1009. """
  1010. from gateway.platforms.api_server import _error_response, _idem_cache, _make_request_fingerprint
  1011. from hermes_state_runtime import RuntimeStoreError
  1012. idempotency_key = request.headers.get("Idempotency-Key")
  1013. try:
  1014. # Durable /v1/runs admissions own idempotency under session authority; the in-memory
  1015. # cache only serves the pre-authority path.
  1016. if idempotency_key and getattr(self.gateway_runner, 'session_authority', None) is None:
  1017. principal_scope = self._run_idempotency_scope(request)
  1018. scoped_key = f"{principal_scope}\0{route}\0{idempotency_key}"
  1019. fp = _make_request_fingerprint(body, keys=fingerprint_keys)
  1020. result, usage = await _idem_cache.get_or_set(scoped_key, fp, compute)
  1021. else:
  1022. result, usage = await compute()
  1023. return (result, usage), None
  1024. except RuntimeStoreError as exc:
  1025. from gateway.platforms.api_server import _openai_error
  1026. from gateway.platforms.api_server_authority_runs import store_error_response
  1027. return None, store_error_response(_openai_error, exc)
  1028. except Exception as e:
  1029. logger.error("Error running agent for %s: %s", log_label, e, exc_info=True)
  1030. message = "" if getattr(e, "_notification_presentation_suppressed", False) is True else f"Internal server error: {e}"
  1031. return None, _error_response(message, 500, err_type="server_error")
  1032. async def _prepare_sse_response(
  1033. self, request: web.Request, session_id: Optional[str], gateway_session_key: Optional[str],
  1034. ) -> web.StreamResponse:
  1035. """Open a prepared SSE StreamResponse with CORS + session headers (the CORS middleware
  1036. can't inject headers after ``prepare()`` flushes them, so they are resolved here)."""
  1037. sse_headers = self._sse_headers(request)
  1038. if session_id:
  1039. sse_headers["X-Hermes-Session-Id"] = session_id
  1040. if gateway_session_key:
  1041. sse_headers["X-Hermes-Session-Key"] = gateway_session_key
  1042. response = web.StreamResponse(status=200, headers=sse_headers)
  1043. await response.prepare(request)
  1044. return response
  1045. async def _write_sse_chat_completion(
  1046. self, request: web.Request, completion_id: str, model: str,
  1047. created: int, stream_q, agent_task, agent_ref=None, session_id: str | None = None,
  1048. gateway_session_key: str | None = None) -> web.StreamResponse:
  1049. """Stream ``chat.completion.chunk`` frames from the agent's delta queue. On client
  1050. disconnect the agent is interrupted (stops LLM calls), then its task wrapper cancelled."""
  1051. from gateway.platforms.api_server import (
  1052. _abandon_agent_task, _chat_usage_payload, _resolve_media_to_data_urls, _sse_frame)
  1053. response = await self._prepare_sse_response(request, session_id, gateway_session_key)
  1054. def _chunk(delta: dict[str, Any], finish_reason=None, **extra) -> dict[str, Any]:
  1055. return {"id": completion_id, "object": "chat.completion.chunk", "created": created,
  1056. "model": model,
  1057. "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], **extra}
  1058. content_sent = False
  1059. try:
  1060. await response.write(_sse_frame(_chunk({"role": "assistant"})))
  1061. async for delta in _iter_stream_items(stream_q, agent_task, response):
  1062. if delta is None:
  1063. break
  1064. if isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__tool_progress__":
  1065. # Custom event: tool lifecycle for frontends without markers in history.
  1066. if not self._tool_progress_events:
  1067. continue # opted out for strict OpenAI clients (#12020)
  1068. await response.write(_sse_frame(delta[1], event="hermes.tool.progress"))
  1069. elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__reasoning__":
  1070. # DeepSeek-style ``delta.reasoning_content`` (#99552), the field Open WebUI,
  1071. # opencode and the Vercel AI SDK render as a thinking block.
  1072. await response.write(_sse_frame(_chunk({"reasoning_content": delta[1]})))
  1073. elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__status__":
  1074. await response.write(_sse_frame(delta[1], event="hermes.status"))
  1075. elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__approval__":
  1076. await response.write(_sse_frame(delta[1], event="approval.request"))
  1077. else:
  1078. if delta:
  1079. content_sent = True
  1080. await response.write(_sse_frame(_chunk({"content": delta})))
  1081. # The agent can fail after the queue drains (task raises / result flagged failed or
  1082. # partial): surface a non-"stop" finish_reason like the non-streaming path.
  1083. usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
  1084. result = agent_error = None
  1085. try:
  1086. result, agent_usage = await agent_task
  1087. usage = agent_usage or usage
  1088. except Exception as exc:
  1089. agent_error = exc
  1090. logger.error("Agent task %s failed during SSE streaming: %s", completion_id, exc)
  1091. completed, is_partial, is_failed, err_msg = _result_flags(result)
  1092. if agent_error is not None:
  1093. is_failed = True
  1094. err_msg = err_msg or str(agent_error)
  1095. budget_exhausted = _budget_exhausted(result)
  1096. finish_reason = _finish_reason(completed, is_partial, is_failed, err_msg, agent_error,
  1097. budget_exhausted=budget_exhausted)
  1098. finish_chunk = _chunk({}, finish_reason, usage=_chat_usage_payload(usage))
  1099. presentation_muted = (
  1100. (isinstance(result, dict) and result.get("_notification_presentation_suppressed") is True)
  1101. or getattr(agent_error, "_notification_presentation_suppressed", False) is True
  1102. )
  1103. # Recovery paths (guardrail halt, partial_stream_recovery, fallback prior-turn
  1104. # content) can return a final_response without firing any content delta; emit it
  1105. # once so the client does not see an empty stream (#31449). Mirrors
  1106. # _ResponsesStream.collect_result for /v1/responses.
  1107. if not content_sent and not presentation_muted and isinstance(result, dict):
  1108. fallback_text = _resolve_media_to_data_urls(result.get("final_response") or "")
  1109. if fallback_text:
  1110. await response.write(_sse_frame(_chunk({"content": fallback_text})))
  1111. elif not presentation_muted:
  1112. # Chat chunks can only append: a non-append rewrite follows the streamed text (as in the CLI).
  1113. tail, appended = _post_stream_transform(result)
  1114. if tail:
  1115. await response.write(_sse_frame(_chunk({"content": tail if appended else _transformed_notice() + tail})))
  1116. if finish_reason != "stop":
  1117. if err_msg and not presentation_muted:
  1118. finish_chunk["error"] = {
  1119. "message": err_msg,
  1120. "type": type(agent_error).__name__ if agent_error else "agent_error"}
  1121. finish_chunk["hermes"] = _hermes_extras(
  1122. completed, is_partial, is_failed, "" if presentation_muted else err_msg, finish_reason,
  1123. budget_exhausted=budget_exhausted)
  1124. await response.write(_sse_frame(finish_chunk))
  1125. await response.write(b"data: [DONE]\n\n")
  1126. except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
  1127. await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
  1128. logger.info("SSE client disconnected; interrupted agent task %s", completion_id)
  1129. except Exception:
  1130. # Agent crashed mid-stream: an error chunk beats a TransferEncodingError.
  1131. import traceback as _tb
  1132. logger.error("Agent crashed mid-stream for %s: %s", completion_id, _tb.format_exc()[:300])
  1133. with suppress(Exception):
  1134. await response.write(_sse_frame(_chunk({}, "error")))
  1135. await response.write(b"data: [DONE]\n\n")
  1136. return response
  1137. async def _write_sse_responses(
  1138. self, request: web.Request, response_id: str, model: str, created_at: int, stream_q,
  1139. agent_task, agent_ref, conversation_history: list[dict[str, str]], user_message: str,
  1140. instructions: Optional[str], conversation: Optional[str], store: bool, session_id: str,
  1141. gateway_session_key: Optional[str] = None, durable_key=None, terminal_replay=False) -> web.StreamResponse:
  1142. """Write the SSE stream for POST /v1/responses.
  1143. Events: ``response.created`` -> ``output_text.delta/done`` + ``output_item.added/done``
  1144. (reasoning / function_call / function_call_output) + ``reasoning_summary_part/text.*``
  1145. -> ``response.completed`` (non-streaming envelope)
  1146. or ``response.failed``. On disconnect the agent is interrupted and, with ``store=True``,
  1147. an ``incomplete`` snapshot replaces ``in_progress`` so GET / chaining still work.
  1148. ``durable_key`` (canonical Idempotency-Key record) is written only once the admitted turn
  1149. settled, so an exact retry replays this stream instead of admitting different history.
  1150. """
  1151. from gateway.platforms.api_server import _abandon_agent_task, _redact_api_error_text
  1152. response = await self._prepare_sse_response(request, session_id, gateway_session_key)
  1153. st = _ResponsesStream(
  1154. self, response, response_id=response_id, model=model, created_at=created_at,
  1155. conversation_history=conversation_history, user_message=user_message,
  1156. instructions=instructions, conversation=conversation, store=store, session_id=session_id,
  1157. durable_key=durable_key, terminal_replay=terminal_replay)
  1158. try:
  1159. await st.emit_created()
  1160. async for item in _iter_stream_items(stream_q, agent_task, response):
  1161. if item is None: # EOS sentinel
  1162. st.cancel_batch_timer()
  1163. await st.flush_batch()
  1164. break
  1165. await st.dispatch(item)
  1166. await st.flush_batch()
  1167. await st.collect_result(agent_task)
  1168. await st.close_message_item()
  1169. if st.agent_error and not (durable_key and st.result):
  1170. await st.emit_failed()
  1171. else:
  1172. await st.emit_completed()
  1173. except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
  1174. await st.persist_incomplete_if_needed()
  1175. await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
  1176. logger.info("SSE client disconnected; interrupted agent task %s", response_id)
  1177. except asyncio.CancelledError:
  1178. # Server-side cancellation (shutdown, timeout): abandon the agent first (a second cancel
  1179. # landing in the persist await must not skip it), persist incomplete, then re-raise.
  1180. await _abandon_agent_task(
  1181. agent_ref, agent_task, "SSE task cancelled",
  1182. reap_source="api_server_sse_cancelled", await_cancel=False)
  1183. await st.persist_incomplete_if_needed()
  1184. logger.info("SSE task cancelled; persisted incomplete snapshot for %s", response_id)
  1185. raise
  1186. except Exception as exc:
  1187. # Unhandled agent error (BadRequestError, AuthenticationError, ...): emit
  1188. # response.failed and end the stream cleanly (no TransferEncodingError).
  1189. import traceback as _tb
  1190. await st.persist_incomplete_if_needed()
  1191. st.agent_error = _redact_api_error_text(_tb.format_exc())
  1192. with suppress(Exception):
  1193. await st.emit_crash(exc)
  1194. logger.error("Agent crashed mid-stream for %s: %s", response_id, str(st.agent_error)[:300])
  1195. return response
  1196. async def _replay_sse_responses(self, request: web.Request, replay: dict[str, Any]) -> web.StreamResponse:
  1197. """Re-emit a settled idempotent record as SSE. A record made by the streaming path replays
  1198. its exact frames; one made by the nonstreaming path gets ``created`` + its terminal event."""
  1199. from gateway.platforms.api_server import _sse_frame
  1200. headers = replay.get('headers') or {}
  1201. response = await self._prepare_sse_response(
  1202. request, headers.get('X-Hermes-Session-Id'), headers.get('X-Hermes-Session-Key'))
  1203. events = replay.get('events')
  1204. if not events:
  1205. env = replay['response']
  1206. created = {k: env[k] for k in ('id', 'object', 'created_at', 'model')}
  1207. created.update(status='in_progress', output=[])
  1208. events = [['response.created', {'type': 'response.created', 'response': created, 'sequence_number': 0}],
  1209. ['response.' + env['status'], {'type': 'response.' + env['status'], 'response': env,
  1210. 'sequence_number': 1}]]
  1211. for event_type, data in events:
  1212. await response.write(_sse_frame(data, event=event_type))
  1213. return response
  1214. async def _responses_durable_replay(self, request: web.Request, body: dict[str, Any], stream: bool) -> tuple:
  1215. """Canonical Idempotency-Key record for POST /v1/responses ->
  1216. ``(durable_key, idempotency_scope, idempotency_key, replayed_response_or_None)``."""
  1217. durable_key = idempotency_scope = None
  1218. idempotency_key = request.headers.get('Idempotency-Key')
  1219. canonical = getattr(self.gateway_runner, 'session_authority', None) is not None
  1220. if canonical and idempotency_key:
  1221. # An exact retry replays the committed response BEFORE the conversation name is
  1222. # expanded: the first success already advanced the conversation, so re-expanding
  1223. # would build a different admission payload and refuse the retry as a conflict.
  1224. # Streaming and nonstreaming share the record: transport mode is not identity.
  1225. from gateway.platforms.api_server import _make_request_fingerprint
  1226. idempotency_scope = self._run_idempotency_scope(request)
  1227. durable_key = (f'idem:{idempotency_scope}:{idempotency_key}',
  1228. _make_request_fingerprint(body, keys=_responses_fingerprint_keys(body)))
  1229. from gateway.platforms.api_server import _openai_error
  1230. from gateway.platforms.api_server_response_admissions import request_binding
  1231. response_store = self._current_response_store()
  1232. binding = request_binding(self, request, durable_key)
  1233. matches, replay = await response_store.off_loop(_matching_replay, response_store, durable_key[0], binding)
  1234. if not matches:
  1235. return durable_key, idempotency_scope, idempotency_key, web.json_response(
  1236. _openai_error('admission_conflict', code='admission_conflict'), status=409)
  1237. if replay is not None:
  1238. if replay.get('fingerprint') != durable_key[1]:
  1239. return durable_key, idempotency_scope, idempotency_key, web.json_response(
  1240. _openai_error('admission_conflict', code='admission_conflict'), status=409)
  1241. if not _settled(replay):
  1242. # Never re-run a request whose earlier attempt may have executed.
  1243. return durable_key, idempotency_scope, idempotency_key, web.json_response(
  1244. _openai_error('unknown_execution', code='unknown_execution'), status=409)
  1245. if stream:
  1246. return (durable_key, idempotency_scope, idempotency_key,
  1247. await self._replay_sse_responses(request, replay))
  1248. return durable_key, idempotency_scope, idempotency_key, web.json_response(
  1249. replay['response'], headers=replay.get('headers') or {})
  1250. return durable_key, idempotency_scope, idempotency_key, None
  1251. async def _handle_responses(self, request: web.Request) -> web.Response:
  1252. """POST /v1/responses — OpenAI Responses API format."""
  1253. from gateway.platforms.api_server import (
  1254. ThreadSafeAsyncQueue, _coerce_request_bool,
  1255. _error_response, _invalid_request, _redact_api_error_text,
  1256. _resolve_media_to_data_urls, _responses_usage_payload)
  1257. # Bound total in-flight agent runs (configurable; #7483).
  1258. limited = self._concurrency_limited_response()
  1259. if limited is not None:
  1260. return limited
  1261. gateway_session_key, key_err = self._parse_session_key_header(request)
  1262. if key_err is not None:
  1263. return key_err
  1264. try:
  1265. body = await request.json()
  1266. except Exception:
  1267. return _invalid_request("Invalid JSON in request body")
  1268. raw_input = body.get("input")
  1269. if raw_input is None:
  1270. return _error_response("Missing 'input' field", 400)
  1271. instructions = body.get("instructions")
  1272. previous_response_id = body.get("previous_response_id")
  1273. conversation = body.get("conversation")
  1274. store = _coerce_request_bool(body.get("store"), default=True)
  1275. if conversation and previous_response_id:
  1276. return _error_response("Cannot use both 'conversation' and 'previous_response_id'", 400)
  1277. stream = _coerce_request_bool(body.get("stream"), default=False)
  1278. durable_key, idempotency_scope, idempotency_key, replayed = await self._responses_durable_replay(
  1279. request, body, stream)
  1280. if replayed is not None:
  1281. return replayed
  1282. from gateway.platforms.api_server_response_admissions import prepare_context
  1283. from hermes_state_runtime import RuntimeStoreError
  1284. try:
  1285. context, error = await prepare_context(
  1286. self, request, body, gateway_session_key, durable_key, idempotency_scope, idempotency_key)
  1287. except RuntimeStoreError as exc:
  1288. from gateway.platforms.api_server import _openai_error
  1289. from gateway.platforms.api_server_authority_runs import store_error_response
  1290. return store_error_response(_openai_error, exc)
  1291. if error is not None:
  1292. return error
  1293. session_id, user_message = context['session_id'], context['user_message']
  1294. conversation_history, instructions = context['conversation_history'], context['instructions']
  1295. run_kwargs, run_agent = context['run_kwargs'], context['run_agent']
  1296. response_store = self._current_response_store()
  1297. response_id, created_at = await _response_identity(response_store, durable_key)
  1298. response_model = body.get('model', self._model_name)
  1299. if durable_key:
  1300. response_model = await response_store.off_loop(response_store.request_model, durable_key[0], response_model)
  1301. if stream:
  1302. _stream_q = ThreadSafeAsyncQueue()
  1303. def _on_tool_progress(event_type, name, preview, args, **kwargs):
  1304. return # structured start/complete callbacks carry the call id; progress ignored
  1305. def _on_tool_start(tool_call_id, function_name, function_args):
  1306. _stream_q.put_threadsafe(("__tool_started__", {
  1307. "tool_call_id": tool_call_id, "name": function_name,
  1308. "arguments": function_args or {}}))
  1309. def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
  1310. _stream_q.put_threadsafe(("__tool_completed__", {
  1311. "tool_call_id": tool_call_id, "name": function_name,
  1312. "arguments": function_args or {}, "result": function_result}))
  1313. def _on_commentary(text, *, already_streamed: bool = False):
  1314. # Already-streamed text went out as output_text.delta of the final item; a second
  1315. # copy as a commentary item would duplicate it.
  1316. if not already_streamed and isinstance(text, str) and text.strip():
  1317. _stream_q.put_threadsafe(("__commentary__", {"text": text}))
  1318. agent_task, agent_ref = self._spawn_stream_agent(
  1319. _stream_q, run_agent=run_agent, tool_progress_callback=_on_tool_progress,
  1320. tool_start_callback=_on_tool_start, tool_complete_callback=_on_tool_complete,
  1321. interim_assistant_callback=_on_commentary, **run_kwargs)
  1322. return await self._write_sse_responses(
  1323. request=request, response_id=response_id,
  1324. model=response_model, created_at=created_at,
  1325. stream_q=_stream_q, agent_task=agent_task, agent_ref=agent_ref,
  1326. conversation_history=conversation_history, user_message=user_message,
  1327. instructions=instructions, conversation=conversation, store=store,
  1328. session_id=session_id, gateway_session_key=gateway_session_key,
  1329. durable_key=durable_key, terminal_replay=context.get('terminal_replay', False))
  1330. async def _compute_response():
  1331. return await run_agent(**run_kwargs)
  1332. outcome, err = await self._run_idempotent(
  1333. request, body, _compute_response, log_label="responses",
  1334. fingerprint_keys=list(_responses_fingerprint_keys(body)), route="responses",
  1335. )
  1336. if err is not None:
  1337. return err
  1338. if durable_key is not None:
  1339. response_store = self._current_response_store()
  1340. replay = await response_store.off_loop(response_store.get, durable_key[0])
  1341. if _settled(replay):
  1342. return web.json_response(replay['response'], headers=replay.get('headers') or {})
  1343. result, usage = outcome
  1344. final_response = _resolve_media_to_data_urls(result.get("final_response", ""))
  1345. if not final_response:
  1346. final_response = _redact_api_error_text(result.get("error", "(No response generated)"))
  1347. full_history = self._build_response_conversation_history(
  1348. conversation_history, user_message, result, final_response,
  1349. tool_output_max_chars=self._history_tool_output_max_chars)
  1350. # _run_agent's effective session id carries compression rotations; storing it keeps
  1351. # previous_response_id chaining off the pre-rotation session (else compression re-fires).
  1352. _result_sid = result.get("session_id") if isinstance(result, dict) else None
  1353. _effective_session_id = (
  1354. _result_sid if isinstance(_result_sid, str) and _result_sid else session_id)
  1355. # Output items = current turn only (AIAgent returns a full transcript; mocked paths
  1356. # only the current-turn suffix).
  1357. output_start_index = self._response_messages_turn_start_index(
  1358. conversation_history, user_message, result)
  1359. response_data = {
  1360. "id": response_id, "object": "response", "status": _response_status(result),
  1361. "created_at": created_at, "model": response_model,
  1362. "output": self._extract_output_items(result, start_index=output_start_index,
  1363. **({'response_id': response_id} if durable_key else {})),
  1364. "usage": _responses_usage_payload(usage)}
  1365. response_store = self._current_response_store() # at use time: the store can be swapped mid-turn
  1366. if durable_key:
  1367. response_data['output'] = await response_store.off_loop(
  1368. response_store.order_output, durable_key[0], response_data['output'])
  1369. if response_data['status'] == 'failed':
  1370. response_data['error'] = {'code': 'agent_error', 'message': _redact_api_error_text(
  1371. result.get('error') or 'The admitted turn failed.')}
  1372. snapshot = (response_id, {"response": response_data, "conversation_history": full_history,
  1373. "instructions": instructions, "session_id": _effective_session_id}) if store else None
  1374. response_headers = {"X-Hermes-Session-Id": _effective_session_id}
  1375. if gateway_session_key:
  1376. response_headers["X-Hermes-Session-Key"] = gateway_session_key
  1377. replay_record = (durable_key[0], {'fingerprint': durable_key[1], 'response': response_data,
  1378. 'headers': response_headers}) if durable_key is not None else None
  1379. if snapshot is not None or replay_record is not None:
  1380. # Awaited before answering, so the returned id (and its replay record) read back at once.
  1381. await response_store.off_loop(_store_nonstream_response, response_store, snapshot, replay_record,
  1382. conversation, bool(context.get('terminal_replay')))
  1383. return web.json_response(response_data, headers=response_headers)
  1384. async def _handle_get_response(self, request: web.Request) -> web.Response:
  1385. """GET /v1/responses/{response_id} — retrieve a stored response."""
  1386. from gateway.platforms.api_server import _error_response
  1387. auth_err = self._check_auth(request)
  1388. if auth_err:
  1389. return auth_err
  1390. response_id = request.match_info["response_id"]
  1391. store = self._current_response_store()
  1392. stored = await store.off_loop(store.get, response_id)
  1393. if stored is None:
  1394. return _error_response(f"Response not found: {response_id}", 404)
  1395. return web.json_response(stored["response"])
  1396. async def _handle_delete_response(self, request: web.Request) -> web.Response:
  1397. """DELETE /v1/responses/{response_id} — delete a stored response."""
  1398. from gateway.platforms.api_server import _error_response
  1399. auth_err = self._check_auth(request)
  1400. if auth_err:
  1401. return auth_err
  1402. response_id = request.match_info["response_id"]
  1403. store = self._current_response_store()
  1404. if not await store.off_loop(store.delete, response_id):
  1405. return _error_response(f"Response not found: {response_id}", 404)
  1406. return web.json_response({"id": response_id, "object": "response", "deleted": True})
  1407. @staticmethod
  1408. def _build_response_conversation_history(
  1409. conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
  1410. final_response: Any, *, tool_output_max_chars: int = 0) -> list[dict[str, Any]]:
  1411. """Build the stored Responses transcript without duplicating history.
  1412. A compressed transcript (``result["_compressed"]``) shares no input-history prefix, so
  1413. turn-start detection fails; prepending the uncompressed history would bloat the stored
  1414. context and re-trigger compression every request — it is stored as-is instead.
  1415. ``tool_output_max_chars`` > 0 caps tool outputs / tool-call argument blobs in the
  1416. stored copy (gateway.api_server.history_tool_output_max_chars; 0 = store verbatim).
  1417. """
  1418. from gateway.platforms.api_server import APIServerAdapter
  1419. prior = list(conversation_history)
  1420. current_user = {"role": "user", "content": user_message}
  1421. if isinstance(result, dict) and isinstance(result.get("_turn_user_row"), dict):
  1422. current_user = dict(result["_turn_user_row"]) # settled canonical turn: carries api_content
  1423. agent_messages = result.get("messages") if isinstance(result, dict) else None
  1424. if isinstance(agent_messages, list) and agent_messages:
  1425. turn_start = APIServerAdapter._response_messages_turn_start_index(
  1426. conversation_history, user_message, result)
  1427. # turn_start == 0: compression rewrote the transcript or agent_messages is turn-only.
  1428. if turn_start or result.get("_compressed"):
  1429. history = list(agent_messages)
  1430. else:
  1431. history = prior + [current_user] + agent_messages
  1432. else:
  1433. history = prior + [current_user, {"role": "assistant", "content": final_response}]
  1434. return _cap_history_tool_outputs(history, tool_output_max_chars)
  1435. @staticmethod
  1436. def _response_messages_turn_start_index(
  1437. conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
  1438. ) -> int:
  1439. """Index where this turn starts in a transcript-shaped result["messages"] (0 = all)."""
  1440. from gateway.platforms.api_server_turn_boundary import response_turn_start_index
  1441. return response_turn_start_index(conversation_history, user_message, result)
  1442. @classmethod
  1443. def _turn_transcript_messages(
  1444. cls, conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
  1445. ) -> list[dict[str, Any]]:
  1446. """This turn's assistant/tool messages in client-safe shape: clients accumulating
  1447. ``assistant.delta`` into one buffer cannot reconstruct assistant segments that preceded
  1448. tool calls, so ``run.completed`` carries the authoritative per-turn transcript.
  1449. Emitting the authoritative per-turn transcript on ``run.completed`` lets any SSE consumer reconcile
  1450. its live view against ground truth without a separate ``GET /messages`` round-trip. Purely additive:
  1451. clients that ignore the field are unaffected. Refs #34703.
  1452. """
  1453. agent_messages = result.get("messages") if isinstance(result, dict) else None
  1454. if not isinstance(agent_messages, list) or not agent_messages:
  1455. return []
  1456. start = cls._response_messages_turn_start_index(conversation_history, user_message, result)
  1457. out: list[dict[str, Any]] = []
  1458. for msg in agent_messages[start:]:
  1459. if not isinstance(msg, dict) or msg.get("role") not in {"assistant", "tool"}:
  1460. continue
  1461. # _message_response projects compaction scaffolding; pure handoffs are "hidden".
  1462. projected = cls._message_response(msg)
  1463. if projected.get("display_kind") != "hidden":
  1464. out.append(projected)
  1465. return out
  1466. @staticmethod
  1467. def _extract_output_items(result: dict[str, Any], start_index: int = 0, *, response_id=None) -> list[dict[str, Any]]:
  1468. """Output items from ``result["messages"][start_index:]``: ``function_call`` per assistant
  1469. tool_call, ``function_call_output`` per tool message, then the final ``message``."""
  1470. from gateway.platforms.api_server import _redact_api_error_text
  1471. items: list[dict[str, Any]] = []
  1472. messages = result.get("messages", [])
  1473. if start_index > 0:
  1474. messages = messages[start_index:]
  1475. for msg in messages:
  1476. role = msg.get("role")
  1477. reasoning = msg.get("reasoning") if role == "assistant" else None
  1478. if isinstance(reasoning, str) and reasoning.strip():
  1479. # Precedes this message's function_call items, like the SSE writer closes a
  1480. # thinking burst before the next tool item opens (#99552).
  1481. items.append(_reasoning_item(reasoning))
  1482. if role == "assistant" and msg.get("tool_calls"):
  1483. for tc in msg["tool_calls"]:
  1484. func = tc.get("function", {})
  1485. # Already executed server-side; replayed for structured tool UI only, so
  1486. # marked completed (matching the SSE path) — never pending client calls.
  1487. items.append({
  1488. "id": f"fc_{uuid.uuid4().hex[:24]}", "type": "function_call",
  1489. "status": "completed", "name": func.get("name", ""),
  1490. "arguments": func.get("arguments", ""),
  1491. "call_id": tc.get("id", "")})
  1492. elif role == "tool":
  1493. items.append({
  1494. "id": f"fco_{uuid.uuid4().hex[:24]}", "type": "function_call_output",
  1495. "status": "completed", "call_id": msg.get("tool_call_id", ""),
  1496. "output": msg.get("content", "")})
  1497. final = result.get("final_response", "") or _redact_api_error_text(
  1498. result.get("error", "(No response generated)"))
  1499. items.append(_message_item(final))
  1500. if response_id is not None:
  1501. from gateway.platforms.api_server_response_identity import identify_output
  1502. return identify_output(items, response_id)
  1503. return items