| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633 |
- """OpenAI-compatible routes for the API server adapter.
- ``OpenAICompatRoutesMixin`` (inherited by ``APIServerAdapter``) carries ``/v1/chat/completions``,
- ``/v1/responses`` (+ GET/DELETE), their SSE writers and the Responses-transcript helpers.
- api_server-internal helpers are imported lazily inside each method: the origin imports this
- module (top-level import = cycle), and lazy lookup keeps ``patch("...api_server.X")`` effective.
- """
- import asyncio
- import copy
- import json
- import logging
- import re
- import time
- import uuid
- from contextlib import suppress
- from typing import Any, Dict, List, Optional
- from agent.i18n import t
- try:
- from aiohttp import web
- except ImportError: # pragma: no cover - mirrors api_server's optional import
- web = None # type: ignore[assignment]
- # Logger parity with the origin module (moved log records keep their name).
- logger = logging.getLogger("gateway.platforms.api_server")
- async def _iter_stream_items(stream_q, agent_task, response):
- """Yield agent stream items until EOS, writing SSE keepalives while idle.
- Yields the ``None`` sentinel once so callers can run EOS-only work; when ``agent_task``
- is already done the remaining queue is drained and the sentinel swallowed.
- """
- from gateway.platforms.api_server import CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS
- last_activity = time.monotonic()
- while True:
- try:
- item = await asyncio.wait_for(stream_q.get(), timeout=0.5)
- except TimeoutError:
- if agent_task.done():
- while True:
- try:
- item = stream_q.get_nowait()
- except asyncio.QueueEmpty:
- return
- if item is None:
- return
- yield item
- last_activity = time.monotonic()
- if time.monotonic() - last_activity >= CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS:
- await response.write(b": keepalive\n\n")
- last_activity = time.monotonic()
- continue
- if item is None:
- yield None
- return
- yield item
- last_activity = time.monotonic()
- def _result_flags(result: Any) -> tuple:
- """``(completed, partial, failed, error)`` from an agent result dict (defaults if not a dict)."""
- if not isinstance(result, dict):
- return True, False, False, None
- return (bool(result.get("completed", True)) and not result.get('interrupted'), bool(result.get("partial")),
- bool(result.get("failed")), result.get("error"))
- def _budget_exhausted(result: Any) -> bool:
- """The turn ran out of its iteration budget; its final text is the agent's own wrap-up."""
- reason = str(result.get("turn_exit_reason") or "") if isinstance(result, dict) else ""
- return reason.startswith(("budget_exhausted", "max_iterations_reached("))
- def _finish_reason(completed, is_partial, is_failed, err_msg, agent_error=None, *, budget_exhausted=False) -> str:
- """OpenAI ``finish_reason``: "length" for truncation, "error" for failure, else "stop"."""
- # OpenAI uses "length" for truncation, "stop" for normal completion, and downstream SDKs accept "error"
- # / custom codes. See issue #22496.
- if is_partial and err_msg and "truncat" in err_msg.lower():
- return "length"
- if agent_error is not None or is_failed or (not completed and err_msg):
- return "error"
- # "cancelled" means the user interrupted; a budget-capped turn stopped short of the task instead,
- # and a broker reading "cancelled" discards a usable wrap-up answer.
- if budget_exhausted:
- return "length"
- return 'cancelled' if not completed else 'stop'
- _RESPONSES_FINGERPRINT_KEYS = (
- "input", "instructions", "previous_response_id", "conversation", "conversation_history", "model",
- "provider", "model_options", "tools",
- )
- def _settled(record):
- """A stored idempotency record that carries its answer. A claim written without one (an
- older release recorded the claim before the run) has an unknown outcome and never replays."""
- return record is not None and record.get('response') is not None
- def _responses_fingerprint_keys(body):
- """``truncation`` joins the fingerprint only when sent, so an exact retry of a request
- recorded before it was fingerprinted keeps matching its stored key."""
- return _RESPONSES_FINGERPRINT_KEYS + (("truncation",) if body.get("truncation") is not None else ())
- async def _response_identity(store, durable_key):
- if durable_key is not None:
- from gateway.platforms.api_server_response_identity import durable_response_id
- return durable_response_id(durable_key[0]), await store.off_loop(store.request_created_at, durable_key[0])
- return f'resp_{uuid.uuid4().hex[:28]}', int(time.time())
- def _matching_replay(store, key, binding):
- """-> ``(binding matches, stored replay record or None)``, read as one store unit."""
- if not store.request_key_matches(key, binding):
- return False, None
- return True, store.get(key)
- def _store_nonstream_response(store, snapshot, replay_record, conversation, terminal_replay):
- """Non-streaming /v1/responses persistence as one store unit: body, conversation pointer, replay record."""
- if snapshot is not None:
- response_id, data = snapshot
- advance = conversation and (not terminal_replay or store.get_conversation(conversation) in (None, response_id))
- store.put(response_id, data)
- if advance:
- store.set_conversation(conversation, response_id)
- if replay_record is not None:
- store.put(*replay_record)
- def _response_status(result):
- if result.get('interrupted'):
- return 'cancelled'
- if result.get('failed') or result.get('error'):
- return 'failed'
- if result.get('completed') is False:
- return 'incomplete'
- return 'completed'
- def _hermes_extras(completed, is_partial, is_failed, err_msg, finish_reason: str, *,
- budget_exhausted: bool = False) -> dict[str, Any]:
- error_code = ("iteration_budget_exhausted" if budget_exhausted and finish_reason == "length"
- else "output_truncated" if finish_reason == "length" else "agent_error")
- return {"completed": completed, "partial": is_partial, "failed": is_failed, "error": err_msg,
- "error_code": error_code}
- def _transformed_notice() -> str:
- return t("platform.api_server.transformed_notice")
- def _post_stream_transform(result: Any) -> tuple:
- """``(text, appended)`` a stream still owes after ``transform_llm_output`` rewrote the final
- (the deltas already carried the raw reply): the appended suffix, or the whole rewrite with
- ``appended=False`` when it is not a pure append. ``("", False)`` when nothing was transformed."""
- if not isinstance(result, dict) or not result.get("response_transformed"):
- return "", False
- final = result.get("final_response") or ""
- original = result.get("pre_transform_response") or ""
- if original and final.startswith(original):
- return final[len(original):], True
- return final, False
- def _message_item(text: Any) -> dict[str, Any]:
- """Responses ``message`` output item carrying one ``output_text`` part."""
- return {"type": "message", "role": "assistant",
- "content": [{"type": "output_text", "text": text}]}
- def _cap_text(text: str, keep: int) -> str:
- """Head of ``text`` plus a marker saying how much was cut (the Responses truncation rule)."""
- return text[:keep] + "...[" + str(len(text) - keep) + " more chars]"
- def _cap_history_tool_outputs(history: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]:
- """Copy of ``history`` with tool outputs and string tool-call arguments longer than
- ``max_chars`` cut down. Only tool rows and ``tool_calls`` blobs change; user/assistant text
- is left alone, and the agent's own transcript rows are never mutated (rows are copied).
- Opt-in via gateway.api_server.history_tool_output_max_chars: a single stored snapshot
- embeds the full cumulative history, so a few large tool outputs pushed one
- response_store.db write to ~677 KB (#82513)."""
- if max_chars <= 0:
- return history
- out: list[dict[str, Any]] = []
- for msg in history:
- if not isinstance(msg, dict):
- out.append(msg)
- continue
- content = msg.get("content")
- if msg.get("role") == "tool" and isinstance(content, str) and len(content) > max_chars:
- msg = {**msg, "content": _cap_text(content, max_chars)}
- tool_calls = msg.get("tool_calls")
- if msg.get("role") == "assistant" and isinstance(tool_calls, list):
- capped_calls = []
- for call in tool_calls:
- fn = call.get("function") if isinstance(call, dict) else None
- raw = fn.get("arguments") if isinstance(fn, dict) else None
- if isinstance(raw, str) and len(raw) > max_chars:
- try:
- args = json.loads(raw)
- except ValueError:
- args = None
- if isinstance(args, dict):
- for k, v in args.items():
- if isinstance(v, str) and len(v) > max_chars:
- args[k] = _cap_text(v, max_chars)
- call = {**call, "function": {**fn, "arguments": json.dumps(args)}}
- capped_calls.append(call)
- msg = {**msg, "tool_calls": capped_calls}
- out.append(msg)
- return out
- def _reasoning_item(text: str) -> dict[str, Any]:
- """Completed Responses ``reasoning`` output item (same shape the SSE writer closes with)."""
- return {"id": f"rs_{uuid.uuid4().hex[:24]}", "type": "reasoning", "status": "completed",
- "summary": [{"type": "summary_text", "text": text}]}
- def _is_reasoning_input_item(item: Any) -> bool:
- """Echoed-back ``reasoning`` output item: Responses SDK clients replay a prior response's
- ``output`` list as the next ``input``. It carries no message content, so it must be
- skipped rather than parsed into an empty ``user`` turn (#99552)."""
- return isinstance(item, dict) and item.get("type") == "reasoning"
- def _turn_reasoning_text(
- conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any]) -> str:
- """Reasoning the model produced on this turn, joined for a non-streaming
- ``message.reasoning_content``. Read from the assistant messages the agent already
- persisted (``build_assistant_message`` stores the structured reasoning under
- ``reasoning``) rather than re-accumulating callback deltas, so it is exactly what the
- stream would have carried and cannot double-count the post-response fallback."""
- messages = result.get("messages") if isinstance(result, dict) else None
- if not isinstance(messages, list):
- return ""
- start = OpenAICompatRoutesMixin._response_messages_turn_start_index(
- conversation_history, user_message, result)
- parts = [m["reasoning"] for m in messages[start:]
- if isinstance(m, dict) and m.get("role") == "assistant"
- and isinstance(m.get("reasoning"), str) and m["reasoning"].strip()]
- return "\n\n".join(parts)
- def _trim_tool_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Trim large tool payloads in place so response.completed stays under ~100KB (clients
- already received the full details via the incremental events)."""
- for item in items:
- if item.get("type") == "function_call":
- try:
- raw = item.get("arguments")
- args = json.loads(raw) if isinstance(raw, str) else item.get("arguments", {})
- if isinstance(args, dict):
- for k in ("content", "query", "pattern", "old_string", "new_string"):
- if isinstance(args.get(k), str) and len(args[k]) > 500:
- args[k] = f"[{len(args[k])} chars — truncated for response.completed]"
- item["arguments"] = json.dumps(args)
- except Exception:
- pass
- elif item.get("type") == "function_call_output":
- output = item.get("output", [])
- if isinstance(output, list) and output:
- first = output[0]
- if isinstance(first, dict) and first.get("type") == "input_text":
- text = first.get("text", "")
- if len(text) > 1000:
- first["text"] = _cap_text(text, 500)
- item["output"] = [first]
- return items
- class _ResponsesStream:
- """Per-request state and event emitters for the POST /v1/responses SSE writer.
- Every event carries a monotonic ``sequence_number`` (canonical Responses SSE schema). Text
- deltas are batched (50ms) against Open WebUI re-render storms; tool events flush first.
- """
- def __init__(self, adapter, response, *, response_id: str, model: str, created_at: int,
- conversation_history: list[dict[str, str]], user_message: str,
- instructions: Optional[str], conversation: Optional[str], store: bool, session_id: str,
- durable_key=None, terminal_replay=False):
- from gateway.platforms import api_server as api
- self._api = api
- self.adapter, self.response, self.response_id = adapter, response, response_id
- self.model, self.created_at, self.conversation_history = model, created_at, conversation_history
- self.user_message, self.instructions = user_message, instructions
- self.conversation, self.store, self.session_id = conversation, store, session_id
- self.durable_key = durable_key
- self.terminal_replay = terminal_replay
- # Resolved in the request's profile scope: a snapshot written after it (disconnect) must not follow another.
- self.response_store = adapter._current_response_store()
- self.final_text_parts: list[str] = []
- self.pending_tool_calls: list[dict[str, Any]] = [] # open function_call items, in order
- self.emitted_items: list[dict[str, Any]] = [] # output items so far (terminal payload)
- self.output_index = 0
- self.call_counter = 0 # call_id fallback when the agent supplies no tool_call_id
- self.sequence_number = 0
- self._identity_counts = {}
- self._published_output_ids = []
- self._suppress_incremental = False
- self.message_item_id = self._item_id('message')
- self.message_output_index: Optional[int] = None
- self.message_opened = False
- self.reasoning_item: Optional[dict[str, Any]] = None # open ``reasoning`` output item
- self.final_response_text = ""
- self.transformed_final = "" # non-append transform_llm_output rewrite; replaces the deltas
- self.agent_error: Optional[str] = None
- self.usage: dict[str, int] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
- self.terminal_snapshot_persisted = False
- self.recorded_events: list[tuple] = [] # (event_type, data) as written, for idempotent replay
- self.result: Any = None
- self._batch_buf: list[str] = []
- self._batch_timer: Optional[asyncio.Task] = None
- self._batch_lock = asyncio.Lock()
- async def write_event(self, event_type: str, data: dict[str, Any]) -> None:
- if self.durable_key and 'output_index' in data:
- if self._suppress_incremental:
- return
- index = data['output_index']
- if event_type == 'response.output_item.added':
- valid = index == len(self._published_output_ids)
- if valid:
- self._published_output_ids.append(data['item']['id'])
- else:
- identity = data.get('item_id') or data.get('item', {}).get('id')
- valid = 0 <= index < len(self._published_output_ids) and (
- identity is None or self._published_output_ids[index] == identity)
- if not valid:
- # A late/backpressured observer may miss a globally allocated prefix. Keep the
- # prefix it already published; complete from the canonical terminal snapshot.
- self._suppress_incremental = True
- return
- if "sequence_number" not in data:
- data["sequence_number"] = self.sequence_number
- self.sequence_number += 1
- if self.durable_key: # the only reader is the keyed replay record (emit_completed)
- # Snapshot: terminal trimming later edits the nested tool output these frames share.
- self.recorded_events.append((event_type, copy.deepcopy(data)))
- await self.response.write(self._api._sse_frame(data, event=event_type))
- def _item_id(self, kind, *, call_id=None, commentary=False):
- from gateway.platforms.api_server_response_identity import next_item_id
- return next_item_id(self.response_id, self._identity_counts, kind,
- call_id=call_id, commentary=commentary)
- async def _output_index(self, item_id):
- index = (await self.response_store.off_loop(self.response_store.output_index, self.durable_key[0], item_id)
- if self.durable_key else self.output_index)
- self.output_index = max(self.output_index, index + 1)
- return index
- def envelope(self, status: str) -> dict[str, Any]:
- return {"id": self.response_id, "object": "response", "status": status,
- "created_at": self.created_at, "model": self.model}
- def terminal_envelope(self, status: str, output: list[dict[str, Any]], *, error=None) -> dict:
- """``envelope`` + ``output`` (+ ``error`` when given) + ``usage``, in wire key order."""
- env = self.envelope(status)
- env["output"] = output
- if error is not None:
- env["error"] = ({"code": "agent_error", "message": error} if self.durable_key
- else {"message": error, "type": "server_error"})
- env["usage"] = self._api._responses_usage_payload(self.usage)
- return env
- def _history_with_user(self) -> list[dict[str, Any]]:
- return list(self.conversation_history) + [{"role": "user", "content": self.user_message}]
- async def persist_snapshot(self, response_env: dict[str, Any], *, history=None, session_id=None):
- if self.store:
- await self.response_store.off_loop(self._persist_snapshot, response_env, history, session_id)
- def _persist_snapshot(self, response_env, history, session_id):
- """One store unit (worker thread, store lock held): the settled check, the body and the
- conversation pointer can not interleave with another observer's."""
- if self.durable_key and _settled(self.response_store.get(self.durable_key[0])):
- return # A concurrent observer already committed the exact terminal envelope.
- if response_env.get('status') == 'incomplete' and self.terminal_snapshot_persisted:
- # A cancel that interrupted the await of a terminal commit: its worker still finished it.
- return
- advance_conversation = self.conversation and (not self.terminal_replay
- or self.response_store.get_conversation(self.conversation) in (None, self.response_id))
- self.response_store.put(self.response_id, {
- "response": response_env,
- "conversation_history": self._history_with_user() if history is None else history,
- "instructions": self.instructions,
- "session_id": session_id or self.session_id})
- if advance_conversation:
- self.response_store.set_conversation(self.conversation, self.response_id)
- if response_env.get('status') != 'in_progress':
- # Set by the worker, under the store lock: a cancel of the awaiting handler cannot lose it.
- self.terminal_snapshot_persisted = True
- async def persist_incomplete_if_needed(self) -> None:
- """Persist an ``incomplete`` snapshot when no terminal one was written (disconnect /
- cancel paths), so GET /v1/responses/{id} and ``previous_response_id`` chaining survive."""
- if not self.store or self.terminal_snapshot_persisted:
- return
- text = "".join(self.final_text_parts) or self.final_response_text
- items = list(self.emitted_items)
- history = self._history_with_user()
- if text:
- items.append(_message_item(text))
- history.append({"role": "assistant", "content": text})
- await self.persist_snapshot(self.terminal_envelope("incomplete", items), history=history)
- async def emit_created(self) -> None:
- env = self.envelope("in_progress")
- env["output"] = []
- if not self.terminal_replay:
- # Stored before the id goes out: a client that reads it back at once must find it.
- await self.persist_snapshot(env)
- await self.write_event("response.created", {"type": "response.created", "response": env})
- async def _open_message_item(self) -> None:
- """Emit output_item.added for the assistant message on the first text delta."""
- if self.message_opened:
- return
- self.message_opened = True
- self.message_output_index = await self._output_index(self.message_item_id)
- await self.write_event("response.output_item.added", {
- "type": "response.output_item.added", "output_index": self.message_output_index,
- "item": {"id": self.message_item_id, "type": "message", "status": "in_progress",
- "role": "assistant", "content": []}})
- async def emit_text_delta(self, delta_text: str) -> None:
- if self.terminal_replay:
- self.final_text_parts.append(delta_text)
- return
- await self.close_reasoning_item()
- await self._open_message_item()
- self.final_text_parts.append(delta_text)
- await self.write_event("response.output_text.delta", {
- "type": "response.output_text.delta", "item_id": self.message_item_id,
- "output_index": self.message_output_index, "content_index": 0, "delta": delta_text,
- "logprobs": []})
- async def emit_reasoning_delta(self, delta_text: str) -> None:
- """Responses reasoning-summary family (#99552): one ``reasoning`` output item per
- thinking burst, closed before the next message/tool item opens."""
- if self.reasoning_item is None:
- item = {"id": self._item_id('reasoning'), "type": "reasoning", "status": "in_progress",
- "summary": []}
- self.reasoning_item = {"item": item, "output_index": await self._output_index(item['id']), "parts": []}
- await self.write_event("response.output_item.added", {
- "type": "response.output_item.added",
- "output_index": self.reasoning_item["output_index"], "item": item})
- await self.write_event("response.reasoning_summary_part.added", {
- "type": "response.reasoning_summary_part.added", "item_id": item["id"],
- "output_index": self.reasoning_item["output_index"], "summary_index": 0,
- "part": {"type": "summary_text", "text": ""}})
- rs = self.reasoning_item
- rs["parts"].append(delta_text)
- await self.write_event("response.reasoning_summary_text.delta", {
- "type": "response.reasoning_summary_text.delta", "item_id": rs["item"]["id"],
- "output_index": rs["output_index"], "summary_index": 0, "delta": delta_text})
- async def close_reasoning_item(self) -> None:
- rs, self.reasoning_item = self.reasoning_item, None
- if rs is None:
- return
- text = "".join(rs["parts"])
- item = dict(rs["item"], status="completed", summary=[{"type": "summary_text", "text": text}])
- base = {"item_id": item["id"], "output_index": rs["output_index"], "summary_index": 0}
- await self.write_event("response.reasoning_summary_text.done", {
- "type": "response.reasoning_summary_text.done", **base, "text": text})
- await self.write_event("response.reasoning_summary_part.done", {
- "type": "response.reasoning_summary_part.done", **base,
- "part": {"type": "summary_text", "text": text}})
- self.emitted_items.append(item)
- await self.write_event("response.output_item.done", {
- "type": "response.output_item.done", "output_index": rs["output_index"], "item": item})
- async def emit_commentary(self, text: str) -> None:
- """Mid-turn assistant commentary as its own completed ``message`` item carrying
- ``"phase": "commentary"`` — never appended to ``final_text_parts``, so the final answer
- item stays clean (#67580). Closes any open reasoning item first so a reasoning item
- never straddles a message item."""
- await self.close_reasoning_item()
- item = {"id": self._item_id('message', commentary=True), "status": "completed", "phase": "commentary",
- **_message_item(text)}
- idx = await self._output_index(item['id'])
- self.emitted_items.append({"phase": "commentary", **_message_item(text)})
- for event in ("response.output_item.added", "response.output_item.done"):
- await self.write_event(event, {"type": event, "output_index": idx, "item": item})
- async def emit_tool_started(self, payload: dict[str, Any]) -> None:
- """function_call ``output_item.added``; the agent's tool_call_id beats a generated call id."""
- await self.close_reasoning_item()
- self.call_counter += 1
- call_id = payload.get("tool_call_id") or f"call_{self.response_id[5:]}_{self.call_counter}"
- args = payload.get("arguments", {})
- arguments_str = json.dumps(args) if isinstance(args, dict) else str(args)
- name = payload.get("name", "")
- item = {"id": self._item_id('function_call', call_id=call_id), "type": "function_call",
- "status": "in_progress", "name": name, "call_id": call_id, "arguments": arguments_str}
- idx = await self._output_index(item['id'])
- self.pending_tool_calls.append({
- "call_id": call_id, "name": name, "arguments": arguments_str, "item_id": item["id"],
- "output_index": idx})
- self.emitted_items.append(
- {"type": "function_call", "name": name, "arguments": arguments_str, "call_id": call_id})
- await self.write_event("response.output_item.added", {
- "type": "response.output_item.added", "output_index": idx, "item": item})
- async def emit_tool_completed(self, payload: dict[str, Any]) -> None:
- """function_call ``output_item.done`` + function_call_output added/done; orphans skipped."""
- call_id = payload.get("tool_call_id")
- pending = next((p for p in self.pending_tool_calls if p["call_id"] == call_id), None)
- if not call_id or pending is None:
- return
- self.pending_tool_calls.remove(pending)
- done_item = {"id": pending["item_id"], "type": "function_call", "status": "completed",
- "name": pending["name"], "call_id": pending["call_id"],
- "arguments": pending["arguments"]}
- await self.write_event("response.output_item.done", {
- "type": "response.output_item.done", "output_index": pending["output_index"],
- "item": done_item})
- result = payload.get("result", "")
- result_str = result if isinstance(result, str) else json.dumps(result)
- output_parts = [{"type": "input_text", "text": result_str}]
- output_item = {"id": self._item_id('function_call_output', call_id=call_id), "type": "function_call_output",
- "call_id": pending["call_id"], "output": output_parts, "status": "completed"}
- idx = await self._output_index(output_item['id'])
- self.emitted_items.append(
- {"type": "function_call_output", "call_id": pending["call_id"], "output": output_parts})
- for event in ("response.output_item.added", "response.output_item.done"):
- await self.write_event(event, {"type": event, "output_index": idx, "item": output_item})
- async def emit_status(self, payload: dict[str, Any]) -> None:
- """Lifecycle/warning status (provider wait, auto-recovery countdown, fallback switch) as a
- ``hermes.status`` custom event; not a Responses output item."""
- await self.response.write(self._api._sse_frame(payload, event="hermes.status"))
- # queue tag -> (method name, payload adapter)
- _TAG_HANDLERS = {
- "__tool_started__": ("emit_tool_started", lambda p: p),
- "__tool_completed__": ("emit_tool_completed", lambda p: p),
- "__commentary__": ("emit_commentary", lambda p: p["text"]),
- "__reasoning__": ("emit_reasoning_delta", lambda p: p),
- "__status__": ("emit_status", lambda p: p),
- }
- async def dispatch(self, item: Any) -> None:
- """Route one queue item: tagged tuples emit immediately, strings are batched, others dropped."""
- if self.terminal_replay:
- return
- if isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str):
- tag, payload = item
- await self.flush_batch()
- handler = self._TAG_HANDLERS.get(tag)
- if handler is not None:
- method, adapt = handler
- await getattr(self, method)(adapt(payload))
- elif isinstance(item, str):
- self._batch_buf.append(item)
- if self._batch_timer is None:
- self._batch_timer = asyncio.create_task(self._batch_flush_after(0.05))
- async def _batch_flush_after(self, delay: float) -> None:
- try:
- await asyncio.sleep(delay)
- except asyncio.CancelledError:
- return
- # Clear the timer BEFORE flushing so new deltas can start a fresh timer while we emit.
- self._batch_timer = None
- await self.flush_batch()
- def cancel_batch_timer(self) -> None:
- if self._batch_timer and not self._batch_timer.done():
- self._batch_timer.cancel()
- self._batch_timer = None
- async def flush_batch(self) -> None:
- """Emit a single delta for all buffered text. Always takes the lock, even on an empty buffer: a
- timer flush may still be opening the message item (its keyed output index is read off the loop),
- and a caller that closes the item next must wait for that index."""
- async with self._batch_lock:
- if self._batch_buf:
- combined = "".join(self._batch_buf)
- self._batch_buf = []
- await self.emit_text_delta(combined)
- async def collect_result(self, agent_task) -> None:
- """Await the agent; when it produced a final_response but streamed no deltas
- (some providers only emit the full text at the end), emit one fallback delta."""
- try:
- result, agent_usage = await agent_task
- self.result = result
- self.usage = agent_usage or self.usage
- agent_final = result.get("final_response", "") if isinstance(result, dict) else ""
- if self.terminal_replay:
- # The receipt already contains the transformed final; it is not a raw provider
- # stream onto which a verifier/footer transform should be applied a second time.
- self.final_response_text = agent_final or self._api._redact_api_error_text(
- result.get('error', '(No response generated)'))
- self.final_text_parts = [self.final_response_text]
- return
- tail, appended = _post_stream_transform(result)
- if tail and self.final_text_parts:
- if appended:
- await self.emit_text_delta(tail)
- else:
- self.transformed_final = agent_final
- if agent_final and not self.final_text_parts:
- await self.emit_text_delta(agent_final)
- if agent_final and not self.final_response_text:
- self.final_response_text = agent_final
- if isinstance(result, dict) and result.get("error") and not self.final_response_text:
- self.agent_error = self._api._redact_api_error_text(result["error"])
- except Exception as e:
- logger.error("Error running agent for streaming responses: %s", e, exc_info=True)
- self.agent_error = self._api._redact_api_error_text(e)
- async def close_message_item(self) -> None:
- await self.close_reasoning_item()
- self.final_response_text = (
- self.transformed_final or "".join(self.final_text_parts) or self.final_response_text)
- if not self.message_opened:
- return
- await self.write_event("response.output_text.done", {
- "type": "response.output_text.done", "item_id": self.message_item_id,
- "output_index": self.message_output_index, "content_index": 0,
- "text": self.final_response_text, "logprobs": []})
- await self.write_event("response.output_item.done", {
- "type": "response.output_item.done", "output_index": self.message_output_index,
- "item": {"id": self.message_item_id, "type": "message", "status": "completed",
- "role": "assistant",
- "content": [{"type": "output_text", "text": self.final_response_text}]}})
- def _final_items(self) -> list[dict[str, Any]]:
- """Emitted items (trimmed) plus a final message item, so clients that only parse
- the terminal payload still see the assistant text (mirrors _extract_output_items)."""
- items = _trim_tool_items(list(self.emitted_items))
- redact = self._api._redact_api_error_text
- text = self.final_response_text or (redact(self.agent_error) if self.agent_error else "")
- items.append(_message_item(text))
- return items
- async def emit_failed(self) -> None:
- redact = self._api._redact_api_error_text
- env = self.terminal_envelope("failed", self._final_items(), error=redact(self.agent_error))
- history = self._history_with_user()
- history.append(
- {"role": "assistant", "content": self.final_response_text or redact(self.agent_error)})
- await self.persist_snapshot(env, history=history)
- await self.write_event("response.failed", {"type": "response.failed", "response": env})
- async def emit_completed(self) -> dict[str, Any]:
- result = self.result
- status = _response_status(result)
- output = self._final_items()
- if self.durable_key:
- start = self.adapter._response_messages_turn_start_index(self.conversation_history, self.user_message, result)
- output = self.adapter._extract_output_items(result, start_index=start, response_id=self.response_id)
- output = await self.response_store.off_loop(self.response_store.order_output, self.durable_key[0], output)
- env = self.terminal_envelope(status, output,
- error=self._api._redact_api_error_text(result.get('error') or 'The admitted turn failed.')
- if status == 'failed' else None)
- full_history = self.adapter._build_response_conversation_history(
- self.conversation_history, self.user_message, result, self.final_response_text,
- tool_output_max_chars=self.adapter._history_tool_output_max_chars)
- # Transcript substitution for result["_compressed"] happens in the history builder; only
- # a compression-rotated session_id is propagated so chaining resumes the child session.
- sid = result.get("session_id") if isinstance(result, dict) else None
- record = None
- if self.durable_key and not self.terminal_replay:
- # Only the publisher recorded the deltas; a replay observer winning the
- # INSERT OR IGNORE would leave every later replay without them. Built here, on the
- # loop, so the worker never reads headers or events the loop may still append to.
- headers = {k: v for k, v in self.response.headers.items() if k.startswith('X-Hermes-')}
- terminal_event = {'type': 'response.' + status, 'response': env, 'sequence_number': self.sequence_number}
- record = {'fingerprint': self.durable_key[1], 'response': env, 'headers': headers,
- 'events': [*self.recorded_events, ('response.' + status, terminal_event)]}
- stored = await self.response_store.off_loop(
- self._commit_terminal, env, full_history, sid if isinstance(sid, str) and sid else None, record)
- if self.durable_key:
- if _settled(stored):
- env = stored['response']
- status = env['status']
- await self.write_event(
- 'response.' + status, {'type': 'response.' + status, 'response': env})
- return env
- def _commit_terminal(self, env, full_history, session_id, record):
- """The terminal snapshot plus, when keyed, the replay record and its read-back: one store
- unit, so no other observer of the same key interleaves between them. -> stored record.
- ``store: false`` skips only the snapshot; the keyed replay record is idempotency state."""
- if self.store:
- self._persist_snapshot(env, full_history, session_id)
- if not self.durable_key:
- return None
- if record is not None:
- self.response_store.put(self.durable_key[0], record)
- return self.response_store.get(self.durable_key[0])
- async def emit_crash(self, exc: BaseException) -> None:
- error = self._api._redact_api_error_text(exc, limit=500)
- env = self.terminal_envelope("failed", list(self.emitted_items), error=error)
- await self.write_event("response.failed", {"type": "response.failed", "response": env})
- def _parse_chat_messages(messages: list[Any]) -> tuple:
- """Chat ``messages`` -> ``(system_prompt, conversation_messages, error_response_or_None)``.
- System messages -> ephemeral system prompt layered ON TOP of core, flattened to text
- (Anthropic rejects images there, OpenAI text models ignore them)."""
- from gateway.platforms.api_server import (
- _multimodal_validation_error, _normalize_chat_content, _normalize_multimodal_content)
- system_prompt = None
- conversation_messages: list[dict[str, str]] = []
- for idx, msg in enumerate(messages):
- role = msg.get("role", "")
- raw_content = msg.get("content", "")
- if role == "system":
- content = _normalize_chat_content(raw_content)
- system_prompt = content if system_prompt is None else system_prompt + "\n" + content
- elif role in {"user", "assistant"}:
- try:
- content = _normalize_multimodal_content(raw_content)
- except ValueError as exc:
- return None, None, _multimodal_validation_error(exc, param=f"messages[{idx}].content")
- conversation_messages.append({"role": role, "content": content})
- return system_prompt, conversation_messages, None
- def _parse_responses_input(raw_input: Any) -> tuple:
- """Responses ``input`` (string or item array) -> ``(input_messages, error_response_or_None)``."""
- from gateway.platforms.api_server import (
- _error_response, _multimodal_validation_error, _normalize_multimodal_content)
- input_messages: list[dict[str, Any]] = []
- if isinstance(raw_input, str):
- input_messages = [{"role": "user", "content": raw_input}]
- elif isinstance(raw_input, list):
- for idx, item in enumerate(raw_input):
- if isinstance(item, str):
- input_messages.append({"role": "user", "content": item})
- elif _is_reasoning_input_item(item):
- continue
- elif isinstance(item, dict):
- try:
- content = _normalize_multimodal_content(item.get("content", ""))
- except ValueError as exc:
- return None, _multimodal_validation_error(exc, param=f"input[{idx}].content")
- input_messages.append({"role": item.get("role", "user"), "content": content})
- else:
- return None, _error_response("'input' must be a string or array", 400)
- return input_messages, None
- def _parse_conversation_history(raw_history: Any) -> tuple:
- """Explicit Responses ``conversation_history`` -> ``(history, error_response_or_None)``."""
- from gateway.platforms.api_server import (
- _error_response, _multimodal_validation_error, _normalize_multimodal_content)
- conversation_history: list[dict[str, Any]] = []
- if not isinstance(raw_history, list):
- return None, _error_response("'conversation_history' must be an array of message objects", 400)
- for i, entry in enumerate(raw_history):
- if _is_reasoning_input_item(entry):
- continue
- if not isinstance(entry, dict) or "role" not in entry or "content" not in entry:
- return None, _error_response(f"conversation_history[{i}] must have 'role' and 'content' fields", 400)
- try:
- entry_content = _normalize_multimodal_content(entry["content"])
- except ValueError as exc:
- return None, _multimodal_validation_error(exc, param=f"conversation_history[{i}].content")
- conversation_history.append({"role": str(entry["role"]), "content": entry_content})
- return conversation_history, None
- class OpenAICompatRoutesMixin:
- """/v1/chat/completions and /v1/responses handlers + SSE writers."""
- def _select_request_route(
- self, body: dict[str, Any], *, session_id, gateway_session_key, model_alias) -> tuple:
- """Resolve the model_routes alias + per-request overrides ->
- ``(route, agent_overrides, error_response_or_None)``."""
- from gateway.platforms.api_server import _error_response, _request_agent_overrides
- route = self._resolve_route(model_alias)
- overrides = _request_agent_overrides(
- body, virtual_model=self._model_name, allow_bare_model=self._direct_model_requests)
- err = self._request_route_conflict_error(
- session_id=session_id, gateway_session_key=gateway_session_key,
- requested_model=overrides.get("requested_model"),
- requested_provider=overrides.get("requested_provider"), route=route)
- return route, overrides, (_error_response(err, 400) if err else None)
- def _register_stream_approval(self, request, completion_id, stream_q, session_id) -> tuple:
- """Expose a streaming completion as a run for ``POST /v1/runs/{completion_id}/approval``
- (#51871) -> ``(notify, on_done)``. Owner + status are stamped like the session stream, so
- ``_load_owned_run`` finds it; keyed by the completion id (never the shared session key) so
- concurrent turns can't cross-resolve. ``on_done`` retires it when the agent task ends."""
- from gateway.platforms.api_server import _approval_request_event
- self._run_owners[completion_id] = self._run_idempotency_scope(request)
- self._set_run_status(completion_id, "running", session_id=session_id or "")
- self._run_approval_sessions[completion_id] = completion_id
- def _approval_notify(approval_data):
- event = _approval_request_event(completion_id, approval_data, session_id=session_id or "")
- self._set_run_status(completion_id, "waiting_for_approval", last_event="approval.request",
- approval=event)
- stream_q.put_threadsafe(("__approval__", event))
- def _on_done(fut):
- from gateway.platforms.api_server_runs import terminal_run_status
- self._run_approval_sessions.pop(completion_id, None)
- if fut.cancelled():
- status = "cancelled"
- elif fut.exception() is not None:
- status = "failed"
- else:
- result = (fut.result() or (None,))[0]
- status = terminal_run_status(result)[0] if isinstance(result, dict) else "completed"
- self._set_run_status(completion_id, status)
- self._release_run_owner_if_forgotten(completion_id)
- return _approval_notify, _on_done
- def _spawn_stream_agent(self, stream_q, *, on_done=None, run_agent=None, **run_kwargs) -> tuple:
- """Start ``_run_agent`` for an SSE writer -> ``(agent_task, agent_ref)``. ``agent_ref[0]``
- lets the writer interrupt on disconnect; the EOS sentinel is enqueued from the task's done
- callback so drain loops never race a polled ``agent_task.done()``. ``on_done(task)`` runs
- in that same callback, before EOS."""
- def _on_delta(delta):
- # None from the agent is a CLI box-close signal, not EOS — forwarding it would end
- # the stream early. Called from the run_conversation worker thread: put_threadsafe.
- if delta is not None:
- stream_q.put_threadsafe(delta)
- def _on_reasoning(text):
- # Structured reasoning deltas (#99552): the agent's reasoning_callback, not the
- # lossy 500-char ``reasoning.available`` progress preview. Tagged so the writers
- # keep them distinct from answer text.
- if text:
- stream_q.put_threadsafe(("__reasoning__", text))
- def _on_status(kind, message=None):
- # Lifecycle/warning status (provider wait, auto-recovery countdown, fallback switch) as a
- # ``hermes.status`` event, so a client sees why the stream is silent instead of a dead socket.
- from gateway.platforms.api_server import _redact_api_error_text
- text = _redact_api_error_text(message if message is not None else kind or "").strip()
- if text:
- stream_q.put_threadsafe(("__status__", {"kind": str(kind), "text": text}))
- agent_ref = [None]
- agent_task = asyncio.ensure_future((run_agent or self._run_agent)(
- stream_delta_callback=_on_delta, reasoning_callback=_on_reasoning, status_callback=_on_status,
- agent_ref=agent_ref, **run_kwargs))
- def _done(fut):
- try:
- if on_done is not None:
- on_done(fut)
- except Exception:
- logger.debug("[api_server] stream on_done hook failed", exc_info=True)
- finally:
- stream_q.put_nowait(None)
- agent_task.add_done_callback(_done)
- return agent_task, agent_ref
- async def _handle_chat_completions(self, request: web.Request) -> web.Response:
- """POST /v1/chat/completions — OpenAI Chat Completions format."""
- from gateway.platforms.api_server import (
- ThreadSafeAsyncQueue, _api_request_profile, _chat_usage_payload, _coerce_request_bool,
- _content_has_visible_payload, _derive_chat_session_id, _error_response, _invalid_request,
- _openai_error, _redact_api_error_text, _resolve_media_to_data_urls)
- # Bound total in-flight agent runs (configurable; #7483).
- limited = self._concurrency_limited_response()
- if limited is not None:
- return limited
- try:
- body = await request.json()
- except Exception:
- return _error_response("Invalid JSON in request body", 400)
- from gateway.platforms.api_server import _request_relay_metadata
- relay_metadata = _request_relay_metadata(body)
- messages = body.get("messages")
- if not messages or not isinstance(messages, list):
- return _invalid_request("Missing or invalid 'messages' field")
- stream = _coerce_request_bool(body.get("stream"), default=False)
- system_prompt, conversation_messages, messages_err = _parse_chat_messages(messages)
- if messages_err is not None:
- return messages_err
- user_message: Any = (conversation_messages[-1].get("content", "") if conversation_messages else "")
- history = conversation_messages[:-1]
- if not _content_has_visible_payload(user_message):
- return _invalid_request("No user message found in messages")
- # X-Hermes-Session-Key scopes long-term memory per channel; independent of
- # X-Hermes-Session-Id (the key persists across transcripts, the id rotates on /new).
- gateway_session_key, key_err = self._parse_session_key_header(request)
- if key_err is not None:
- return key_err
- # X-Hermes-Session-Id continues an existing session (history from state.db, not the body);
- # requires a configured API key or any client could read history by guessing ids.
- provided_session_id = request.headers.get("X-Hermes-Session-Id", "").strip()
- if provided_session_id:
- if not self._api_key:
- logger.warning(
- "Session continuation via X-Hermes-Session-Id rejected: "
- "no API key configured. Set API_SERVER_KEY to enable "
- "session continuity.")
- return _error_response("Session continuation requires API key authentication. "
- "Configure API_SERVER_KEY to enable this feature.", 403)
- # Same guard as the native gateway: ids are interpolated into on-disk filenames.
- from gateway.session import _is_path_unsafe
- if re.search(r'[\r\n\x00]', provided_session_id) or _is_path_unsafe(provided_session_id):
- return _invalid_request("Invalid session ID")
- if len(provided_session_id) > self._MAX_SESSION_HEADER_LEN:
- return _invalid_request("Session ID too long")
- session_id = provided_session_id
- try:
- db = await self._ensure_session_db_async()
- if db is not None:
- # #98619/#13437: a client-addressed id from before a compression rotation
- # must adopt the live continuation tip — history loads from it, the turn and
- # the wake target bind it, and a detached delegation delivery row persisted
- # on the tip is what this continuation consumes. Same canonical resolution
- # the delivery writer (gateway/wake.py) and /v1/runs use; fails open.
- from gateway.platforms.api_server_runs import _resolve_live_session_id
- session_id = await _resolve_live_session_id(self, provided_session_id)
- history = await asyncio.to_thread(db.get_messages_as_conversation, session_id)
- except Exception as e:
- logger.warning("Failed to load session history for %s: %s", session_id, e)
- history = []
- else:
- # Stable id from the conversation fingerprint so Open WebUI-style clients map onto
- # one Hermes session; namespaced by the routed profile (#123989).
- first_user = next(
- (cm.get("content", "") for cm in conversation_messages if cm.get("role") == "user"), "")
- session_id = _derive_chat_session_id(system_prompt, first_user, _api_request_profile.get())
- session_id, lookup_error = await self._implicit_chat_session(request, session_id)
- if lookup_error is not None:
- return lookup_error
- completion_id = f"chatcmpl-{uuid.uuid4().hex[:29]}"
- model_name = body.get("model", self._model_name)
- created = int(time.time())
- route, agent_overrides, selection_error = self._select_request_route(
- body, session_id=session_id, gateway_session_key=gateway_session_key,
- model_alias=model_name)
- if selection_error is not None:
- return selection_error
- run_kwargs = dict(
- user_message=user_message, conversation_history=history,
- ephemeral_system_prompt=system_prompt, session_id=session_id,
- gateway_session_key=gateway_session_key, **agent_overrides, route=route,
- relay_metadata=relay_metadata,
- # #98619: only an explicitly provided X-Hermes-Session-Id is wake-capable (the
- # header is 403-gated on API_SERVER_KEY, so the wake self-post can authenticate
- # and the client can resume the session by sending it again). A fingerprint-derived
- # id from a header-less client is NOT: delegate_task keeps its forced-sync fallback
- # there — the wake would hard-fail or land in history that client never reloads.
- session_history_delivery=("1" if provided_session_id else ""))
- run_kwargs.update(self._authority_chat_kwargs(request, provided_session_id))
- # This is presentation only. The ordinary API-key/session authorization
- # above still applies; it grants no internal ingress or control authority.
- if provided_session_id and body.get("hermes_notification_category") == "diagnostic":
- run_kwargs["notification_category"] = "diagnostic"
- if stream:
- _stream_q = ThreadSafeAsyncQueue()
- # tool_call_ids with an emitted "running": a "completed" without one (internal/
- # filtered tools) is dropped rather than orphaned on the wire.
- _started_tool_call_ids: set[str] = set()
- def _on_tool_start(tool_call_id, function_name, function_args):
- """``hermes.tool.progress`` status=running; ``_``-prefixed tools stay off the wire."""
- if not tool_call_id or function_name.startswith("_"):
- return
- _started_tool_call_ids.add(tool_call_id)
- from agent.display import build_tool_preview, get_tool_emoji
- label = build_tool_preview(function_name, function_args) or function_name
- _stream_q.put_threadsafe(("__tool_progress__", {
- "tool": function_name, "emoji": get_tool_emoji(function_name), "label": label,
- "toolCallId": tool_call_id, "status": "running"}))
- def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
- if not tool_call_id or tool_call_id not in _started_tool_call_ids:
- return
- _started_tool_call_ids.discard(tool_call_id)
- _stream_q.put_threadsafe(("__tool_progress__", {
- "tool": function_name, "toolCallId": tool_call_id, "status": "completed"}))
- # tool_progress_callback deliberately NOT wired: it would duplicate the structured
- # start/complete callbacks (which carry the tool_call id).
- # The completion id doubles as the run id; on_done drops the approval mapping once the
- # turn ends (POST /v1/runs/{id}/approval then answers 409) and sets the terminal status.
- approval_notify, end_stream_run = self._register_stream_approval(
- request, completion_id, _stream_q, session_id)
- agent_task, agent_ref = self._spawn_stream_agent(
- _stream_q, on_done=end_stream_run, tool_start_callback=_on_tool_start,
- tool_complete_callback=_on_tool_complete, approval_notify_callback=approval_notify,
- approval_session_key=completion_id, **run_kwargs)
- # #13437 identity contract: an explicit-header client keeps addressing the id it
- # sent; the response echoes that stable id while reads/writes adopt the live tip,
- # so a rotation mid-turn (after these headers are prepared) never changes what the
- # client should send next — it re-sends the same id and the tip resolution above
- # finds whatever session is live by then.
- return await self._write_sse_chat_completion(
- request, completion_id, model_name, created, _stream_q,
- agent_task, agent_ref, session_id=(provided_session_id or session_id),
- gateway_session_key=gateway_session_key)
- async def _compute_completion():
- return await self._run_agent(**run_kwargs)
- outcome, err = await self._run_idempotent(
- request, body, _compute_completion, log_label="chat completions",
- fingerprint_keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream",
- "hermes_notification_category"],
- route="chat_completions",
- )
- if err is not None:
- return err
- result, usage = outcome
- presentation_muted = result.get("_notification_presentation_suppressed") is True
- final_response = _resolve_media_to_data_urls(result.get("final_response") or "")
- completed, is_partial, is_failed, err_msg = _result_flags(result)
- if err_msg:
- err_msg = _redact_api_error_text(err_msg)
- budget_exhausted = _budget_exhausted(result)
- finish_reason = _finish_reason(completed, is_partial, is_failed, err_msg, budget_exhausted=budget_exhausted)
- # Same #13437 identity contract as the SSE path: an explicit-header client is echoed
- # the stable id it sent; a fingerprint-derived (header-less) turn keeps reporting the
- # id the agent actually resolved, so headerless clients still learn where the turn went.
- response_headers = {"X-Hermes-Session-Id": (provided_session_id or result.get("session_id", session_id))}
- if gateway_session_key:
- response_headers["X-Hermes-Session-Key"] = gateway_session_key
- # Hard fail (no usable text AND a real failure) -> 502 OpenAI error envelope so SDK
- # clients raise instead of rendering the failure string as message.content.
- if not final_response and (is_failed or is_partial):
- err_body = _openai_error(
- "" if presentation_muted else (err_msg or "Agent run did not produce a response."), err_type="server_error",
- code="agent_incomplete")
- err_body["error"]["hermes"] = {
- "completed": completed, "partial": is_partial, "failed": is_failed}
- response_headers["X-Hermes-Completed"] = "false"
- response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
- return web.json_response(err_body, status=502, headers=response_headers)
- # Soft partial (some text, run incomplete): 200 + finish_reason="length"/Hermes extras.
- response_data = {
- "id": completion_id, "object": "chat.completion", "created": created,
- "model": model_name,
- "choices": [{"index": 0, "message": {"role": "assistant", "content": "" if presentation_muted else final_response},
- "finish_reason": finish_reason}],
- "usage": _chat_usage_payload(usage)}
- # Non-streaming twin of ``delta.reasoning_content`` (#99552).
- reasoning_text = _turn_reasoning_text(history, user_message, result)
- if reasoning_text and not presentation_muted:
- response_data["choices"][0]["message"]["reasoning_content"] = reasoning_text
- if is_partial or is_failed or not completed:
- response_data["hermes"] = _hermes_extras(
- completed, is_partial, is_failed, "" if presentation_muted else err_msg, finish_reason,
- budget_exhausted=budget_exhausted)
- response_headers["X-Hermes-Completed"] = "false"
- response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
- if err_msg and not presentation_muted:
- response_headers["X-Hermes-Error"] = _redact_api_error_text(err_msg, limit=200)
- return web.json_response(response_data, headers=response_headers)
- def _chat_request_id(self, request: web.Request) -> Optional[str]:
- """Durable admission identity of a Chat Completions Idempotency-Key. It carries the
- authenticated namespace: a rotated API key (or another profile) reusing a client key is a
- new principal, not a retry."""
- key = request.headers.get('Idempotency-Key')
- return f'chat:{self._run_idempotency_scope(request)}:{key}' if key else None
- def _authority_chat_kwargs(self, request: web.Request, provided_session_id) -> dict:
- """Admission identity of a Chat Completions turn under session authority; none before it."""
- if getattr(self.gateway_runner, 'session_authority', None) is None:
- return {}
- return dict(request_id=self._chat_request_id(request), history_from_session=bool(provided_session_id),
- run_owner_scope=self._run_idempotency_scope(request))
- async def _implicit_chat_session(self, request: web.Request, session_id: str) -> tuple:
- """A derived (header-less) id whose session was deleted names a new conversation with the
- same opening: the owning profile's store picks its generation (``implicit_api_session``).
- -> ``(session_id, None)`` or ``(session_id, error response)``, before either response starts:
- a refusal keeps its reason and a busy or unreadable store is a retryable 503
- ``storage_unavailable`` (``store_error_response``); anything else is the OpenAI 500."""
- if getattr(self.gateway_runner, 'session_authority', None) is None:
- return session_id, None
- from gateway.session_api import implicit_api_session
- from gateway.session_authorities import active_authority
- authority = active_authority(self.gateway_runner)
- if authority is None:
- return session_id, None # admission refuses the unserved profile with its own verdict
- import sqlite3
- from hermes_state_runtime import RuntimeStoreError
- try:
- return await asyncio.to_thread(implicit_api_session, authority.db, session_id,
- self._chat_request_id(request)), None
- except (RuntimeStoreError, sqlite3.Error, OSError) as exc:
- from gateway.platforms.api_server import _openai_error
- from gateway.platforms.api_server_authority_runs import store_error_response
- return session_id, store_error_response(_openai_error, exc)
- except Exception as e:
- from gateway.platforms.api_server import _error_response
- logger.error("Error resolving the chat completions session: %s", e, exc_info=True)
- return session_id, _error_response(f"Internal server error: {e}", 500, err_type="server_error")
- async def _run_idempotent(
- self, request: web.Request, body: dict[str, Any], compute, *,
- log_label: str, fingerprint_keys: list[str], route: str) -> tuple:
- """Run ``compute()`` once per (principal scope, logical route, Idempotency-Key) + body fingerprint
- -> ``((result, usage), None)`` or ``(None, 500 response)``.
- ``_idem_cache`` is process-global: under ``gateway.multiplex_profiles`` every profile's
- ``/p/<profile>/v1/...`` mirror shares it, so the key carries ``_run_idempotency_scope`` (the same
- ``sha256(profile, expected API key)`` namespace the durable ``/v1/runs`` API uses) — a client key
- colliding across profiles, or a rotated API_SERVER_KEY, never replays another principal's response.
- ``route`` is the logical endpoint (``/v1/...`` and its ``/p/<profile>/v1/...`` alias are the same
- route), folded into the key because the store keeps the fingerprint only as the slot's value.
- """
- from gateway.platforms.api_server import _error_response, _idem_cache, _make_request_fingerprint
- from hermes_state_runtime import RuntimeStoreError
- idempotency_key = request.headers.get("Idempotency-Key")
- try:
- # Durable /v1/runs admissions own idempotency under session authority; the in-memory
- # cache only serves the pre-authority path.
- if idempotency_key and getattr(self.gateway_runner, 'session_authority', None) is None:
- principal_scope = self._run_idempotency_scope(request)
- scoped_key = f"{principal_scope}\0{route}\0{idempotency_key}"
- fp = _make_request_fingerprint(body, keys=fingerprint_keys)
- result, usage = await _idem_cache.get_or_set(scoped_key, fp, compute)
- else:
- result, usage = await compute()
- return (result, usage), None
- except RuntimeStoreError as exc:
- from gateway.platforms.api_server import _openai_error
- from gateway.platforms.api_server_authority_runs import store_error_response
- return None, store_error_response(_openai_error, exc)
- except Exception as e:
- logger.error("Error running agent for %s: %s", log_label, e, exc_info=True)
- message = "" if getattr(e, "_notification_presentation_suppressed", False) is True else f"Internal server error: {e}"
- return None, _error_response(message, 500, err_type="server_error")
- async def _prepare_sse_response(
- self, request: web.Request, session_id: Optional[str], gateway_session_key: Optional[str],
- ) -> web.StreamResponse:
- """Open a prepared SSE StreamResponse with CORS + session headers (the CORS middleware
- can't inject headers after ``prepare()`` flushes them, so they are resolved here)."""
- sse_headers = self._sse_headers(request)
- if session_id:
- sse_headers["X-Hermes-Session-Id"] = session_id
- if gateway_session_key:
- sse_headers["X-Hermes-Session-Key"] = gateway_session_key
- response = web.StreamResponse(status=200, headers=sse_headers)
- await response.prepare(request)
- return response
- async def _write_sse_chat_completion(
- self, request: web.Request, completion_id: str, model: str,
- created: int, stream_q, agent_task, agent_ref=None, session_id: str | None = None,
- gateway_session_key: str | None = None) -> web.StreamResponse:
- """Stream ``chat.completion.chunk`` frames from the agent's delta queue. On client
- disconnect the agent is interrupted (stops LLM calls), then its task wrapper cancelled."""
- from gateway.platforms.api_server import (
- _abandon_agent_task, _chat_usage_payload, _resolve_media_to_data_urls, _sse_frame)
- response = await self._prepare_sse_response(request, session_id, gateway_session_key)
- def _chunk(delta: dict[str, Any], finish_reason=None, **extra) -> dict[str, Any]:
- return {"id": completion_id, "object": "chat.completion.chunk", "created": created,
- "model": model,
- "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], **extra}
- content_sent = False
- try:
- await response.write(_sse_frame(_chunk({"role": "assistant"})))
- async for delta in _iter_stream_items(stream_q, agent_task, response):
- if delta is None:
- break
- if isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__tool_progress__":
- # Custom event: tool lifecycle for frontends without markers in history.
- if not self._tool_progress_events:
- continue # opted out for strict OpenAI clients (#12020)
- await response.write(_sse_frame(delta[1], event="hermes.tool.progress"))
- elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__reasoning__":
- # DeepSeek-style ``delta.reasoning_content`` (#99552), the field Open WebUI,
- # opencode and the Vercel AI SDK render as a thinking block.
- await response.write(_sse_frame(_chunk({"reasoning_content": delta[1]})))
- elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__status__":
- await response.write(_sse_frame(delta[1], event="hermes.status"))
- elif isinstance(delta, tuple) and len(delta) == 2 and delta[0] == "__approval__":
- await response.write(_sse_frame(delta[1], event="approval.request"))
- else:
- if delta:
- content_sent = True
- await response.write(_sse_frame(_chunk({"content": delta})))
- # The agent can fail after the queue drains (task raises / result flagged failed or
- # partial): surface a non-"stop" finish_reason like the non-streaming path.
- usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
- result = agent_error = None
- try:
- result, agent_usage = await agent_task
- usage = agent_usage or usage
- except Exception as exc:
- agent_error = exc
- logger.error("Agent task %s failed during SSE streaming: %s", completion_id, exc)
- completed, is_partial, is_failed, err_msg = _result_flags(result)
- if agent_error is not None:
- is_failed = True
- err_msg = err_msg or str(agent_error)
- budget_exhausted = _budget_exhausted(result)
- finish_reason = _finish_reason(completed, is_partial, is_failed, err_msg, agent_error,
- budget_exhausted=budget_exhausted)
- finish_chunk = _chunk({}, finish_reason, usage=_chat_usage_payload(usage))
- presentation_muted = (
- (isinstance(result, dict) and result.get("_notification_presentation_suppressed") is True)
- or getattr(agent_error, "_notification_presentation_suppressed", False) is True
- )
- # Recovery paths (guardrail halt, partial_stream_recovery, fallback prior-turn
- # content) can return a final_response without firing any content delta; emit it
- # once so the client does not see an empty stream (#31449). Mirrors
- # _ResponsesStream.collect_result for /v1/responses.
- if not content_sent and not presentation_muted and isinstance(result, dict):
- fallback_text = _resolve_media_to_data_urls(result.get("final_response") or "")
- if fallback_text:
- await response.write(_sse_frame(_chunk({"content": fallback_text})))
- elif not presentation_muted:
- # Chat chunks can only append: a non-append rewrite follows the streamed text (as in the CLI).
- tail, appended = _post_stream_transform(result)
- if tail:
- await response.write(_sse_frame(_chunk({"content": tail if appended else _transformed_notice() + tail})))
- if finish_reason != "stop":
- if err_msg and not presentation_muted:
- finish_chunk["error"] = {
- "message": err_msg,
- "type": type(agent_error).__name__ if agent_error else "agent_error"}
- finish_chunk["hermes"] = _hermes_extras(
- completed, is_partial, is_failed, "" if presentation_muted else err_msg, finish_reason,
- budget_exhausted=budget_exhausted)
- await response.write(_sse_frame(finish_chunk))
- await response.write(b"data: [DONE]\n\n")
- except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
- await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
- logger.info("SSE client disconnected; interrupted agent task %s", completion_id)
- except Exception:
- # Agent crashed mid-stream: an error chunk beats a TransferEncodingError.
- import traceback as _tb
- logger.error("Agent crashed mid-stream for %s: %s", completion_id, _tb.format_exc()[:300])
- with suppress(Exception):
- await response.write(_sse_frame(_chunk({}, "error")))
- await response.write(b"data: [DONE]\n\n")
- return response
- async def _write_sse_responses(
- self, request: web.Request, response_id: str, model: str, created_at: int, stream_q,
- agent_task, agent_ref, conversation_history: list[dict[str, str]], user_message: str,
- instructions: Optional[str], conversation: Optional[str], store: bool, session_id: str,
- gateway_session_key: Optional[str] = None, durable_key=None, terminal_replay=False) -> web.StreamResponse:
- """Write the SSE stream for POST /v1/responses.
- Events: ``response.created`` -> ``output_text.delta/done`` + ``output_item.added/done``
- (reasoning / function_call / function_call_output) + ``reasoning_summary_part/text.*``
- -> ``response.completed`` (non-streaming envelope)
- or ``response.failed``. On disconnect the agent is interrupted and, with ``store=True``,
- an ``incomplete`` snapshot replaces ``in_progress`` so GET / chaining still work.
- ``durable_key`` (canonical Idempotency-Key record) is written only once the admitted turn
- settled, so an exact retry replays this stream instead of admitting different history.
- """
- from gateway.platforms.api_server import _abandon_agent_task, _redact_api_error_text
- response = await self._prepare_sse_response(request, session_id, gateway_session_key)
- st = _ResponsesStream(
- self, response, response_id=response_id, model=model, created_at=created_at,
- conversation_history=conversation_history, user_message=user_message,
- instructions=instructions, conversation=conversation, store=store, session_id=session_id,
- durable_key=durable_key, terminal_replay=terminal_replay)
- try:
- await st.emit_created()
- async for item in _iter_stream_items(stream_q, agent_task, response):
- if item is None: # EOS sentinel
- st.cancel_batch_timer()
- await st.flush_batch()
- break
- await st.dispatch(item)
- await st.flush_batch()
- await st.collect_result(agent_task)
- await st.close_message_item()
- if st.agent_error and not (durable_key and st.result):
- await st.emit_failed()
- else:
- await st.emit_completed()
- except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
- await st.persist_incomplete_if_needed()
- await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
- logger.info("SSE client disconnected; interrupted agent task %s", response_id)
- except asyncio.CancelledError:
- # Server-side cancellation (shutdown, timeout): abandon the agent first (a second cancel
- # landing in the persist await must not skip it), persist incomplete, then re-raise.
- await _abandon_agent_task(
- agent_ref, agent_task, "SSE task cancelled",
- reap_source="api_server_sse_cancelled", await_cancel=False)
- await st.persist_incomplete_if_needed()
- logger.info("SSE task cancelled; persisted incomplete snapshot for %s", response_id)
- raise
- except Exception as exc:
- # Unhandled agent error (BadRequestError, AuthenticationError, ...): emit
- # response.failed and end the stream cleanly (no TransferEncodingError).
- import traceback as _tb
- await st.persist_incomplete_if_needed()
- st.agent_error = _redact_api_error_text(_tb.format_exc())
- with suppress(Exception):
- await st.emit_crash(exc)
- logger.error("Agent crashed mid-stream for %s: %s", response_id, str(st.agent_error)[:300])
- return response
- async def _replay_sse_responses(self, request: web.Request, replay: dict[str, Any]) -> web.StreamResponse:
- """Re-emit a settled idempotent record as SSE. A record made by the streaming path replays
- its exact frames; one made by the nonstreaming path gets ``created`` + its terminal event."""
- from gateway.platforms.api_server import _sse_frame
- headers = replay.get('headers') or {}
- response = await self._prepare_sse_response(
- request, headers.get('X-Hermes-Session-Id'), headers.get('X-Hermes-Session-Key'))
- events = replay.get('events')
- if not events:
- env = replay['response']
- created = {k: env[k] for k in ('id', 'object', 'created_at', 'model')}
- created.update(status='in_progress', output=[])
- events = [['response.created', {'type': 'response.created', 'response': created, 'sequence_number': 0}],
- ['response.' + env['status'], {'type': 'response.' + env['status'], 'response': env,
- 'sequence_number': 1}]]
- for event_type, data in events:
- await response.write(_sse_frame(data, event=event_type))
- return response
- async def _responses_durable_replay(self, request: web.Request, body: dict[str, Any], stream: bool) -> tuple:
- """Canonical Idempotency-Key record for POST /v1/responses ->
- ``(durable_key, idempotency_scope, idempotency_key, replayed_response_or_None)``."""
- durable_key = idempotency_scope = None
- idempotency_key = request.headers.get('Idempotency-Key')
- canonical = getattr(self.gateway_runner, 'session_authority', None) is not None
- if canonical and idempotency_key:
- # An exact retry replays the committed response BEFORE the conversation name is
- # expanded: the first success already advanced the conversation, so re-expanding
- # would build a different admission payload and refuse the retry as a conflict.
- # Streaming and nonstreaming share the record: transport mode is not identity.
- from gateway.platforms.api_server import _make_request_fingerprint
- idempotency_scope = self._run_idempotency_scope(request)
- durable_key = (f'idem:{idempotency_scope}:{idempotency_key}',
- _make_request_fingerprint(body, keys=_responses_fingerprint_keys(body)))
- from gateway.platforms.api_server import _openai_error
- from gateway.platforms.api_server_response_admissions import request_binding
- response_store = self._current_response_store()
- binding = request_binding(self, request, durable_key)
- matches, replay = await response_store.off_loop(_matching_replay, response_store, durable_key[0], binding)
- if not matches:
- return durable_key, idempotency_scope, idempotency_key, web.json_response(
- _openai_error('admission_conflict', code='admission_conflict'), status=409)
- if replay is not None:
- if replay.get('fingerprint') != durable_key[1]:
- return durable_key, idempotency_scope, idempotency_key, web.json_response(
- _openai_error('admission_conflict', code='admission_conflict'), status=409)
- if not _settled(replay):
- # Never re-run a request whose earlier attempt may have executed.
- return durable_key, idempotency_scope, idempotency_key, web.json_response(
- _openai_error('unknown_execution', code='unknown_execution'), status=409)
- if stream:
- return (durable_key, idempotency_scope, idempotency_key,
- await self._replay_sse_responses(request, replay))
- return durable_key, idempotency_scope, idempotency_key, web.json_response(
- replay['response'], headers=replay.get('headers') or {})
- return durable_key, idempotency_scope, idempotency_key, None
- async def _handle_responses(self, request: web.Request) -> web.Response:
- """POST /v1/responses — OpenAI Responses API format."""
- from gateway.platforms.api_server import (
- ThreadSafeAsyncQueue, _coerce_request_bool,
- _error_response, _invalid_request, _redact_api_error_text,
- _resolve_media_to_data_urls, _responses_usage_payload)
- # Bound total in-flight agent runs (configurable; #7483).
- limited = self._concurrency_limited_response()
- if limited is not None:
- return limited
- gateway_session_key, key_err = self._parse_session_key_header(request)
- if key_err is not None:
- return key_err
- try:
- body = await request.json()
- except Exception:
- return _invalid_request("Invalid JSON in request body")
- raw_input = body.get("input")
- if raw_input is None:
- return _error_response("Missing 'input' field", 400)
- instructions = body.get("instructions")
- previous_response_id = body.get("previous_response_id")
- conversation = body.get("conversation")
- store = _coerce_request_bool(body.get("store"), default=True)
- if conversation and previous_response_id:
- return _error_response("Cannot use both 'conversation' and 'previous_response_id'", 400)
- stream = _coerce_request_bool(body.get("stream"), default=False)
- durable_key, idempotency_scope, idempotency_key, replayed = await self._responses_durable_replay(
- request, body, stream)
- if replayed is not None:
- return replayed
- from gateway.platforms.api_server_response_admissions import prepare_context
- from hermes_state_runtime import RuntimeStoreError
- try:
- context, error = await prepare_context(
- self, request, body, gateway_session_key, durable_key, idempotency_scope, idempotency_key)
- except RuntimeStoreError as exc:
- from gateway.platforms.api_server import _openai_error
- from gateway.platforms.api_server_authority_runs import store_error_response
- return store_error_response(_openai_error, exc)
- if error is not None:
- return error
- session_id, user_message = context['session_id'], context['user_message']
- conversation_history, instructions = context['conversation_history'], context['instructions']
- run_kwargs, run_agent = context['run_kwargs'], context['run_agent']
- response_store = self._current_response_store()
- response_id, created_at = await _response_identity(response_store, durable_key)
- response_model = body.get('model', self._model_name)
- if durable_key:
- response_model = await response_store.off_loop(response_store.request_model, durable_key[0], response_model)
- if stream:
- _stream_q = ThreadSafeAsyncQueue()
- def _on_tool_progress(event_type, name, preview, args, **kwargs):
- return # structured start/complete callbacks carry the call id; progress ignored
- def _on_tool_start(tool_call_id, function_name, function_args):
- _stream_q.put_threadsafe(("__tool_started__", {
- "tool_call_id": tool_call_id, "name": function_name,
- "arguments": function_args or {}}))
- def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
- _stream_q.put_threadsafe(("__tool_completed__", {
- "tool_call_id": tool_call_id, "name": function_name,
- "arguments": function_args or {}, "result": function_result}))
- def _on_commentary(text, *, already_streamed: bool = False):
- # Already-streamed text went out as output_text.delta of the final item; a second
- # copy as a commentary item would duplicate it.
- if not already_streamed and isinstance(text, str) and text.strip():
- _stream_q.put_threadsafe(("__commentary__", {"text": text}))
- agent_task, agent_ref = self._spawn_stream_agent(
- _stream_q, run_agent=run_agent, tool_progress_callback=_on_tool_progress,
- tool_start_callback=_on_tool_start, tool_complete_callback=_on_tool_complete,
- interim_assistant_callback=_on_commentary, **run_kwargs)
- return await self._write_sse_responses(
- request=request, response_id=response_id,
- model=response_model, created_at=created_at,
- stream_q=_stream_q, agent_task=agent_task, agent_ref=agent_ref,
- conversation_history=conversation_history, user_message=user_message,
- instructions=instructions, conversation=conversation, store=store,
- session_id=session_id, gateway_session_key=gateway_session_key,
- durable_key=durable_key, terminal_replay=context.get('terminal_replay', False))
- async def _compute_response():
- return await run_agent(**run_kwargs)
- outcome, err = await self._run_idempotent(
- request, body, _compute_response, log_label="responses",
- fingerprint_keys=list(_responses_fingerprint_keys(body)), route="responses",
- )
- if err is not None:
- return err
- if durable_key is not None:
- response_store = self._current_response_store()
- replay = await response_store.off_loop(response_store.get, durable_key[0])
- if _settled(replay):
- return web.json_response(replay['response'], headers=replay.get('headers') or {})
- result, usage = outcome
- final_response = _resolve_media_to_data_urls(result.get("final_response", ""))
- if not final_response:
- final_response = _redact_api_error_text(result.get("error", "(No response generated)"))
- full_history = self._build_response_conversation_history(
- conversation_history, user_message, result, final_response,
- tool_output_max_chars=self._history_tool_output_max_chars)
- # _run_agent's effective session id carries compression rotations; storing it keeps
- # previous_response_id chaining off the pre-rotation session (else compression re-fires).
- _result_sid = result.get("session_id") if isinstance(result, dict) else None
- _effective_session_id = (
- _result_sid if isinstance(_result_sid, str) and _result_sid else session_id)
- # Output items = current turn only (AIAgent returns a full transcript; mocked paths
- # only the current-turn suffix).
- output_start_index = self._response_messages_turn_start_index(
- conversation_history, user_message, result)
- response_data = {
- "id": response_id, "object": "response", "status": _response_status(result),
- "created_at": created_at, "model": response_model,
- "output": self._extract_output_items(result, start_index=output_start_index,
- **({'response_id': response_id} if durable_key else {})),
- "usage": _responses_usage_payload(usage)}
- response_store = self._current_response_store() # at use time: the store can be swapped mid-turn
- if durable_key:
- response_data['output'] = await response_store.off_loop(
- response_store.order_output, durable_key[0], response_data['output'])
- if response_data['status'] == 'failed':
- response_data['error'] = {'code': 'agent_error', 'message': _redact_api_error_text(
- result.get('error') or 'The admitted turn failed.')}
- snapshot = (response_id, {"response": response_data, "conversation_history": full_history,
- "instructions": instructions, "session_id": _effective_session_id}) if store else None
- response_headers = {"X-Hermes-Session-Id": _effective_session_id}
- if gateway_session_key:
- response_headers["X-Hermes-Session-Key"] = gateway_session_key
- replay_record = (durable_key[0], {'fingerprint': durable_key[1], 'response': response_data,
- 'headers': response_headers}) if durable_key is not None else None
- if snapshot is not None or replay_record is not None:
- # Awaited before answering, so the returned id (and its replay record) read back at once.
- await response_store.off_loop(_store_nonstream_response, response_store, snapshot, replay_record,
- conversation, bool(context.get('terminal_replay')))
- return web.json_response(response_data, headers=response_headers)
- async def _handle_get_response(self, request: web.Request) -> web.Response:
- """GET /v1/responses/{response_id} — retrieve a stored response."""
- from gateway.platforms.api_server import _error_response
- auth_err = self._check_auth(request)
- if auth_err:
- return auth_err
- response_id = request.match_info["response_id"]
- store = self._current_response_store()
- stored = await store.off_loop(store.get, response_id)
- if stored is None:
- return _error_response(f"Response not found: {response_id}", 404)
- return web.json_response(stored["response"])
- async def _handle_delete_response(self, request: web.Request) -> web.Response:
- """DELETE /v1/responses/{response_id} — delete a stored response."""
- from gateway.platforms.api_server import _error_response
- auth_err = self._check_auth(request)
- if auth_err:
- return auth_err
- response_id = request.match_info["response_id"]
- store = self._current_response_store()
- if not await store.off_loop(store.delete, response_id):
- return _error_response(f"Response not found: {response_id}", 404)
- return web.json_response({"id": response_id, "object": "response", "deleted": True})
- @staticmethod
- def _build_response_conversation_history(
- conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
- final_response: Any, *, tool_output_max_chars: int = 0) -> list[dict[str, Any]]:
- """Build the stored Responses transcript without duplicating history.
- A compressed transcript (``result["_compressed"]``) shares no input-history prefix, so
- turn-start detection fails; prepending the uncompressed history would bloat the stored
- context and re-trigger compression every request — it is stored as-is instead.
- ``tool_output_max_chars`` > 0 caps tool outputs / tool-call argument blobs in the
- stored copy (gateway.api_server.history_tool_output_max_chars; 0 = store verbatim).
- """
- from gateway.platforms.api_server import APIServerAdapter
- prior = list(conversation_history)
- current_user = {"role": "user", "content": user_message}
- if isinstance(result, dict) and isinstance(result.get("_turn_user_row"), dict):
- current_user = dict(result["_turn_user_row"]) # settled canonical turn: carries api_content
- agent_messages = result.get("messages") if isinstance(result, dict) else None
- if isinstance(agent_messages, list) and agent_messages:
- turn_start = APIServerAdapter._response_messages_turn_start_index(
- conversation_history, user_message, result)
- # turn_start == 0: compression rewrote the transcript or agent_messages is turn-only.
- if turn_start or result.get("_compressed"):
- history = list(agent_messages)
- else:
- history = prior + [current_user] + agent_messages
- else:
- history = prior + [current_user, {"role": "assistant", "content": final_response}]
- return _cap_history_tool_outputs(history, tool_output_max_chars)
- @staticmethod
- def _response_messages_turn_start_index(
- conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
- ) -> int:
- """Index where this turn starts in a transcript-shaped result["messages"] (0 = all)."""
- from gateway.platforms.api_server_turn_boundary import response_turn_start_index
- return response_turn_start_index(conversation_history, user_message, result)
- @classmethod
- def _turn_transcript_messages(
- cls, conversation_history: list[dict[str, Any]], user_message: Any, result: dict[str, Any],
- ) -> list[dict[str, Any]]:
- """This turn's assistant/tool messages in client-safe shape: clients accumulating
- ``assistant.delta`` into one buffer cannot reconstruct assistant segments that preceded
- tool calls, so ``run.completed`` carries the authoritative per-turn transcript.
- Emitting the authoritative per-turn transcript on ``run.completed`` lets any SSE consumer reconcile
- its live view against ground truth without a separate ``GET /messages`` round-trip. Purely additive:
- clients that ignore the field are unaffected. Refs #34703.
- """
- agent_messages = result.get("messages") if isinstance(result, dict) else None
- if not isinstance(agent_messages, list) or not agent_messages:
- return []
- start = cls._response_messages_turn_start_index(conversation_history, user_message, result)
- out: list[dict[str, Any]] = []
- for msg in agent_messages[start:]:
- if not isinstance(msg, dict) or msg.get("role") not in {"assistant", "tool"}:
- continue
- # _message_response projects compaction scaffolding; pure handoffs are "hidden".
- projected = cls._message_response(msg)
- if projected.get("display_kind") != "hidden":
- out.append(projected)
- return out
- @staticmethod
- def _extract_output_items(result: dict[str, Any], start_index: int = 0, *, response_id=None) -> list[dict[str, Any]]:
- """Output items from ``result["messages"][start_index:]``: ``function_call`` per assistant
- tool_call, ``function_call_output`` per tool message, then the final ``message``."""
- from gateway.platforms.api_server import _redact_api_error_text
- items: list[dict[str, Any]] = []
- messages = result.get("messages", [])
- if start_index > 0:
- messages = messages[start_index:]
- for msg in messages:
- role = msg.get("role")
- reasoning = msg.get("reasoning") if role == "assistant" else None
- if isinstance(reasoning, str) and reasoning.strip():
- # Precedes this message's function_call items, like the SSE writer closes a
- # thinking burst before the next tool item opens (#99552).
- items.append(_reasoning_item(reasoning))
- if role == "assistant" and msg.get("tool_calls"):
- for tc in msg["tool_calls"]:
- func = tc.get("function", {})
- # Already executed server-side; replayed for structured tool UI only, so
- # marked completed (matching the SSE path) — never pending client calls.
- items.append({
- "id": f"fc_{uuid.uuid4().hex[:24]}", "type": "function_call",
- "status": "completed", "name": func.get("name", ""),
- "arguments": func.get("arguments", ""),
- "call_id": tc.get("id", "")})
- elif role == "tool":
- items.append({
- "id": f"fco_{uuid.uuid4().hex[:24]}", "type": "function_call_output",
- "status": "completed", "call_id": msg.get("tool_call_id", ""),
- "output": msg.get("content", "")})
- final = result.get("final_response", "") or _redact_api_error_text(
- result.get("error", "(No response generated)"))
- items.append(_message_item(final))
- if response_id is not None:
- from gateway.platforms.api_server_response_identity import identify_output
- return identify_output(items, response_id)
- return items
|