api_server_runs.py 82 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572
  1. """Durable ``/v1/runs`` admission, status, events, and control handlers."""
  2. import asyncio
  3. import contextvars
  4. import functools
  5. import hashlib
  6. import json
  7. import logging
  8. import os
  9. import threading
  10. import time
  11. import uuid
  12. from collections import deque
  13. from contextlib import suppress
  14. from dataclasses import dataclass
  15. from typing import Any, Callable, Dict, List, Optional, Tuple
  16. try:
  17. from aiohttp import web
  18. except ImportError:
  19. web = None # type: ignore[assignment]
  20. try:
  21. from aiohttp.web_request import RequestKey
  22. except ImportError:
  23. # Separate block: aiohttp < 3.14 lacks RequestKey, and a shared except
  24. # would reset the already-imported ``web`` to None (500 on POST /v1/runs).
  25. RequestKey = None # type: ignore[assignment,misc]
  26. from gateway.platforms.api_server_room_grants import _json_error, _room_grant_error_response
  27. from gateway.turn_executor import _UnboundedThreadExecutor
  28. from gateway.platforms.api_server_run_idempotency import TERMINAL_STATUSES
  29. logger = logging.getLogger("gateway.platforms.api_server")
  30. # Executor-thread lifetime for API-server agent turns (#116535). The handler-side
  31. # ``_inflight_agent_runs`` count in api_server.py drops in the handler's ``finally`` when the
  32. # handler task is cancelled (disconnect/shutdown) while the executor thread behind
  33. # ``run_in_executor`` keeps running -- so the shutdown close gate cannot rely on it. This
  34. # module-level count is taken on the submitting thread before ``run_in_executor`` and released
  35. # in the worker thread's own ``finally``, which also keeps it visible past ``adapters.clear()``.
  36. _API_WORKER_LOCK = threading.Lock()
  37. _API_WORKER_LIVE = 0
  38. # API-server agent turns and cron fires run on their own executor, as Desktop and messaging turns
  39. # do (``GatewayRunner._get_executor``). asyncio's default pool is ``min(32, cpu_count + 4)`` threads
  40. # shared with every ``asyncio.to_thread`` SessionDB read in the session handlers, so once live
  41. # turns filled it each /api/sessions read queued behind a whole turn (#144362). Turn concurrency is
  42. # bounded where it is refused: ``gateway.api_server.max_concurrent_runs`` (429 past the cap).
  43. # Created on first use and retired at adapter disconnect (``_close_run_state``), so its parked
  44. # threads do not outlive the adapter that used them; the loop's default executor this replaced
  45. # went away with its loop the same way.
  46. _API_TURN_EXECUTOR: _UnboundedThreadExecutor | None = None
  47. _API_TURN_EXECUTOR_LOCK = threading.Lock()
  48. def api_turn_executor() -> _UnboundedThreadExecutor:
  49. """The executor that runs API-server agent turns and cron fires (never the loop's default)."""
  50. global _API_TURN_EXECUTOR
  51. with _API_TURN_EXECUTOR_LOCK:
  52. if _API_TURN_EXECUTOR is None:
  53. _API_TURN_EXECUTOR = _UnboundedThreadExecutor(thread_name_prefix="hermes-api-turn")
  54. return _API_TURN_EXECUTOR
  55. def retire_api_turn_executor() -> _UnboundedThreadExecutor | None:
  56. """Shut the API turn executor down without waiting and return it (None if never used).
  57. Parked workers exit now; a worker still inside a turn finishes it, releases its
  58. ``api_worker_live_count`` share as before, and exits instead of parking. A later submit gets
  59. a fresh executor, so another adapter still serving keeps working.
  60. """
  61. global _API_TURN_EXECUTOR
  62. with _API_TURN_EXECUTOR_LOCK:
  63. executor, _API_TURN_EXECUTOR = _API_TURN_EXECUTOR, None
  64. if executor is not None:
  65. executor.shutdown(wait=False)
  66. return executor
  67. def api_worker_live_count() -> int:
  68. """Executor threads still inside an API-server agent turn (``_run_agent`` and ``/v1/runs``)."""
  69. with _API_WORKER_LOCK:
  70. return _API_WORKER_LIVE
  71. async def on_turn_executor(fn, /, *args, **kwargs):
  72. """``asyncio.to_thread`` (context copied the same way) on the API turn executor."""
  73. call = functools.partial(contextvars.copy_context().run, fn, *args, **kwargs)
  74. return await asyncio.get_running_loop().run_in_executor(api_turn_executor(), call)
  75. def _submit_api_worker(loop, fn):
  76. """``loop.run_in_executor(api_turn_executor(), fn)`` with the worker-lifetime count held for the submission.
  77. Increment on the submitting (handler) thread so the count is live before the worker can
  78. exit; decrement in the worker thread's own ``finally`` so handler cancellation cannot drop
  79. it early. When submission itself fails (executor shut down, or no thread can be started
  80. -> RuntimeError) the worker never runs, so the count is released here instead — a leaked
  81. count would make the shutdown close gate skip the SessionDB close for the process lifetime.
  82. """
  83. global _API_WORKER_LIVE
  84. with _API_WORKER_LOCK:
  85. _API_WORKER_LIVE += 1
  86. def _counted():
  87. try:
  88. return fn()
  89. finally:
  90. global _API_WORKER_LIVE
  91. with _API_WORKER_LOCK:
  92. _API_WORKER_LIVE -= 1
  93. try:
  94. return loop.run_in_executor(api_turn_executor(), _counted)
  95. except BaseException:
  96. with _API_WORKER_LOCK:
  97. _API_WORKER_LIVE -= 1
  98. raise
  99. _ROOM_RETENTION_REQUEST_KEY = (
  100. RequestKey("hermes.room_run_retention_until", float) if RequestKey is not None
  101. else "hermes.room_run_retention_until")
  102. # Forwarded subagent lifecycle fields; free-text ones are secret-redacted.
  103. _SUBAGENT_EVENT_KEYS = (
  104. "goal", "task_count", "task_index", "subagent_id", "child_session_id", "delegation_id", "parent_id",
  105. "depth", "model", "tool_count", "status", "summary", "duration_seconds", "input_tokens",
  106. "output_tokens", "reasoning_tokens", "api_calls", "cost_usd", "files_read", "files_written",
  107. "output_tail")
  108. _SUBAGENT_TEXT_KEYS = ("goal", "summary", "output_tail")
  109. # Terminal usage payload: (wire key, agent attribute), in wire order. Cache reads ride along so a
  110. # cost poller does not book them as full-price input (#102101).
  111. _USAGE_FIELDS = (
  112. ("input_tokens", "session_prompt_tokens"), ("output_tokens", "session_completion_tokens"),
  113. ("total_tokens", "session_total_tokens"), ("cache_read_tokens", "session_cache_read_tokens"),
  114. ("cache_write_tokens", "session_cache_write_tokens"))
  115. # Tool-progress event -> SSE payload fields (tool_name, preview, kwargs); key order is wire format.
  116. def _call_id(kw: dict[str, Any]) -> dict[str, Any]:
  117. return {"tool_call_id": kw["tool_call_id"]} if kw.get("tool_call_id") else {}
  118. _FIXED_EVENT_FIELDS = {
  119. "tool.started": lambda tool, preview, kw: {"tool": tool, "preview": preview, **_call_id(kw)},
  120. "tool.completed": lambda tool, preview, kw: {
  121. "tool": tool, "duration": round(kw.get("duration", 0), 3), "error": kw.get("is_error", False),
  122. **_call_id(kw)},
  123. "reasoning.available": lambda tool, preview, kw: {"text": preview or ""}}
  124. _TOOL_COMPLETED_PREVIEW_MAX_CHARS = 500
  125. def _tool_completed_preview(result: Any, redact_sensitive_text: Callable[..., str]) -> str:
  126. """Bounded, secret-redacted result summary for the public run stream — redacted BEFORE
  127. truncation so a cut never leaves a secret's prefix on the wire."""
  128. if result is None:
  129. return ""
  130. text = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False, default=str)
  131. preview = redact_sensitive_text(text, force=True)
  132. limit = _TOOL_COMPLETED_PREVIEW_MAX_CHARS
  133. return preview if len(preview) <= limit else preview[: limit - 3] + "..."
  134. _RUN_STREAM_SUBSCRIBER_OVERFLOW = object()
  135. _RUN_STREAM_WRITE_TIMEOUT = 5.0
  136. class _RunStream:
  137. """Sequence and retain one run's events while fanning out to SSE clients."""
  138. BACKLOG_LIMIT = 1000
  139. SUBSCRIBER_QUEUE_LIMIT = 256
  140. def __init__(self) -> None:
  141. self.subscribers: set[asyncio.Queue] = set()
  142. self.backlog: deque[tuple[int, Optional[dict[str, Any]]]] = deque(
  143. maxlen=self.BACKLOG_LIMIT
  144. )
  145. self.next_seq = 0
  146. self.terminal = False
  147. def put_nowait(self, event: Optional[dict[str, Any]]) -> None:
  148. if self.terminal:
  149. return
  150. seq = self.next_seq
  151. self.next_seq += 1
  152. self.backlog.append((seq, event))
  153. if event is None:
  154. self.terminal = True
  155. for queue in list(self.subscribers):
  156. try:
  157. queue.put_nowait((seq, event))
  158. except asyncio.QueueFull:
  159. self.detach(queue)
  160. while not queue.empty():
  161. queue.get_nowait()
  162. queue.put_nowait((seq, _RUN_STREAM_SUBSCRIBER_OVERFLOW))
  163. def attach(
  164. self, last_seq: int = -1
  165. ) -> tuple[asyncio.Queue, list[tuple[int, Optional[dict[str, Any]]]]]:
  166. replay = [(seq, event) for seq, event in self.backlog if seq > last_seq]
  167. # Headroom for events produced while the replay is still being written.
  168. queue: asyncio.Queue = asyncio.Queue(maxsize=self.SUBSCRIBER_QUEUE_LIMIT + len(replay))
  169. self.subscribers.add(queue)
  170. return queue, replay
  171. def detach(self, queue: asyncio.Queue) -> None:
  172. self.subscribers.discard(queue)
  173. def _remember_room_retention(request: web.Request, claims: dict[str, Any]) -> None:
  174. value = float(claims.get("status_expires_at") or claims.get("expires_at") or 0)
  175. try:
  176. request[_ROOM_RETENTION_REQUEST_KEY] = value
  177. except (AttributeError, TypeError):
  178. request._hermes_room_run_retention_until = value
  179. def _room_retention_until(request: web.Request) -> float:
  180. try:
  181. value = request.get(_ROOM_RETENTION_REQUEST_KEY, 0)
  182. except AttributeError:
  183. value = getattr(request, "_hermes_room_run_retention_until", 0)
  184. return max(0.0, float(value or 0))
  185. def _run_event(run_id: str, name: str, **fields: Any) -> dict[str, Any]:
  186. """Build one SSE event payload (key order is part of the wire format)."""
  187. return {"event": name, "run_id": run_id, "timestamp": time.time(), **fields}
  188. def terminal_run_status(result: dict[str, Any]) -> tuple[str, dict[str, Any]]:
  189. """Map a ``run_conversation`` result to its terminal run status and the wire fields every
  190. terminal event/status carries. An interrupted turn is ``cancelled``; a turn that ended
  191. without finishing (``failed``, ``partial``, or ``completed=False`` such as the iteration
  192. budget) is ``failed``, so ``completed: true`` never rides next to ``partial: true``."""
  193. interrupted = bool(result.get("interrupted"))
  194. finished = (
  195. not interrupted and not result.get("failed") and not result.get("partial")
  196. and result.get("completed") is not False
  197. )
  198. status = "cancelled" if interrupted else "completed" if finished else "failed"
  199. fields: dict[str, Any] = {
  200. "completed": finished, "partial": bool(result.get("partial")), "interrupted": interrupted,
  201. }
  202. if not finished and result.get("turn_exit_reason"):
  203. fields["turn_exit_reason"] = str(result["turn_exit_reason"])
  204. if result.get("pending_steer"):
  205. # Undelivered steer text rides on every terminal event/status for client replay.
  206. fields["pending_steer"] = result["pending_steer"]
  207. return status, fields
  208. def _run_not_found(_openai_error, run_id: str) -> web.Response:
  209. return _json_error(_openai_error, f"Run not found: {run_id}", code="run_not_found", status=404)
  210. def _uses_room_run_auth(self, request: web.Request) -> bool:
  211. return request.path.endswith("/v1/runs") and bool(self._room_grant_token(request))
  212. def _initialize_run_state(self, *, store_factory) -> None:
  213. """Initialize adapter-owned durable and live ``/v1/runs`` state."""
  214. self._run_idempotency_store = store_factory()
  215. self._run_owner_pid = os.getpid()
  216. try:
  217. from gateway.status import get_process_start_time
  218. self._run_owner_started = int(get_process_start_time(self._run_owner_pid) or 0)
  219. except Exception:
  220. self._run_owner_started = 0
  221. # All keyed by run_id: SSE fanout streams (+creation time for the TTL sweep), live
  222. # agent/task refs for cooperative stop (the executor thread may outlive the request,
  223. # hence the separate stopping set), pollable statuses, and approval session keys
  224. # (approval core resolves by session key, clients by run_id).
  225. self._run_idempotency_ids: set[str] = set()
  226. self._stopping_run_ids: set[str] = set()
  227. self._shutdown_interrupted_run_ids: set[str] = set()
  228. self._run_shutdown_requested_at: Optional[float] = None
  229. # Streaming run id -> its canonical admission id, held while that request observes the turn.
  230. self._run_admission_aliases: dict[str, str] = {}
  231. (
  232. self._run_owners, self._run_streams, self._run_streams_created, self._active_run_agents,
  233. self._active_run_tasks, self._run_statuses, self._run_approval_sessions,
  234. ) = ({} for _ in range(7))
  235. def _http_routes(self) -> list[tuple[str, str, Any]]:
  236. return [
  237. ("POST", "/v1/runs", self._handle_runs), ("GET", "/v1/runs/{run_id}", self._handle_get_run),
  238. ("GET", "/v1/runs/{run_id}/events", self._handle_run_events),
  239. ("POST", "/v1/runs/{run_id}/approval", self._handle_run_approval),
  240. ("POST", "/v1/runs/{run_id}/clarify", self._handle_run_clarify),
  241. ("POST", "/v1/runs/{run_id}/steer", self._handle_steer_run),
  242. ("POST", "/v1/runs/{run_id}/resolve-unknown", self._handle_resolve_unknown_run),
  243. ("POST", "/v1/runs/{run_id}/stop", self._handle_stop_run)]
  244. def _idempotency_capabilities(self, *, store_type) -> dict[str, Any]:
  245. return {
  246. "supported": True,
  247. "durable": self._run_idempotency_store.durable,
  248. "retention_seconds": store_type.RETENTION_SECONDS}
  249. def _close_run_state(self) -> None:
  250. retire_api_turn_executor()
  251. try:
  252. if getattr(self, "_run_idempotency_store", None) is not None:
  253. self._run_idempotency_store.close()
  254. except Exception:
  255. logger.debug("Failed to close run idempotency store for %s", self.name, exc_info=True)
  256. def _set_run_status(self, run_id: str, status: str, **fields: Any) -> dict[str, Any]:
  257. """Update pollable run status without exposing private agent objects."""
  258. now = time.time()
  259. current = self._run_statuses.get(run_id, {})
  260. previous_status = str(current.get("status") or "")
  261. field_names = set(fields)
  262. current.update({"object": "hermes.run", "run_id": run_id, "status": status, "updated_at": now})
  263. current.setdefault("created_at", fields.pop("created_at", now))
  264. current.update(fields)
  265. shutdown_requested_at = getattr(self, "_run_shutdown_requested_at", None)
  266. if shutdown_requested_at is not None and status not in TERMINAL_STATUSES:
  267. current.setdefault("shutdown_requested_at", shutdown_requested_at)
  268. if status != "waiting_for_approval":
  269. current.pop("approval", None)
  270. self._run_statuses[run_id] = current
  271. should_persist = (
  272. status != previous_status
  273. or status in TERMINAL_STATUSES
  274. or bool(field_names & {
  275. "output", "error", "usage", "pending_steer", "session_id", "shutdown_requested_at"}))
  276. if run_id in self._run_idempotency_ids and should_persist:
  277. try:
  278. self._run_idempotency_store.update_status(run_id, current)
  279. except Exception:
  280. logger.exception("[api_server] failed to persist idempotent run status %s", run_id)
  281. return current
  282. def _mark_shutdown_interrupted_runs(self, run_ids) -> None:
  283. """Publish the shutdown outcome before cooperative interruption can race teardown."""
  284. for run_id in run_ids:
  285. self._shutdown_interrupted_run_ids.add(run_id)
  286. self._set_run_status(
  287. run_id,
  288. "interrupted",
  289. error="Gateway shutdown interrupted the run.",
  290. last_event="run.interrupted",
  291. )
  292. def _mark_shutdown_requested(self) -> int:
  293. """Persist when gateway shutdown began without changing the run state vocabulary."""
  294. shutdown_requested_at = getattr(self, "_run_shutdown_requested_at", None)
  295. if shutdown_requested_at is None:
  296. shutdown_requested_at = time.time()
  297. self._run_shutdown_requested_at = shutdown_requested_at
  298. marked = 0
  299. for run_id, current in list(self._run_statuses.items()):
  300. if current.get("status") in TERMINAL_STATUSES or "shutdown_requested_at" in current:
  301. continue
  302. self._set_run_status(
  303. run_id, str(current.get("status") or "running"),
  304. shutdown_requested_at=shutdown_requested_at)
  305. marked += 1
  306. return marked
  307. def _make_run_event_callback(self, run_id: str, loop: asyncio.AbstractEventLoop, *, _api_server):
  308. """Return a callback that pushes structured events to the run SSE queue."""
  309. redact_sensitive_text = _api_server.redact_sensitive_text
  310. def _push(event: dict[str, Any]) -> None:
  311. self._set_run_status(
  312. run_id, self._run_statuses.get(run_id, {}).get("status", "running"), last_event=event.get("event"))
  313. q = self._run_streams.get(run_id)
  314. if q is not None:
  315. with suppress(Exception):
  316. loop.call_soon_threadsafe(q.put_nowait, event)
  317. def _callback(event_type: str, tool_name: str | None = None, preview: str | None = None, args=None, **kwargs):
  318. # _thinking / subagent.tool / subagent_progress are deliberately dropped (UI noise);
  319. # lifecycle boundaries must land so clients can observe delegate_task failures.
  320. fields = _FIXED_EVENT_FIELDS.get(event_type)
  321. if fields is not None:
  322. event_fields = fields(tool_name, preview, kwargs)
  323. if event_type == "tool.completed":
  324. event_fields["preview"] = _tool_completed_preview(
  325. kwargs.get("result"), redact_sensitive_text)
  326. _push(_run_event(run_id, event_type, **event_fields))
  327. elif event_type in {"subagent.start", "subagent.complete"}:
  328. event = _run_event(run_id, event_type)
  329. if preview is not None:
  330. event["preview"] = redact_sensitive_text(str(preview), force=True)
  331. for key in _SUBAGENT_EVENT_KEYS:
  332. value = kwargs.get(key)
  333. if value is not None:
  334. # Free text may carry child tool output: force secret redaction on this public stream.
  335. redact = key in _SUBAGENT_TEXT_KEYS and isinstance(value, str)
  336. event[key] = redact_sensitive_text(value, force=True) if redact else value
  337. _push(event)
  338. return _callback
  339. # Answering a prompt the run raised is one capability whichever tool asked: approvals
  340. # and clarify share the grant's ``approve`` permission. Acknowledging a lost owner epoch
  341. # ends the run like a stop, so ``/resolve-unknown`` rides the ``stop`` permission.
  342. _ROOM_CONTROL_PERMISSIONS = {
  343. "/stop": "stop", "/resolve-unknown": "stop", "/approval": "approve", "/clarify": "approve"}
  344. def _room_permission_for(request: web.Request) -> str:
  345. for suffix, permission in _ROOM_CONTROL_PERMISSIONS.items():
  346. if request.path.endswith(suffix):
  347. return permission
  348. return "status" if request.method == "GET" else "dispatch"
  349. def _run_idempotency_scope(self, request: web.Request, *, _api_server) -> str:
  350. """Opaque auth/profile namespace; never persist bearer credentials."""
  351. if self._room_grant_token(request):
  352. claims = self._room_grant_claims(request, permission=_room_permission_for(request))
  353. _remember_room_retention(request, claims)
  354. parts = (claims[k] for k in (
  355. "room_id", "home_install_id", "authority_gateway_id", "authority_epoch",
  356. "member_id", "target_install_id", "target_profile"))
  357. else:
  358. parts = (_api_server._api_request_profile.get() or "default",
  359. self._expected_api_key() or "unauthenticated-test-listener")
  360. return hashlib.sha256("\0".join(map(str, parts)).encode()).hexdigest()
  361. def _check_run_auth(self, request: web.Request, *, permission: str, _api_server) -> web.Response | None:
  362. if not self._room_grant_token(request):
  363. return self._check_auth(request)
  364. try:
  365. self._room_grant_claims(request, permission=permission)
  366. except Exception as exc:
  367. return _room_grant_error_response(exc, _openai_error=_api_server._openai_error)
  368. return None
  369. def _owner_alive(owner_pid: int, owner_started: int) -> bool:
  370. """True when the recorded owner pid still exists and is the same process incarnation."""
  371. try:
  372. from gateway.status import _pid_exists, get_process_start_time, start_time_fingerprints_match
  373. return owner_pid > 0 and bool(_pid_exists(owner_pid)) and (
  374. not owner_started
  375. or start_time_fingerprints_match(owner_started, get_process_start_time(owner_pid) or 0))
  376. except Exception:
  377. return False
  378. def _durable_run_status(self, request: web.Request, run_id: str) -> dict[str, Any] | None:
  379. """Hydrate a scoped run status and fail stale owners closed."""
  380. from gateway.platforms.api_server_authority_runs import run_projection
  381. canonical = run_projection(self, run_id)
  382. if canonical is not None:
  383. # The documented poll shape: the canonical projection decides status/output/usage; the
  384. # retained run record still names the object, the requested model and the served runtime.
  385. retained = self._run_statuses.get(run_id) or {}
  386. return {"object": "hermes.run", **{key: retained[key] for key in ("model", "runtime", "created_at")
  387. if key in retained}, **canonical}
  388. status = self._run_statuses.get(run_id)
  389. if status is not None:
  390. if run_id in self._run_idempotency_ids:
  391. scope = self._run_idempotency_scope(request)
  392. self._run_idempotency_store.extend_retention(scope, run_id, _room_retention_until(request))
  393. return status
  394. scope = self._run_idempotency_scope(request)
  395. record = self._run_idempotency_store.status_for_run(
  396. scope, run_id, retention_until=_room_retention_until(request))
  397. if record is None:
  398. return None
  399. status = dict(record["status"])
  400. if status.get("status") not in TERMINAL_STATUSES and not _owner_alive(
  401. int(record.get("owner_pid") or 0), int(record.get("owner_started") or 0)):
  402. status.update(
  403. status="interrupted", error="The gateway restarted before this run settled.",
  404. last_event="run.interrupted", updated_at=time.time())
  405. self._run_idempotency_store.update_status(run_id, status)
  406. self._run_statuses[run_id] = status
  407. self._run_idempotency_ids.add(run_id)
  408. self._run_owners[run_id] = scope
  409. return status
  410. def _resolve_conversation_history(
  411. self, body: dict, raw_input: Any, *, _openai_error
  412. ) -> tuple[list[dict[str, str]], Any, Any, web.Response | None]:
  413. """Return ``(history, instructions, stored_session_id, error)``; precedence:
  414. ``conversation_history`` > ``previous_response_id`` chain > all-but-last ``input`` messages."""
  415. instructions = body.get("instructions")
  416. previous_response_id = body.get("previous_response_id")
  417. conversation_history: list[dict[str, str]] = []
  418. raw_history = body.get("conversation_history")
  419. if raw_history:
  420. if not isinstance(raw_history, list):
  421. return [], instructions, None, _json_error(
  422. _openai_error, "'conversation_history' must be an array of message objects", status=400)
  423. for i, entry in enumerate(raw_history):
  424. if not isinstance(entry, dict) or {"role", "content"} - set(entry):
  425. return [], instructions, None, _json_error(
  426. _openai_error, f"conversation_history[{i}] must have 'role' and 'content' fields",
  427. status=400)
  428. conversation_history.append({"role": str(entry["role"]), "content": str(entry["content"])})
  429. if previous_response_id:
  430. logger.debug("Both conversation_history and previous_response_id provided; using conversation_history")
  431. stored_session_id = None
  432. if not conversation_history and previous_response_id:
  433. stored = self._current_response_store().get(previous_response_id)
  434. if stored:
  435. conversation_history = list(stored.get("conversation_history", []))
  436. stored_session_id = stored.get("session_id")
  437. if instructions is None:
  438. instructions = stored.get("instructions")
  439. if not conversation_history and isinstance(raw_input, list) and len(raw_input) > 1:
  440. for msg in raw_input[:-1]:
  441. if isinstance(msg, dict) and msg.get("role") and msg.get("content"):
  442. content = msg["content"]
  443. if isinstance(content, list): # flatten multi-part content blocks to text
  444. content = " ".join(p.get("text", "") for p in content
  445. if isinstance(p, dict) and p.get("type") == "text")
  446. conversation_history.append({"role": msg["role"], "content": str(content)})
  447. return conversation_history, instructions, stored_session_id, None
  448. async def _resolve_chained_history(self, body: dict, raw_input: Any, *, _openai_error):
  449. """``_resolve_conversation_history`` with the chain's live session: a deleted chain moves to
  450. its next generation (an explicit body ``session_id`` still wins in the caller), and a store
  451. that cannot answer is the retryable store verdict (``continued_session``). The last item is the
  452. stored session the chain moved past (``walked_from``), else None."""
  453. from gateway.platforms.api_server_authority_runs import continued_session
  454. # Off the loop: a previous_response_id chain reads (and json-decodes) the stored history.
  455. history, instructions, chain_root, error = await asyncio.to_thread(
  456. _resolve_conversation_history, self, body, raw_input, _openai_error=_openai_error)
  457. stored_session_id = chain_root
  458. if error is None:
  459. stored_session_id, error = await continued_session(self, chain_root)
  460. return history, instructions, stored_session_id, error, (
  461. chain_root if stored_session_id and stored_session_id != chain_root else None)
  462. def _accepted_response(run_id: str, status: str, gateway_session_key, *, replayed: bool) -> web.Response:
  463. """202 admission response; replays are flagged via ``Idempotency-Replayed``."""
  464. headers = {"Idempotency-Replayed": "true"} if replayed else {}
  465. if gateway_session_key:
  466. headers["X-Hermes-Session-Key"] = gateway_session_key
  467. return web.json_response(
  468. {"run_id": run_id, "status": status, "replayed": replayed}, status=202, headers=headers)
  469. def _replay_or_conflict(self, request, outcome, record, gateway_session_key, _openai_error) -> web.Response:
  470. """409 for a fingerprint conflict, else a 202 replay of the already-admitted run."""
  471. if outcome == "conflict":
  472. return _json_error(
  473. _openai_error, "Idempotency-Key was already used with a different request payload",
  474. code="idempotency_key_conflict", status=409)
  475. original_id = str(record["run_id"])
  476. status = self._durable_run_status(request, original_id) or record["status"]
  477. return _accepted_response(original_id, status.get("status", "queued"), gateway_session_key, replayed=True)
  478. @dataclass(slots=True)
  479. class _RunLaunch:
  480. """State for an admitted run's background task; contextvars are captured here
  481. because the task outlives the request (and its middleware profile scope)."""
  482. owner: Any
  483. run_id: str
  484. queue: _RunStream
  485. session_id: str
  486. gateway_session_key: Optional[str]
  487. declared_selected: bool
  488. user_message: str
  489. conversation_history: list[dict[str, str]]
  490. # #98619: only continuation paths that reload session history may grant wake authority —
  491. # a previous_response_id continuation consumes its ResponseStore snapshot instead, and a
  492. # caller-supplied conversation_history is authoritative for the turn; neither consumes a
  493. # SessionDB delivery row, so both stay default-denied.
  494. session_history_delivery: bool
  495. agent_kwargs: dict # ``_create_agent`` keyword arguments (prompt, model overrides, route, room policy)
  496. request_profile: Any
  497. turn_author: Optional[dict[str, Any]] = None # memory-attribution label only; grants nothing
  498. admission: Any = None
  499. walked_from: Optional[str] = None # the swept stored session the chain moved past (``bind_api_steps``)
  500. @property
  501. def approval_session_key(self) -> str:
  502. # Isolated per run: session ids are conversation scopes, not authorization namespaces.
  503. return self.run_id
  504. def put_event(self, event: Optional[dict]) -> None:
  505. """Enqueue only while this run still owns live transport state."""
  506. if self.owner._run_streams.get(self.run_id) is self.queue:
  507. self.queue.put_nowait(event)
  508. def _forget_run(self, run_id: str, *tables) -> None:
  509. """Drop *run_id* from the given run-keyed dicts/sets, then release its owner stamp."""
  510. for table in tables:
  511. (table.discard if isinstance(table, set) else lambda k: table.pop(k, None))(run_id)
  512. self._release_run_owner_if_forgotten(run_id)
  513. def _retire_live_run(self, run_id: str) -> None:
  514. """Retire agent/task/approval control state once the executor-backed task is done."""
  515. _forget_run(self, run_id, self._active_run_agents, self._active_run_tasks, self._run_approval_sessions,
  516. self._stopping_run_ids, self._shutdown_interrupted_run_ids)
  517. def _drop_run_transport(self, run_id: str) -> None:
  518. _forget_run(
  519. self,
  520. run_id,
  521. self._run_streams,
  522. self._run_streams_created,
  523. )
  524. async def _resolve_live_session_id(self, session_id: str) -> str:
  525. """Adopt the live compression-continuation tip for a client-addressed session (#98619):
  526. a /v1/runs run bound to a pre-rotation id would otherwise load a stale history slice and
  527. write its turn into the closed parent (CompressionSessionClosedError). Same canonical
  528. resolution ``/api/sessions/{id}/messages`` reads use; fails open to the original id."""
  529. db = await self._ensure_session_db_async()
  530. resolver = getattr(db, "resolve_resume_session_id", None) if db is not None else None
  531. if not callable(resolver):
  532. return session_id
  533. try:
  534. resolved = await asyncio.to_thread(resolver, session_id)
  535. return str(resolved) if resolved else session_id
  536. except Exception:
  537. logger.debug("/v1/runs live-session resolve failed for %s", session_id, exc_info=True)
  538. return session_id
  539. async def run_internal_session_turn(self, *, session_id: str, text: str, profile: str,
  540. notification_category: str = "result", _api_server) -> None:
  541. """Run one background wake turn against a raw session id IN-PROCESS (no HTTP, no API key).
  542. The HTTP wake self-post cannot serve a multiplexed *served* profile: ``/p/<profile>/`` on
  543. the shared listener authenticates with that profile's own ``API_SERVER_KEY`` — which a
  544. route-only profile legitimately does not have — while an unprefixed self-post would resume
  545. the session in the DEFAULT profile's store. ``gateway.wake`` therefore runs the turn here,
  546. inside the owner profile's runtime scope (the session DB, model resolution and tool policy
  547. all follow the ambient scope), with ``profile`` naming the profile the caller proved owns
  548. the session — never derived here, so a missing proof cannot silently become the default.
  549. Raises on failure so the caller can rewind its cursor: a draining gateway fails at once
  550. (the HTTP self-post's 503) while a saturated concurrent-run cap is retried with the same
  551. backoff the HTTP self-post uses for a 429.
  552. """
  553. from gateway.wake import _RETRY_DELAYS_SECONDS
  554. profile = (profile or "").strip()
  555. if not profile:
  556. raise ValueError("run_internal_session_turn requires the owning profile")
  557. token = _api_server._api_request_profile.set(profile)
  558. attempts = 1 + len(_RETRY_DELAYS_SECONDS)
  559. last_err: Optional[BaseException] = None
  560. try:
  561. for attempt in range(attempts):
  562. if attempt:
  563. await asyncio.sleep(_RETRY_DELAYS_SECONDS[attempt - 1])
  564. if self._draining_response() is not None:
  565. raise RuntimeError(
  566. f"internal wake refused for session {session_id}: the gateway is draining")
  567. # Transient: the cap clears on its own, exactly as the HTTP self-post's 429 does.
  568. if self._concurrency_limited_response() is not None:
  569. last_err = RuntimeError(
  570. "internal wake deferred: the API server is at its concurrent-run cap")
  571. logger.warning("%s; attempt %d/%d", last_err, attempt + 1, attempts)
  572. continue
  573. # #98619/#13437: adopt the live continuation tip first, the same canonical
  574. # resolution the HTTP self-post consumes — a compressed origin must be woken on the
  575. # transcript that is actually live, never the retired parent slice.
  576. resolved = await _resolve_live_session_id(self, session_id)
  577. session, err = await self._get_existing_session_or_404(resolved)
  578. if err is not None or not session:
  579. raise RuntimeError(
  580. f"internal wake target session {resolved!r} is not in the active profile store")
  581. history = await self._conversation_history_for_session(resolved)
  582. # Same route resolution as the HTTP self-post (/v1/chat/completions): a model_routes
  583. # alias for the virtual model applies to the wake turn too.
  584. route, overrides, err = self._select_request_route(
  585. {"model": self._model_name}, session_id=resolved, gateway_session_key=None,
  586. model_alias=self._model_name)
  587. if err is not None:
  588. raise RuntimeError(f"internal wake route conflict for session {resolved!r}")
  589. await self._run_agent(
  590. user_message=text, conversation_history=history, session_id=resolved,
  591. gateway_session_key=None, **overrides, route=route, requested_runtime={},
  592. route_source="global", session_history_delivery="1",
  593. notification_category=notification_category,
  594. )
  595. return
  596. raise RuntimeError(
  597. f"internal wake gave up for session {session_id} after {attempts} attempts: {last_err}")
  598. finally:
  599. if token is not None:
  600. _api_server._api_request_profile.reset(token)
  601. def _run_idempotency_identity(self, request, body, gateway_session_key, idempotency_key) -> tuple:
  602. """``(principal scope, request fingerprint)`` for a keyed POST /v1/runs; ``("", "")`` unkeyed."""
  603. if not idempotency_key:
  604. return "", ""
  605. return self._run_idempotency_scope(request), hashlib.sha256(json.dumps(
  606. {"body": body, "gateway_session_key": gateway_session_key or ""},
  607. sort_keys=True, separators=(",", ":"), ensure_ascii=False,
  608. ).encode()).hexdigest()
  609. def _run_user_message(raw_input, _openai_error) -> tuple:
  610. """POST /v1/runs ``input`` -> ``(user_message, error_response_or_None)``."""
  611. if not raw_input:
  612. return None, _json_error(_openai_error, "Missing 'input' field", status=400)
  613. if isinstance(raw_input, str):
  614. user_message = raw_input
  615. else:
  616. user_message = raw_input[-1].get("content", "") if isinstance(raw_input, list) else ""
  617. if not user_message:
  618. return None, _json_error(_openai_error, "No user message found in input", status=400)
  619. return user_message, None
  620. async def _admit_run_to_authority(
  621. self, launch, run_id, session_history_delivery, _declared_selected,
  622. idempotency_scope, idempotency_key, _openai_error,
  623. ) -> Optional[web.Response]:
  624. """Admit a /v1/runs turn to the session authority FIFO -> ``None``, or the refusal."""
  625. from gateway.platforms.api_server_authority_runs import run_admission, store_error_response
  626. from gateway.session_api_turn import admit_api_turn_async
  627. from gateway.session_contract import SessionRef
  628. from hermes_state_runtime import RuntimeStoreError
  629. try:
  630. with self._profile_scope(launch.request_profile):
  631. launch.admission = await admit_api_turn_async(self, user_message=launch.user_message,
  632. conversation_history=launch.conversation_history, active_run_id=run_id,
  633. run_owner_scope=self._run_owners[run_id],
  634. turn_author=launch.turn_author,
  635. history_from_session=session_history_delivery,
  636. session_history_delivery='1' if session_history_delivery else '',
  637. bind_declared_conversation=_declared_selected, walked_from=launch.walked_from,
  638. **launch.agent_kwargs)
  639. except Exception as exc: # health: allow BLE001 -- answered below; a committed admission is kept
  640. # A storage failure after the admission committed (a later media-release write) still
  641. # owns that run: forgetting it would let a keyed retry admit the same work twice.
  642. owned = None if isinstance(exc, RuntimeStoreError) else await asyncio.to_thread(run_admission, self, run_id)
  643. if owned is not None:
  644. launch.admission = (owned[0], SessionRef(owned[0].profile_id, owned[1]['target_session_id']), owned[1])
  645. return None
  646. # A refused admission owns no run: drop every reservation so an exact
  647. # retry is refused again instead of replaying a run nobody executes.
  648. _forget_run(
  649. self, run_id, self._run_streams, self._run_streams_created, self._run_approval_sessions,
  650. self._run_statuses, self._run_owners, self._run_idempotency_ids)
  651. if idempotency_key:
  652. self._run_idempotency_store.forget(idempotency_scope, idempotency_key)
  653. return store_error_response(_openai_error, exc)
  654. return None
  655. async def _handle_runs(self, request: web.Request, *, _api_server) -> web.Response:
  656. """POST /v1/runs — start an agent run, return run_id immediately."""
  657. _openai_error = _api_server._openai_error
  658. # Long-term memory scope header (see chat_completions for details).
  659. gateway_session_key, key_err = self._parse_session_key_header(request)
  660. if key_err is not None:
  661. return key_err
  662. try:
  663. body = await request.json()
  664. except Exception:
  665. return _json_error(_openai_error, "Invalid JSON", status=400)
  666. body, room_error = await self._normalize_room_dispatch(request, body)
  667. if room_error is not None:
  668. return room_error
  669. room_dispatch, room_execution_policy = (
  670. v if isinstance(v, dict) else None for v in (
  671. (body.get("hosted_room_dispatch"), body.get("_room_execution_policy"))
  672. if isinstance(body, dict) else (None, None)))
  673. idempotency_key = request.headers.get("Idempotency-Key", "").strip()
  674. if len(idempotency_key) > 255 or any(ord(ch) < 33 or ord(ch) > 126 for ch in idempotency_key):
  675. return _json_error(
  676. _openai_error, "Idempotency-Key must be 1-255 visible ASCII characters",
  677. code="invalid_idempotency_key", status=400)
  678. idempotency_scope, idempotency_fingerprint = _run_idempotency_identity(
  679. self, request, body, gateway_session_key, idempotency_key)
  680. raw_input = body.get("input")
  681. user_message, input_err = _run_user_message(raw_input, _openai_error)
  682. if input_err is not None:
  683. return input_err
  684. try:
  685. turn_author = _api_server._request_turn_author(body)
  686. except ValueError as exc:
  687. return _json_error(_openai_error, str(exc), code="invalid_author", status=400)
  688. conversation_history, instructions, stored_session_id, history_err, walked_from = await _resolve_chained_history(
  689. self, body, raw_input, _openai_error=_openai_error)
  690. if history_err is not None:
  691. return history_err
  692. previous_response_id = body.get("previous_response_id")
  693. session_id = body.get("session_id") or stored_session_id
  694. route = self._resolve_route(body.get("model"))
  695. agent_overrides = _api_server._request_agent_overrides(body, virtual_model=self._model_name)
  696. selection_error = self._request_route_conflict_error(
  697. session_id=session_id, gateway_session_key=gateway_session_key,
  698. requested_model=agent_overrides.get("requested_model"),
  699. requested_provider=agent_overrides.get("requested_provider"), route=route)
  700. if selection_error:
  701. return _json_error(_openai_error, selection_error, status=400)
  702. # A lost-acceptance replay must resolve even while the original run holds the last
  703. # concurrency slot; this read reserves nothing (the atomic reserve below closes the race).
  704. if idempotency_key:
  705. outcome, record = self._run_idempotency_store.lookup(
  706. idempotency_scope, idempotency_key, idempotency_fingerprint,
  707. retention_until=_room_retention_until(request))
  708. if outcome == "conflict" or (outcome == "reused" and record is not None):
  709. return _replay_or_conflict(self, request, outcome, record, gateway_session_key, _openai_error)
  710. # Enforce concurrency only for a genuinely new run.
  711. limited = self._concurrency_limited_response()
  712. if limited is not None:
  713. return limited
  714. run_id = f"run_{uuid.uuid4().hex}"
  715. self._run_owners[run_id] = self._run_idempotency_scope(request)
  716. # Same precedence as /v1/responses: body session_id > response chain > X-Hermes-Session-Key
  717. # conversation > run_id (which would otherwise re-key every affinity surface per run).
  718. # An explicit or chained session owns its routing key and is never rebound to the header.
  719. _declared_selected = not session_id and bool(gateway_session_key)
  720. selected_session_id = session_id or (
  721. await asyncio.to_thread(self._declared_conversation_session, gateway_session_key)
  722. if _declared_selected else None)
  723. # A client-addressed id from before a compression rotation must adopt the live tip (#98619):
  724. # history loads from it, the turn writes to it, and a detached delivery row persisted to it
  725. # is what the next same-id run consumes below.
  726. if selected_session_id:
  727. selected_session_id = await _resolve_live_session_id(self, str(selected_session_id))
  728. session_id = selected_session_id or run_id
  729. # History loads for the session the request actually selected — including one resolved from
  730. # a declared X-Hermes-Session-Key, whose persisted delivery rows must reach the next
  731. # same-key run's context (#98619). previous_response_id continuations keep their
  732. # ResponseStore snapshot as history (they cannot consume a SessionDB delivery row and are
  733. # accordingly denied wake capability in _run_agent_sync); the fresh run_id fallback has
  734. # nothing persisted to load yet. Wake authority is fixed here, before the load can
  735. # overwrite ``conversation_history``: a caller-supplied history is authoritative for this
  736. # turn, never consumes the SessionDB delivery row, and is denied on the same contract.
  737. session_history_delivery = not previous_response_id and not conversation_history
  738. if not conversation_history and selected_session_id and not previous_response_id:
  739. conversation_history = await self._conversation_history_for_session(str(selected_session_id))
  740. q = self._run_streams[run_id] = _RunStream()
  741. created_at = self._run_streams_created[run_id] = time.time()
  742. self._run_approval_sessions[run_id] = run_id # approval session key (see _RunLaunch)
  743. initial_status = self._set_run_status(
  744. run_id, "queued", created_at=created_at, session_id=session_id, model=body.get("model", self._model_name))
  745. if idempotency_key:
  746. outcome, record = self._run_idempotency_store.reserve(
  747. idempotency_scope, idempotency_key, idempotency_fingerprint, run_id, initial_status,
  748. owner_pid=self._run_owner_pid, owner_started=self._run_owner_started,
  749. retention_until=_room_retention_until(request))
  750. if outcome != "created":
  751. _forget_run(
  752. self, run_id, self._run_streams, self._run_streams_created, self._run_approval_sessions,
  753. self._run_statuses, self._run_owners)
  754. return _replay_or_conflict(self, request, outcome, record, gateway_session_key, _openai_error)
  755. self._run_idempotency_ids.add(run_id)
  756. launch = _RunLaunch(
  757. self, run_id, q, session_id, gateway_session_key, _declared_selected, user_message,
  758. conversation_history, session_history_delivery,
  759. agent_kwargs=dict(
  760. ephemeral_system_prompt=instructions, session_id=session_id, gateway_session_key=gateway_session_key,
  761. route=route, room_dispatch=room_dispatch, room_execution_policy=room_execution_policy,
  762. **{k: agent_overrides.get(k) for k in ("requested_model", "requested_provider", "model_options")}),
  763. request_profile=_api_server._api_request_profile.get(),
  764. turn_author=turn_author, walked_from=walked_from if session_id == stored_session_id else None)
  765. # A canonical Bot Chat that a Desktop holds live is that Desktop's to run: executing here would
  766. # be a second writer beside its lease (#114959). The owner's mailbox takes the turn and its
  767. # receipt drives this run's status, so `peer run` keeps its run_id and `peer status` still works.
  768. # Under session authority the same call admits the turn to the Bot Chat's own FIFO, which is
  769. # why it is decided BEFORE ``admit_api_turn`` would bind the chat as an API conversation.
  770. admitted = await self._admit_to_live_bot_chat(session_id, user_message, turn_author) if selected_session_id else None
  771. _retarget_to_delivered_chat(self, launch, admitted)
  772. if admitted is None and getattr(self.gateway_runner, 'session_authority', None) is not None:
  773. refused = await _admit_run_to_authority(
  774. self, launch, run_id, session_history_delivery, _declared_selected,
  775. idempotency_scope, idempotency_key, _openai_error)
  776. if refused is not None:
  777. return refused
  778. self._activate_admitted_request()
  779. if admitted is not None:
  780. task = self._active_run_tasks[run_id] = asyncio.create_task(
  781. _execute_run_via_live_owner(self, launch, *admitted, _api_server=_api_server))
  782. else:
  783. task = self._active_run_tasks[run_id] = asyncio.create_task(_execute_run(self, launch, _api_server=_api_server))
  784. with suppress(TypeError):
  785. self._background_tasks.add(task) # tracked for shutdown drain
  786. if hasattr(task, "add_done_callback"):
  787. task.add_done_callback(self._background_tasks.discard)
  788. return _accepted_response(run_id, "started", gateway_session_key, replayed=False)
  789. def _retarget_to_delivered_chat(self, launch: _RunLaunch, admitted) -> None:
  790. """The owner moved an API-held "Bot Chat" title aside and the turn landed in the local chat it
  791. minted (MSG-21): the run record (and its idempotency replay) names that chat, not the old row."""
  792. delivered = admitted[1].get("session_id") if admitted is not None else None
  793. if delivered and delivered != launch.session_id:
  794. launch.session_id = delivered
  795. self._set_run_status(launch.run_id, "queued", session_id=delivered)
  796. def _run_usage(agent) -> dict[str, int]:
  797. """Terminal ``usage`` payload from the agent's session counters; a missing or non-numeric
  798. counter (test doubles, agents without cache accounting) reads as ``0``."""
  799. usage = {}
  800. for key, attr in _USAGE_FIELDS:
  801. value = getattr(agent, attr, 0)
  802. usage[key] = int(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else 0
  803. return usage
  804. def _served_runtime(agent) -> dict[str, str]:
  805. """The ``{provider, model}`` pair that actually served the turn. After a ``fallback_providers``
  806. switch the agent keeps the fallback runtime until the NEXT turn restores the primary, so when
  807. ``run_conversation()`` returns these attributes name the served pair — the run record's
  808. ``model`` field only echoes the request (#102101). Non-string attributes read as ``""``."""
  809. pair = {}
  810. for key in ("provider", "model"):
  811. value = getattr(agent, key, "")
  812. pair[key] = value if isinstance(value, str) else ""
  813. return pair
  814. def _run_agent_sync(self, run: _RunLaunch, agent, approval_notify, *, _api_server):
  815. """Executor-thread body of one run; returns ``(result, usage, served_runtime)``."""
  816. from gateway.session_context import clear_session_vars
  817. from gateway.hosted_room_execution_policy import (
  818. RoomExecutionPolicy, bind_room_execution_policy, reset_room_execution_policy)
  819. # No eager slash-worker pre-warm: slash.exec spawns one on demand (its error path already relies on that
  820. # respawn to recover from a dead worker). Each worker child runs its own MCP discovery (#61891), so
  821. # pre-warming one per session forks the full stdio MCP fleet — ~20 OS processes per retained session on
  822. # a config with a few stdio servers — even for sessions that never run a worker-routed command. Sessions
  823. # held by a live transport are never reaped, so with the desktop app open for days those fleets
  824. # accumulate until the OS refuses new process spawns.
  825. from tools.approval import register_gateway_notify, unregister_gateway_notify
  826. from tools.approval_context import reset_current_session_key, set_current_session_key
  827. session_id = run.session_id
  828. effective_task_id = session_id or run.run_id
  829. # (token, reset) pairs unwound in the finally block; bound only once each step succeeds.
  830. resets: list[tuple[Any, Callable]] = []
  831. with self._profile_scope(run.request_profile):
  832. try:
  833. # Contextvars, not process env: concurrent runs must not share identity.
  834. resets.append((set_current_session_key(run.approval_session_key), reset_current_session_key))
  835. # chat_id carries the raw session id like _run_agent() does; without it
  836. # tools.async_delegation sees no HERMES_SESSION_CHAT_ID and forces delegations sync.
  837. session_tokens = self._bind_api_server_session(
  838. chat_id=session_id or "", session_key=run.approval_session_key, session_id=session_id or "",
  839. profile=run.request_profile or "",
  840. # #98619 audited opt-in: the /v1/runs session id is wake-capable only when its
  841. # own continuation path reloads session history — an explicit body/chained
  842. # session id or a declared X-Hermes-Session-Key conversation (both load
  843. # SessionDB in _handle_runs), or the run_id fallback the client can post back
  844. # as body.session_id. A previous_response_id continuation consumes its
  845. # ResponseStore snapshot instead and can never see a SessionDB delivery row,
  846. # so it stays default-denied until a merge contract exists for that chain;
  847. # likewise a caller-supplied conversation_history is authoritative for the
  848. # turn and never reads the delivery row, so it is denied the same way.
  849. session_history_delivery="1" if run.session_history_delivery else "")
  850. if session_tokens:
  851. resets.append((session_tokens, clear_session_vars))
  852. if run.agent_kwargs["room_dispatch"] is not None:
  853. policy = RoomExecutionPolicy.from_mapping(run.agent_kwargs["room_execution_policy"] or {})
  854. resets.append((bind_room_execution_policy(policy), reset_room_execution_policy))
  855. register_gateway_notify(run.approval_session_key, approval_notify)
  856. # /v1/runs owns its agent lifecycle (no TurnRunner): record process ownership
  857. # so stop/cancel reaps only the background processes this run created.
  858. _api_server._publish_turn_process_ownership(agent, effective_task_id)
  859. # Passed only when set: a human turn keeps today's call shape.
  860. author_kwargs = {"turn_author": run.turn_author} if run.turn_author is not None else {}
  861. r = agent.run_conversation(
  862. user_message=run.user_message, conversation_history=run.conversation_history,
  863. task_id=effective_task_id, **author_kwargs)
  864. finally:
  865. # Clear ownership now so a later stop can't reap work this run left running.
  866. _api_server._clear_turn_process_ownership(agent)
  867. self._memory_sessions.checkin(agent)
  868. # Declared-conversation binding, same precedence gate as _run_agent.
  869. if run.declared_selected:
  870. self._bind_declared_conversation(
  871. getattr(agent, "session_id", None) or session_id, run.gateway_session_key)
  872. try:
  873. unregister_gateway_notify(run.approval_session_key)
  874. finally:
  875. for token, reset in resets:
  876. with suppress(Exception):
  877. reset(token)
  878. return r, _run_usage(agent), _served_runtime(agent)
  879. def _make_approval_notify(self, run: _RunLaunch, *, _api_server) -> Callable[[dict[str, Any]], None]:
  880. """Approval-request bridge: redact, stamp the event envelope, park the run status, enqueue."""
  881. run_id, q, loop = run.run_id, run.queue, asyncio.get_running_loop()
  882. def _approval_notify(approval_data: dict[str, Any]) -> None:
  883. # Clients must never receive the raw flagged command (#48456): the shared builder redacts.
  884. event = _api_server._approval_request_event(run_id, approval_data)
  885. self._set_run_status(run_id, "waiting_for_approval", last_event="approval.request", approval=event)
  886. with suppress(Exception):
  887. loop.call_soon_threadsafe(q.put_nowait, event)
  888. return _approval_notify
  889. async def _execute_run_via_live_owner(self, run: _RunLaunch, home, record: dict[str, Any], *, _api_server) -> None:
  890. """Drive a run whose turn a live Bot Chat owner is executing, from that owner's mailbox receipt.
  891. The receipt is the only truth about the turn: ``settled`` completes the run with the owner's
  892. reply, a failed receipt fails it with the owner's classified reason. ``/stop`` cannot reach the
  893. owner's turn — the mailbox has no recall once a record is claimed — so a stop ends this run
  894. as ``cancelled`` while the chat finishes on its own; the stop handler already reports that a
  895. run without an in-process agent is not interruptible here.
  896. """
  897. from gateway.platforms.api_server_bot_chat import await_live_delivery
  898. run_id = run.run_id
  899. delivery_id = record["delivery_id"]
  900. def _finish(status: str, **fields: Any) -> None:
  901. if run_id in self._shutdown_interrupted_run_ids:
  902. # Shutdown already published "interrupted"; the cancel that follows must not
  903. # rewrite it as a plain "cancelled" (same guard as the native _execute_run).
  904. status, fields = "interrupted", {"error": "Gateway shutdown interrupted the run."}
  905. self._set_run_status(run_id, status, **fields, last_event=f"run.{status}")
  906. with suppress(Exception):
  907. run.put_event(_run_event(run_id, f"run.{status}", **fields))
  908. try:
  909. self._set_run_status(run_id, "running", delivery_id=delivery_id)
  910. record = await await_live_delivery(
  911. self, home, record, None, should_stop=lambda: run_id in self._stopping_run_ids)
  912. if record["status"] in ("queued", "claimed"):
  913. _finish("cancelled", completed=False, partial=False, interrupted=True)
  914. return
  915. if record["status"] == "settled":
  916. _finish("completed", completed=True, partial=False, interrupted=False,
  917. output=record.get("reply") or "", usage={})
  918. elif record["status"] == "cancelled":
  919. _finish("cancelled", completed=False, partial=False, interrupted=True)
  920. else:
  921. _finish("failed", completed=False, partial=False, interrupted=False,
  922. error=record.get("error") or f"Bot Chat delivery {record['status']}",
  923. **({"reason": record["reason"]} if record.get("reason") else {}))
  924. except asyncio.CancelledError:
  925. _finish("cancelled", completed=False, partial=False, interrupted=True)
  926. raise
  927. except Exception as exc:
  928. logger.exception("[api_server] run %s (live Bot Chat) failed", run_id)
  929. _finish("failed", completed=False, partial=False, interrupted=False, error=str(exc))
  930. finally:
  931. with suppress(Exception):
  932. run.put_event(None) # sentinel: close the SSE stream
  933. _retire_live_run(self, run_id)
  934. async def _execute_run(self, run: _RunLaunch, *, _api_server) -> None:
  935. """Drive one admitted run, publish its terminal event/status, release live state."""
  936. _redact_api_error_text = _api_server._redact_api_error_text
  937. run_id, loop = run.run_id, asyncio.get_running_loop()
  938. _run_started_at = time.perf_counter()
  939. def _text_cb(delta: Optional[str]) -> None:
  940. if delta is None or run_id not in self._run_streams:
  941. return
  942. with suppress(Exception):
  943. loop.call_soon_threadsafe(run.put_event, _run_event(run_id, "message.delta", delta=delta))
  944. def _interim_cb(text: str, *, already_streamed: bool = False) -> None:
  945. # Mid-turn assistant commentary (Codex ``phase="commentary"``, text beside tool calls),
  946. # same ``message.interim`` contract as the TUI gateway; reasoning never reaches this
  947. # callback and the final answer still arrives via ``run.completed`` (#67580).
  948. if not isinstance(text, str) or not text.strip() or run_id not in self._run_streams:
  949. return
  950. with suppress(Exception):
  951. loop.call_soon_threadsafe(run.put_event, _run_event(
  952. run_id, "message.interim", text=text, already_streamed=bool(already_streamed)))
  953. def _finish(status: str, extra: Optional[dict] = None, **fields: Any) -> None:
  954. """Terminal status, then best-effort ``run.<status>`` event; key order is wire shape."""
  955. extra = extra or {}
  956. if run_id in self._shutdown_interrupted_run_ids:
  957. status = "interrupted"
  958. fields = {"error": "Gateway shutdown interrupted the run."}
  959. extra = {}
  960. self._set_run_status(run_id, status, **fields, last_event=f"run.{status}", **extra)
  961. with suppress(Exception):
  962. # A worker can finish before its Future is wrapped, so awaiting it
  963. # need not yield to already-scheduled commentary/tool callbacks.
  964. loop.call_soon(run.put_event, _run_event(run_id, f"run.{status}", **fields, **extra))
  965. try:
  966. # Shutdown landed between admission and the task's first tick: nothing to
  967. # interrupt yet, and starting a turn now would outlive the gateway.
  968. if run_id in self._shutdown_interrupted_run_ids:
  969. _finish("interrupted")
  970. return
  971. self._set_run_status(run_id, "running")
  972. if run_id in self._stopping_run_ids:
  973. _finish("cancelled")
  974. return
  975. if run.admission is not None:
  976. from gateway.session_api_turn import observe_api_controls, observe_api_turn
  977. tool_cb = self._make_run_event_callback(run_id, loop)
  978. def _control_cb(event_type, prompt):
  979. # Same authority prompt the WS viewer and GET pending_controls expose, on the
  980. # run's own stream so a streaming-only client learns it must respond.
  981. with suppress(Exception):
  982. loop.call_soon_threadsafe(run.put_event, _run_event(run_id, event_type, **prompt))
  983. def _tool_start(call_id, name, args):
  984. from agent.display import build_tool_preview
  985. tool_cb("tool.started", name, build_tool_preview(name, args or {}) or "", args, tool_call_id=call_id)
  986. def _tool_complete(call_id, name, args, result):
  987. from agent.display import _detect_tool_failure
  988. is_error, _ = _detect_tool_failure(name, result)
  989. tool_cb("tool.completed", name, None, args, tool_call_id=call_id, is_error=bool(is_error))
  990. with observe_api_controls(run.admission, _control_cb):
  991. result, usage = await observe_api_turn(
  992. run.admission, stream_delta_callback=_text_cb,
  993. tool_start_callback=_tool_start, tool_complete_callback=_tool_complete)
  994. # The canonical turn has no agent object here; the served pair rides on the
  995. # committed usage/result runtime metadata when the turn recorded one.
  996. served_runtime = ((usage or {}).get("runtime") if isinstance(usage, dict) else None) or (
  997. result.get("runtime") if isinstance(result, dict) else None) or {}
  998. else:
  999. with self._profile_scope(run.request_profile):
  1000. agent = self._create_agent(
  1001. stream_delta_callback=_text_cb, tool_progress_callback=self._make_run_event_callback(run_id, loop),
  1002. interim_assistant_callback=_interim_cb, **run.agent_kwargs)
  1003. self._active_run_agents[run_id] = agent
  1004. approval_notify = _make_approval_notify(self, run, _api_server=_api_server)
  1005. result, usage, served_runtime = await _submit_api_worker(
  1006. loop, lambda: _run_agent_sync(self, run, agent, approval_notify, _api_server=_api_server))
  1007. # Publish request metrics (daily counters + latency) with each completed run (#52323).
  1008. self._record_api_metrics(usage, time.perf_counter() - _run_started_at)
  1009. if not isinstance(result, dict):
  1010. result = {}
  1011. # The committed outcome decides: a stop issued over WS by another viewer interrupts
  1012. # this run just as much as one issued through this adapter's own /stop.
  1013. status, fields = terminal_run_status(result)
  1014. if status == "cancelled":
  1015. _finish("cancelled", fields)
  1016. elif result.get("failed"):
  1017. # Non-retryable client errors (401/400) return failed=True rather than raising.
  1018. _finish("failed", fields, error=_redact_api_error_text(result.get("error") or "agent run failed"))
  1019. else:
  1020. # ``runtime`` rides on both the pollable status and the run.completed event via _finish, in the
  1021. # canonical shape every other api_server surface emits (route_source/requested, cleaned ids).
  1022. requested = {k: run.agent_kwargs.get(f"requested_{k}") for k in ("provider", "model")}
  1023. served_runtime = self._sanitize_runtime_metadata(
  1024. runtime=served_runtime, requested_runtime=requested if any(requested.values()) else None,
  1025. route_source=("model_routes" if run.agent_kwargs.get("route")
  1026. else "raw_request" if any(requested.values()) else "global"))
  1027. _finish(status, fields, output=result.get("final_response", ""), usage=usage, runtime=served_runtime)
  1028. except asyncio.CancelledError:
  1029. _finish("cancelled")
  1030. raise
  1031. except _api_server._ProviderAuthResolutionError as exc:
  1032. # Same controlled provider-auth message the _run_agent() endpoints give.
  1033. logger.warning("Provider resolution failed for run=%s: %s", run_id, exc)
  1034. _finish("failed", error=exc.user_text())
  1035. except Exception as exc:
  1036. if getattr(exc, "reason", None) == "admission_queued":
  1037. # Still queued at a retiring owner: the next owner runs it once, and GET /v1/runs/{id}
  1038. # reads that row. A run.failed here would invite a poller to resubmit it.
  1039. self._set_run_status(run_id, "queued", last_event="run.queued")
  1040. with suppress(Exception):
  1041. loop.call_soon(run.put_event, _run_event(run_id, "run.queued", message=str(exc)))
  1042. return
  1043. logger.exception("[api_server] run %s failed", run_id)
  1044. _finish("failed", error=_redact_api_error_text(exc))
  1045. finally:
  1046. # On cancellation (/stop) the executor thread may still block on an approval
  1047. # Event; unregistering releases it. Idempotent on normal completion.
  1048. _unregister_approval_notify(run.approval_session_key)
  1049. with suppress(Exception):
  1050. loop.call_soon(run.put_event, None) # close after the queued events
  1051. _retire_live_run(self, run_id)
  1052. def _unregister_approval_notify(approval_session_key: Optional[str]) -> None:
  1053. """Best-effort release of a run's approval waiter (no-op without a key)."""
  1054. with suppress(Exception):
  1055. from tools.approval import unregister_gateway_notify
  1056. if approval_session_key:
  1057. unregister_gateway_notify(approval_session_key)
  1058. def _release_run_owner_if_forgotten(self, run_id: str) -> None:
  1059. """Drop the owner stamp only once nothing keyed by *run_id* survives: ownership must
  1060. outlive every surface it protects (retired on different clocks); ownerless = fail-closed."""
  1061. live = (self._run_statuses, self._active_run_agents, self._active_run_tasks, self._run_streams,
  1062. self._run_approval_sessions)
  1063. if not any(run_id in table for table in live):
  1064. self._run_owners.pop(run_id, None)
  1065. def _request_owns_run(self, request: web.Request, run_id: str) -> bool:
  1066. scope = self._run_idempotency_scope(request)
  1067. owner = self._run_owners.get(run_id)
  1068. if owner is not None:
  1069. return owner == scope
  1070. # No in-memory owner: only a durable record under the caller's scope admits it.
  1071. # Under multiplex_profiles every profile holds a valid key, so ownerless = allow-all.
  1072. # Run state that exists without an owner stamp is an unanswered authorization question, not a run anyone
  1073. # may control — under gateway.multiplex_profiles every served profile holds a valid key, so admitting it
  1074. # would make the boundary allow-all (#93689).
  1075. if self._run_idempotency_store.owns_run(scope, run_id):
  1076. return True
  1077. from gateway.session_api_turn import owns_api_run
  1078. return owns_api_run(self, run_id, scope)
  1079. def _load_owned_run(self, request, *, _api_server, permission: Optional[str], active_fallback: bool):
  1080. """Authenticate (*permission* -> room-grant aware; ``None`` -> API key only) and resolve
  1081. ``(run_id, status, agent, task, error)``; *active_fallback* reports a live in-process run
  1082. without pollable status as ``running`` instead of 404."""
  1083. auth_err = self._check_run_auth(request, permission=permission) if permission else self._check_auth(request)
  1084. if auth_err:
  1085. return None, None, None, None, auth_err
  1086. _openai_error = _api_server._openai_error
  1087. run_id = request.match_info["run_id"]
  1088. if not self._request_owns_run(request, run_id):
  1089. return run_id, None, None, None, _run_not_found(_openai_error, run_id)
  1090. agent = self._active_run_agents.get(run_id)
  1091. task = self._active_run_tasks.get(run_id)
  1092. status = self._durable_run_status(request, run_id)
  1093. if status is None and active_fallback and (agent is not None or task is not None):
  1094. status = self._set_run_status(run_id, "running")
  1095. if status is None:
  1096. return run_id, None, agent, task, _run_not_found(_openai_error, run_id)
  1097. return run_id, status, agent, task, None
  1098. async def _handle_get_run(self, request: web.Request, *, _api_server) -> web.Response:
  1099. """GET /v1/runs/{run_id} — return pollable run status for external UIs."""
  1100. _, status, _, _, err = _load_owned_run(
  1101. self, request, _api_server=_api_server, permission="status", active_fallback=True)
  1102. return err or web.json_response(status)
  1103. async def _handle_run_events(self, request: web.Request, *, _api_server) -> web.StreamResponse:
  1104. """GET /v1/runs/{run_id}/events — stream structured agent lifecycle events."""
  1105. # Same door as GET /v1/runs/{id}: a room grant that may poll the run may stream it.
  1106. auth_err = self._check_run_auth(request, permission="status")
  1107. if auth_err:
  1108. return auth_err
  1109. run_id = request.match_info["run_id"]
  1110. if not self._request_owns_run(request, run_id):
  1111. return _run_not_found(_api_server._openai_error, run_id)
  1112. # Allow subscribing slightly before the run is registered (race window).
  1113. # Confirm the force-kill actually reaped the process before we clear its PID file / scoped locks.
  1114. # SIGKILL can fail to take (e.g. an uninterruptible-sleep or zombie-reaping parent), and if we blindly
  1115. # clear the metadata and start a fresh instance we end up with two live gateways fighting over the same
  1116. # token — the duplicate-gateway failure in #19471.
  1117. for _ in range(20):
  1118. if run_id in self._run_streams:
  1119. break
  1120. await asyncio.sleep(0.05)
  1121. else:
  1122. return _run_not_found(_api_server._openai_error, run_id)
  1123. stream = self._run_streams[run_id]
  1124. raw_last_seq = request.headers.get("Last-Event-ID") or request.query.get("last_seq")
  1125. try:
  1126. last_seq = max(-1, int(str(raw_last_seq).strip())) if raw_last_seq is not None else -1
  1127. except (TypeError, ValueError):
  1128. last_seq = -1
  1129. q, replay = stream.attach(last_seq)
  1130. response = web.StreamResponse(status=200, headers=self._sse_headers(request))
  1131. async def _write(data: bytes) -> None:
  1132. try:
  1133. async with asyncio.timeout(_RUN_STREAM_WRITE_TIMEOUT):
  1134. await response.write(data)
  1135. except TimeoutError:
  1136. with suppress(Exception):
  1137. response.force_close()
  1138. raise
  1139. async def _write_event(seq: int, event: dict[str, Any]) -> None:
  1140. payload = dict(event)
  1141. payload["seq"] = seq
  1142. await _write(_api_server._sse_frame(payload, id=seq))
  1143. prepared = False
  1144. try:
  1145. await response.prepare(request)
  1146. prepared = True
  1147. # Flush the response head before waiting on the queue: aiohttp holds the headers
  1148. # until the first body write, so a subscriber that connects before the run's first
  1149. # event (e.g. before `approval.request`) sees no bytes and fetch()/EventSource never
  1150. # resolve. A comment frame is ignored by every conforming SSE consumer.
  1151. await _write(b": open\n\n")
  1152. if replay and replay[0][0] > last_seq + 1:
  1153. truncation = _run_event(
  1154. run_id,
  1155. "replay.truncated",
  1156. oldest_retained_seq=replay[0][0],
  1157. )
  1158. if raw_last_seq is not None:
  1159. truncation["requested_seq"] = last_seq
  1160. await _write(_api_server._sse_frame(truncation))
  1161. for seq, event in replay:
  1162. if event is None:
  1163. await _write(b": stream closed\n\n")
  1164. return response
  1165. await _write_event(seq, event)
  1166. if stream.terminal and q.empty():
  1167. await _write(b": stream closed\n\n")
  1168. return response
  1169. while True:
  1170. try:
  1171. seq, event = await asyncio.wait_for(
  1172. q.get(), timeout=_api_server.CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS)
  1173. except TimeoutError:
  1174. await _write(b": keepalive\n\n")
  1175. continue
  1176. if event is _RUN_STREAM_SUBSCRIBER_OVERFLOW:
  1177. logger.debug("[api_server] closing slow SSE subscriber for run %s", run_id)
  1178. break
  1179. if event is None: # run finished
  1180. await _write(b": stream closed\n\n")
  1181. break
  1182. await _write_event(seq, event)
  1183. except Exception as exc:
  1184. if not prepared:
  1185. raise
  1186. logger.debug("[api_server] SSE stream error for run %s: %s", run_id, exc)
  1187. finally:
  1188. stream.detach(q)
  1189. self._release_run_owner_if_forgotten(run_id)
  1190. return response
  1191. def _mark_run_event(self, run_id: str, name: str, **fields: Any) -> None:
  1192. """Record a control-plane event on the run status and (best effort) its SSE stream."""
  1193. self._set_run_status(run_id, "running", last_event=name)
  1194. q = self._run_streams.get(run_id)
  1195. if q is not None:
  1196. with suppress(Exception):
  1197. q.put_nowait(_run_event(run_id, name, **fields))
  1198. _APPROVAL_CHOICE_ALIASES = {"approve": "once", "approved": "once", "allow": "once"}
  1199. async def _handle_run_approval(self, request: web.Request, *, _api_server) -> web.Response:
  1200. """POST /v1/runs/{run_id}/approval — resolve a pending run approval."""
  1201. _openai_error = _api_server._openai_error
  1202. run_id, _, _, _, err = _load_owned_run(
  1203. self, request, _api_server=_api_server, permission="approve", active_fallback=False)
  1204. if err is not None:
  1205. return err
  1206. try:
  1207. body = await request.json()
  1208. except Exception:
  1209. return _json_error(_openai_error, "Invalid JSON", status=400)
  1210. if getattr(self.gateway_runner, 'session_authority', None) is not None:
  1211. if self._room_grant_token(request) and body.get('choice') not in {'once', 'deny'}:
  1212. return _json_error(_openai_error, 'Room approvals require once or deny', status=400)
  1213. return await _respond_authority_run(self, run_id, body, kind='approval', _api_server=_api_server)
  1214. raw_choice = str(body.get("choice", "")).strip().lower()
  1215. choice = _APPROVAL_CHOICE_ALIASES.get(raw_choice, raw_choice)
  1216. room_scoped = bool(self._room_grant_token(request))
  1217. raw_request_id = body.get("request_id")
  1218. request_id = raw_request_id.strip() if isinstance(raw_request_id, str) else ""
  1219. # Room grants may resolve exactly one request and never widen to session/always.
  1220. allowed = {"once", "deny"} if room_scoped else {"once", "session", "always", "deny"}
  1221. resolve_all = any(_api_server._coerce_request_bool(body.get(k), default=False) for k in ("all", "resolve_all"))
  1222. approval_session_key = self._run_approval_sessions.get(run_id)
  1223. for failed, message, code, status in (
  1224. (raw_request_id is not None and (not request_id or len(request_id) > 256),
  1225. "Approval request_id is invalid.", "invalid_approval_request", 400),
  1226. (choice not in allowed,
  1227. "Invalid approval choice; expected one of: " + ", ".join(sorted(allowed)),
  1228. "invalid_approval_choice", 400),
  1229. (room_scoped and resolve_all,
  1230. "Room approvals can resolve only one exact request", "invalid_approval_scope", 400),
  1231. (room_scoped and not request_id,
  1232. "Room approvals require the exact request_id.", "approval_request_required", 400),
  1233. (not approval_session_key,
  1234. f"Run has no active approval session: {run_id}", "approval_not_active", 409)):
  1235. if failed:
  1236. return _json_error(_openai_error, message, code=code, status=status)
  1237. try:
  1238. from tools.approval import resolve_gateway_approval
  1239. resolved = resolve_gateway_approval(
  1240. approval_session_key, choice, resolve_all=resolve_all, request_id=request_id or None)
  1241. except Exception as exc:
  1242. logger.exception("[api_server] approval resolution failed for run %s", run_id)
  1243. return _json_error(_openai_error, str(exc), status=500)
  1244. if resolved <= 0:
  1245. return _json_error(
  1246. _openai_error, f"Run has no pending approval: {run_id}", code="approval_not_pending", status=409)
  1247. request_id_field = {"request_id": request_id} if request_id else {}
  1248. _mark_run_event(self, run_id, "approval.responded", choice=choice, **request_id_field, resolved=resolved)
  1249. return web.json_response({
  1250. "object": "hermes.run.approval_response", "run_id": run_id, "choice": choice, **request_id_field,
  1251. "resolved": resolved})
  1252. async def _respond_authority_run(self, run_id, body, *, kind, _api_server):
  1253. from gateway.platforms.api_server_authority_runs import respond_run, store_error_response
  1254. from hermes_state_runtime import RuntimeStoreError
  1255. try:
  1256. result = await respond_run(self, run_id, body, kind=kind)
  1257. return web.json_response({'run_id': run_id, **result})
  1258. except RuntimeStoreError as exc:
  1259. return store_error_response(_api_server._openai_error, exc)
  1260. async def _handle_run_clarify(self, request, *, _api_server):
  1261. run_id, _, _, _, err = _load_owned_run(
  1262. self, request, _api_server=_api_server, permission="approve", active_fallback=False)
  1263. if err is not None:
  1264. return err
  1265. body, err = await self._read_json_body(request)
  1266. if err is not None:
  1267. return err
  1268. return await _respond_authority_run(self, run_id, body, kind='clarify', _api_server=_api_server)
  1269. def _steer_refused(_openai_error, run_id):
  1270. return _json_error(_openai_error, f"Run is not currently accepting steer input: {run_id}",
  1271. code="run_not_accepting_steer", status=409)
  1272. async def _handle_steer_run(self, request: web.Request, *, _api_server) -> web.Response:
  1273. """POST /v1/runs/{run_id}/steer — inject guidance into a running agent."""
  1274. _openai_error = _api_server._openai_error
  1275. run_id, status, agent, _, err = _load_owned_run(
  1276. self, request, _api_server=_api_server, permission=None, active_fallback=False)
  1277. if err is not None:
  1278. return err
  1279. owned = None
  1280. if getattr(self.gateway_runner, 'session_authority', None) is not None:
  1281. from gateway.platforms.api_server_authority_runs import run_admission
  1282. owned = run_admission(self, run_id)
  1283. owned = owned if run_id not in self._stopping_run_ids else None
  1284. agent = None
  1285. # /stop keeps agent refs during cooperative shutdown, so the status gate (not the
  1286. # agent ref) is what rejects stop-then-steer.
  1287. if status.get("status") != "running" or (owned is None and not hasattr(agent, "steer")):
  1288. return _steer_refused(_openai_error, run_id)
  1289. body, err = await self._read_json_body(request)
  1290. if err:
  1291. return err
  1292. raw_text = body.get("input") or body.get("message") or body.get("text") or ""
  1293. steer_text = _api_server._normalize_chat_content(raw_text).strip()
  1294. if not steer_text:
  1295. return _json_error(
  1296. _openai_error, "Missing non-empty steer text; expected 'input', 'message', or 'text'.",
  1297. code="invalid_steer_input", status=400)
  1298. if owned is not None:
  1299. from gateway.session_contract import SessionRef
  1300. from hermes_state_runtime import RuntimeStoreError
  1301. authority, row = owned
  1302. sid, latched = row['target_session_id'], False
  1303. try:
  1304. with authority.sessions[sid].event_stream.lock:
  1305. authority.check_approval_generation(sid, row['generation'])
  1306. # Before the claimed turn adopts its agent the steer is latched for that agent.
  1307. agent, latched = authority.steer_agent(SessionRef(authority.profile_id, sid), row['generation'], steer_text)
  1308. except (KeyError, RuntimeStoreError):
  1309. agent = None
  1310. if not latched and not hasattr(agent, "steer"):
  1311. return _steer_refused(_openai_error, run_id)
  1312. if latched:
  1313. _mark_run_event(self, run_id, "run.steered", accepted=True)
  1314. return web.json_response({"object": "hermes.run.steer", "run_id": run_id, "accepted": True})
  1315. try:
  1316. accepted = bool(agent.steer(steer_text))
  1317. except Exception as exc:
  1318. logger.exception("[api_server] steer failed for run %s", run_id)
  1319. return _json_error(_openai_error, _api_server._redact_api_error_text(exc), code="steer_failed", status=500)
  1320. if not accepted:
  1321. return _json_error(
  1322. _openai_error, f"Run did not accept steer text: {run_id}", code="steer_not_accepted", status=409)
  1323. _mark_run_event(self, run_id, "run.steered", accepted=True)
  1324. return web.json_response({"object": "hermes.run.steer", "run_id": run_id, "accepted": True})
  1325. async def _handle_stop_run(self, request: web.Request, *, _api_server) -> web.Response:
  1326. """POST /v1/runs/{run_id}/stop — interrupt a running agent."""
  1327. _openai_error = _api_server._openai_error
  1328. run_id, status, agent, task, err = _load_owned_run(
  1329. self, request, _api_server=_api_server, permission="stop", active_fallback=True)
  1330. if err is not None:
  1331. return err
  1332. # A run whose turn a live Bot Chat owns (``_execute_run_via_live_owner``) is not an API
  1333. # admission: its stop is the legacy in-process task cancel below, never ``stop_run``.
  1334. if (getattr(self.gateway_runner, 'session_authority', None) is not None
  1335. and not (task is not None and status.get("delivery_id"))):
  1336. from gateway.platforms.api_server_authority_runs import stop_run, store_error_response
  1337. from hermes_state_runtime import RuntimeStoreError
  1338. try:
  1339. return web.json_response(await stop_run(self, run_id))
  1340. except RuntimeStoreError as exc:
  1341. return store_error_response(_openai_error, exc)
  1342. if status.get("status") in TERMINAL_STATUSES:
  1343. return web.json_response(status)
  1344. if agent is None and task is None:
  1345. return _json_error(
  1346. _openai_error, f"Run is not active in this gateway process: {run_id}",
  1347. code="run_not_active", status=409)
  1348. self._set_run_status(run_id, "stopping", last_event="run.stopping")
  1349. self._stopping_run_ids.add(run_id)
  1350. if agent is not None:
  1351. with suppress(Exception):
  1352. _api_server.request_hard_interrupt(agent, "Stop requested via API")
  1353. # Reap only this run's background processes (epoch-gated inside, so a concurrent
  1354. # run on the same session_id keeps its own); no-op if the run already finished.
  1355. _api_server._reap_disconnected_agent_processes(agent, source="api_server_run_stop")
  1356. return web.json_response({"run_id": run_id, "status": "stopping"})
  1357. async def _handle_resolve_unknown_run(
  1358. self, request: web.Request, *, _api_server
  1359. ) -> web.Response:
  1360. """POST /v1/runs/{run_id}/resolve-unknown — acknowledge one lost owner epoch."""
  1361. run_id, _, _, _, err = _load_owned_run(
  1362. self, request, _api_server=_api_server, permission="stop", active_fallback=False)
  1363. if err is not None:
  1364. return err
  1365. try:
  1366. body = await request.json()
  1367. except (ValueError, LookupError): # bad JSON/UTF-8 (ValueError) or unknown charset: no body
  1368. body = None
  1369. from gateway.platforms.api_server_authority_runs import resolve_unknown_run, store_error_response
  1370. from hermes_state_runtime import RuntimeStoreError
  1371. try:
  1372. result = await resolve_unknown_run(self, run_id, body)
  1373. return web.json_response({'run_id': run_id, **result})
  1374. except RuntimeStoreError as exc:
  1375. return store_error_response(_api_server._openai_error, exc)
  1376. async def _sweep_orphaned_runs(self) -> None:
  1377. """Periodically expire transport buffers and terminal status records."""
  1378. while True:
  1379. await asyncio.sleep(60)
  1380. self._sweep_orphaned_runs_once(time.time())
  1381. def _sweep_orphaned_runs_once(self, now: Optional[float] = None) -> None:
  1382. """Expire old SSE buffers without treating transport age as run age."""
  1383. if now is None:
  1384. now = time.time()
  1385. for run_id, created_at in list(self._run_streams_created.items()):
  1386. stream = self._run_streams.get(run_id)
  1387. if now - created_at <= self._RUN_STREAM_TTL or (stream is not None and stream.subscribers):
  1388. continue
  1389. logger.debug("[api_server] sweeping expired run transport %s", run_id)
  1390. task = self._active_run_tasks.get(run_id)
  1391. # Transport TTL bounds buffering; live control state survives until the task returns.
  1392. _drop_run_transport(self, run_id)
  1393. if task is None or task.done():
  1394. _unregister_approval_notify(self._run_approval_sessions.get(run_id))
  1395. _retire_live_run(self, run_id)
  1396. for run_id, status in list(self._run_statuses.items()):
  1397. if (status.get("status") in {"completed", "failed", "cancelled"}
  1398. and now - float(status.get("updated_at", 0) or 0) > self._RUN_STATUS_TTL):
  1399. _forget_run(self, run_id, self._run_statuses, self._run_idempotency_ids)