codex_runtime.py 88 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483
  1. """Codex API runtime — App Server and Responses-API streaming paths. Every entry point takes the parent
  2. AIAgent first: ``run_codex_app_server_turn`` drives one ``codex app-server`` subprocess turn;
  3. ``run_codex_stream`` runs one streaming Codex Responses call."""
  4. from __future__ import annotations
  5. import contextvars
  6. import functools
  7. import json
  8. import logging
  9. import os
  10. import threading
  11. import time
  12. from contextlib import suppress
  13. from types import SimpleNamespace
  14. from typing import Any, Callable, Dict, List
  15. from agent.stream_single_writer import claim_stream_writer, stream_writer_is_current
  16. from agent.transports.hermes_tools_mcp_server import HERMES_TOOLS_MCP_SERVER_NAME
  17. from agent.sdk_transform_bypass import bypass_sdk_request_transform
  18. from agent.stream_diag import buffer_connect_exhausted_notice
  19. from agent.usage_anchor import set_usage_anchor
  20. logger = logging.getLogger(__name__)
  21. _codex_watchdog_state_var: contextvars.ContextVar[Any | None] = contextvars.ContextVar(
  22. "codex_watchdog_state", default=None
  23. )
  24. _DEFAULT_STREAM_DRAIN_TIMEOUT = 2.0
  25. def _stream_drain_timeout() -> float:
  26. """``agent.stream_drain_timeout`` (seconds) — how long the post-terminal SSE drain may block.
  27. The drain is a courtesy to Relay's finalizer, never a correctness requirement: ``final`` is fully
  28. assembled before it starts. A relay that never closes the socket after ``response.completed`` would
  29. otherwise wedge the turn until the idle watchdog discards the already-billed response (#103864).
  30. ``0`` skips the drain entirely.
  31. """
  32. try:
  33. from hermes_cli.config import load_config_readonly
  34. agent_cfg = load_config_readonly().get("agent")
  35. value = agent_cfg.get("stream_drain_timeout") if isinstance(agent_cfg, dict) else None
  36. if isinstance(value, (int, float)) and not isinstance(value, bool):
  37. return max(0.0, float(value))
  38. except Exception:
  39. pass
  40. return _DEFAULT_STREAM_DRAIN_TIMEOUT
  41. def _call_guarded(fn: Callable | None, fail_msg: str, *fail_args: Any, args: tuple = (), kwargs: dict | None = None):
  42. """Invoke an optional display/debug callback; a buggy hook must never tear down the turn."""
  43. if fn is None:
  44. return
  45. try:
  46. fn(*args, **(kwargs or {}))
  47. except Exception:
  48. logger.debug(fail_msg, *fail_args, exc_info=True)
  49. def _codex_request_failure_details(error: BaseException) -> tuple[int | None, str]:
  50. """(serialized request bytes, exception class chain); the buffered ``httpx.Request`` content
  51. on OpenAI connection errors gives the exact byte count without logging payloads or URLs."""
  52. request_body_bytes: int | None = None
  53. exception_classes: list[str] = []
  54. current: BaseException | None = error
  55. seen: set[int] = set()
  56. while current is not None and id(current) not in seen and len(seen) < 8:
  57. seen.add(id(current))
  58. exception_classes.append(type(current).__name__)
  59. if request_body_bytes is None:
  60. content = None
  61. with suppress(Exception):
  62. content = getattr(getattr(current, "request", None), "content", None)
  63. if isinstance(content, str):
  64. request_body_bytes = len(content.encode("utf-8"))
  65. elif isinstance(content, (bytes, bytearray, memoryview)):
  66. request_body_bytes = len(content)
  67. implicit_chain = current.__cause__ is None and not current.__suppress_context__
  68. current = current.__context__ if implicit_chain else current.__cause__
  69. return request_body_bytes, " <- ".join(exception_classes)
  70. def _prune_zero_event_retry_payload(api_kwargs: dict, attempt: int, attempts: int) -> dict:
  71. """#95429 criterion 3: a reconnect after an attempt that produced no stream event must not resend
  72. a pathological payload unchanged. Inline ``function_call_output`` strings over the per-result
  73. threshold (results that escaped commit-time persistence) are spilled through the standard
  74. policy -- bounded preview + recoverable ``<persisted-output>`` reference -- and ONE log line
  75. records the size delta. When nothing is prunable the resend is logged as unchanged. The
  76. caller's kwargs are never mutated; only the retried wire payload changes."""
  77. from tools.budget_config import DEFAULT_BUDGET
  78. from tools.tool_result_storage import maybe_persist_tool_result
  79. items = api_kwargs.get("input")
  80. if not isinstance(items, list):
  81. return api_kwargs
  82. before = len(json.dumps(items, default=str).encode("utf-8"))
  83. if before <= DEFAULT_BUDGET.turn_budget:
  84. return api_kwargs
  85. pruned_items, pruned = [], 0
  86. for item in items:
  87. output = item.get("output") if isinstance(item, dict) and item.get("type") == "function_call_output" else None
  88. if isinstance(output, str):
  89. replaced = maybe_persist_tool_result(content=output, tool_name="codex_zero_event_retry",
  90. tool_use_id=str(item.get("call_id") or "call"),
  91. config=DEFAULT_BUDGET, threshold=DEFAULT_BUDGET.default_result_size)
  92. if replaced != output:
  93. item, pruned = {**item, "output": replaced}, pruned + 1
  94. pruned_items.append(item)
  95. if not pruned:
  96. logger.warning("Codex zero-event retry (attempt %s/%s): no prunable tool output; resending payload "
  97. "unchanged (serialized_input_bytes=%s, model=%s)", attempt, attempts, before, api_kwargs.get("model"))
  98. return api_kwargs
  99. after = len(json.dumps(pruned_items, default=str).encode("utf-8"))
  100. logger.warning("Codex zero-event retry (attempt %s/%s): spilled %d oversized tool output(s) before reconnect, "
  101. "serialized_input_bytes=%s -> %s (model=%s)", attempt, attempts, pruned, before, after,
  102. api_kwargs.get("model"))
  103. return {**api_kwargs, "input": pruned_items}
  104. def _coerce_usage_int(value: Any) -> int:
  105. if isinstance(value, bool):
  106. return 0
  107. if isinstance(value, int):
  108. return max(value, 0)
  109. if isinstance(value, float):
  110. return max(int(value), 0)
  111. if isinstance(value, str):
  112. # Only the str->int parse is guarded; a float NaN still raises like it always has.
  113. with suppress(ValueError):
  114. return max(int(value), 0)
  115. return 0
  116. def _queue_token_counts(agent, fail_msg: str, *fail_extra: Any, counts: Callable[[], dict]) -> None:
  117. """Enqueue per-call accounting for the SessionDB background writer. ``counts`` is built
  118. lazily inside the guarded try so a stub agent without a session DB is never touched."""
  119. if not (agent._session_db and agent.session_id):
  120. return
  121. try:
  122. if not agent._session_db_created:
  123. agent._ensure_db_session()
  124. from agent.turn_usage import _agent_session_source
  125. agent._session_db.queue_token_counts(agent.session_id, source=_agent_session_source(agent), **counts())
  126. except Exception as exc:
  127. logger.debug(fail_msg, agent.session_id, *fail_extra, exc)
  128. def _record_codex_app_server_usage(agent, turn, messages=None) -> dict[str, Any]:
  129. """Translate Codex app-server token usage into Hermes accounting. Prompt bucket = uncached + cached
  130. input (the protocol exposes no cache-write tokens); a turn with no usage still counts as one API call.
  131. ``messages`` (the transcript mirror) lets real usage anchor the next preflight: this runtime bypasses
  132. the main loop's capture, and the mirror is never compacted natively, so without an anchor the rough
  133. estimate grows monotonically and hermes-mode fires thread compaction on tiny threads (#100381)."""
  134. agent.session_api_calls += 1
  135. usage = getattr(turn, "token_usage_last", None)
  136. compressor = getattr(agent, "context_compressor", None)
  137. def billing(**extra):
  138. return dict(model=agent.model, billing_provider=agent.provider, billing_base_url=agent.base_url, api_call_count=1, **extra)
  139. if not isinstance(usage, dict) or not usage:
  140. if compressor is not None and getattr(compressor, "awaiting_real_usage_after_compression", False):
  141. # No usage cannot adjudicate the pending compaction; unlatch preflight deferral.
  142. compressor.update_from_response({})
  143. if compressor is not None and callable(getattr(compressor, "note_usage_less_response", None)):
  144. compressor.note_usage_less_response()
  145. _queue_token_counts(agent, "Codex app-server api-call persistence failed (session=%s): %s",
  146. counts=lambda: billing(billing_mode="subscription_included"))
  147. return {}
  148. from agent.usage_pricing import CanonicalUsage, estimate_usage_cost
  149. # ``inputTokens`` is INCLUSIVE of ``cachedInputTokens`` (same contract as the Responses API, see
  150. # normalize_usage's codex_responses branch); CanonicalUsage.prompt_tokens re-adds cache_read on top of
  151. # input_tokens, so the canonical input bucket must be the UNCACHED remainder or cached tokens count twice.
  152. cache_read_tokens = _coerce_usage_int(usage.get("cachedInputTokens"))
  153. canonical_usage = CanonicalUsage(
  154. input_tokens=max(0, _coerce_usage_int(usage.get("inputTokens")) - cache_read_tokens),
  155. output_tokens=_coerce_usage_int(usage.get("outputTokens")),
  156. cache_read_tokens=cache_read_tokens, cache_write_tokens=0,
  157. reasoning_tokens=_coerce_usage_int(usage.get("reasoningOutputTokens")), raw_usage=usage,
  158. )
  159. prompt_tokens = canonical_usage.prompt_tokens
  160. total_tokens = _coerce_usage_int(usage.get("totalTokens")) or canonical_usage.total_tokens
  161. token_counts = {f: getattr(canonical_usage, f) for f in
  162. ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens")}
  163. usage_dict = {"prompt_tokens": prompt_tokens, "completion_tokens": canonical_usage.output_tokens,
  164. "total_tokens": total_tokens, **token_counts}
  165. if compressor is not None:
  166. try:
  167. compressor.update_from_response(usage_dict)
  168. context_window = getattr(turn, "model_context_window", None)
  169. if isinstance(context_window, int) and context_window > 0:
  170. compressor.context_length = context_window
  171. except Exception:
  172. logger.debug("codex app-server usage update failed", exc_info=True)
  173. if isinstance(messages, list):
  174. from agent.usage_anchor import capture_usage_anchor, set_usage_anchor
  175. anchor = capture_usage_anchor(prompt_tokens, canonical_usage.output_tokens, messages)
  176. if anchor is not None:
  177. set_usage_anchor(agent, anchor)
  178. for key, value in usage_dict.items():
  179. setattr(agent, f"session_{key}", getattr(agent, f"session_{key}") + value)
  180. cost_result = estimate_usage_cost(
  181. agent.model, canonical_usage, provider=agent.provider, base_url=agent.base_url, api_key=getattr(agent, "api_key", ""),
  182. )
  183. cost_usd = float(cost_result.amount_usd) if cost_result.amount_usd is not None else None
  184. if cost_usd is not None:
  185. agent.session_estimated_cost_usd += cost_usd
  186. agent.session_cost_status, agent.session_cost_source = cost_result.status, cost_result.source
  187. cost_fields = {"estimated_cost_usd": cost_usd, "cost_status": cost_result.status, "cost_source": cost_result.source}
  188. _queue_token_counts(
  189. agent, "Codex app-server token persistence failed (session=%s, tokens=%d): %s", total_tokens,
  190. counts=lambda: billing(**token_counts, **cost_fields,
  191. billing_mode="subscription_included" if cost_result.status == "included" else None),
  192. )
  193. return {**usage_dict, "last_prompt_tokens": prompt_tokens, **cost_fields}
  194. def _record_codex_app_server_compaction(agent, turn, *, approx_tokens: int | None = None, force: bool = False) -> bool:
  195. """Record a Codex-native compaction boundary: the app-server owns the compacted thread,
  196. so local transcript rows are NOT rewritten — only session event/usage counters."""
  197. if not force and not getattr(turn, "compacted", False):
  198. return False
  199. thread_id, turn_id = getattr(turn, "thread_id", None) or "", getattr(turn, "turn_id", None) or ""
  200. logger.info("codex app-server compaction observed: session=%s thread=%s turn=%s force=%s",
  201. getattr(agent, "session_id", None) or "none", thread_id, turn_id, force)
  202. if not force:
  203. with suppress(Exception):
  204. from agent.conversation_compression import COMPACTION_STATUS
  205. agent._emit_status(COMPACTION_STATUS)
  206. compressor = getattr(agent, "context_compressor", None)
  207. if compressor is not None:
  208. compressor.compression_count = getattr(compressor, "compression_count", 0) + 1
  209. compressor.last_compression_rough_tokens = approx_tokens or 0
  210. # Codex owns this summary: a prior Hermes deterministic-fallback flag must not leak into it.
  211. record_boundary = getattr(type(compressor), "record_completed_compaction", None)
  212. if callable(record_boundary):
  213. record_boundary(compressor, used_fallback=False)
  214. elif hasattr(compressor, "_verify_compaction_cleared_threshold"):
  215. compressor._verify_compaction_cleared_threshold = True
  216. if not getattr(turn, "token_usage_last", None):
  217. compressor.last_prompt_tokens, compressor.last_completion_tokens = -1, 0
  218. compressor.awaiting_real_usage_after_compression = True
  219. # Provider-side context was rewritten; the usage anchor's transcript snapshot no longer matches.
  220. set_usage_anchor(agent, None)
  221. agent._last_compaction_in_place = False
  222. _call_guarded(getattr(agent, "event_callback", None) or None, "event_callback error on codex session:compress",
  223. args=("session:compress", {
  224. "platform": getattr(agent, "platform", None) or "", "session_id": getattr(agent, "session_id", None) or "",
  225. "old_session_id": "", "in_place": False,
  226. "compression_count": getattr(compressor, "compression_count", 0) if compressor is not None else 0,
  227. "runtime": "codex_app_server", "thread_id": thread_id, "turn_id": turn_id,
  228. }))
  229. if not force:
  230. # Hermes-forced compactions are recorded by their compress_context attempt; this one Codex ran alone.
  231. from agent.compaction_events import publish_provider_native
  232. publish_provider_native(agent)
  233. return True
  234. # --- Codex app-server → Hermes UI bridge -------------------------------------
  235. # The app-server bypasses the Hermes tool loop, so the bridge translates JSON-RPC notifications
  236. # into the callbacks the standard runtime fires (tool_progress_callback, _fire_stream_delta, ...).
  237. # Item types that project to a Hermes tool_call (keep in sync with agent/transports/codex_event_projector.py
  238. # so UI names match recorded names). webSearch is codex's built-in tool; the projector records it under the same id.
  239. _CODEX_TOOL_ITEM_TYPES = frozenset({"commandExecution", "fileChange", "mcpToolCall", "dynamicToolCall", "webSearch"})
  240. # Text-delta notifications → the agent stream hook each one feeds. Single source for both the display
  241. # handlers and the liveness set below, so a new delta method can't stream without refreshing activity (#118410).
  242. _CODEX_TEXT_DELTA_METHODS = (
  243. ("item/agentMessage/delta", "_fire_stream_delta"),
  244. ("item/reasoning/delta", "_fire_reasoning_delta"),
  245. ("item/reasoning/summaryDelta", "_fire_reasoning_delta"),
  246. ("item/reasoning/textDelta", "_fire_reasoning_delta"),
  247. ("item/reasoning/summaryTextDelta", "_fire_reasoning_delta"),
  248. )
  249. # Notifications that prove the turn is alive: every text delta plus tool output streams (no UI handler).
  250. _CODEX_PROGRESS_DELTA_METHODS = frozenset(m for m, _ in _CODEX_TEXT_DELTA_METHODS) | {
  251. "item/commandExecution/outputDelta", "item/fileChange/outputDelta",
  252. }
  253. _CODEX_PROGRESS_ITEM_TYPES = _CODEX_TOOL_ITEM_TYPES | {"agentMessage", "reasoning"}
  254. # Internal MCP server wrapping Hermes' native tools: its inner dispatch has no tool_progress_callback, so the
  255. # codex-level mcpToolCall IS the display event and the mcp.hermes-tools.* prefix is stripped (users see Hermes tools).
  256. _STATIC_TOOL_NAMES = {"commandExecution": "exec_command", "fileChange": "apply_patch", "webSearch": "web_search"}
  257. _STABLE_ID_PREFIXES = {"commandExecution": "exec", "fileChange": "apply_patch"}
  258. _MCP_LIKE_ITEM_TYPES = {"mcpToolCall", "dynamicToolCall"}
  259. # Item types whose preview is the first 120 chars of one string field.
  260. _PREVIEW_FIELDS = {"commandExecution": "command", "webSearch": "query"}
  261. def _delta_text(params: dict) -> str:
  262. """Non-empty text carried by an app-server delta notification, else ""."""
  263. text = params.get("delta") or params.get("text")
  264. return text if isinstance(text, str) else ""
  265. def _item_changes(item: dict) -> list[dict]:
  266. return [c for c in (item.get("changes") or []) if isinstance(c, dict)]
  267. def _codex_item_to_tool_name(item: dict) -> str:
  268. """Synthetic Hermes tool name for a codex item (mirrors CodexEventProjector)."""
  269. item_type = item.get("type") or ""
  270. if item_type == "mcpToolCall":
  271. server, tool = item.get("server") or "mcp", item.get("tool") or "unknown"
  272. return tool if server == HERMES_TOOLS_MCP_SERVER_NAME else f"mcp.{server}.{tool}"
  273. if item_type == "dynamicToolCall":
  274. return item.get("tool") or "dynamic"
  275. return _STATIC_TOOL_NAMES.get(item_type) or item_type or "unknown"
  276. def _codex_item_to_args(item: dict) -> dict:
  277. """Args dict for tool_progress_callback("tool.started"); mirrors the projector shapes."""
  278. item_type = item.get("type") or ""
  279. if item_type == "commandExecution":
  280. return {"command": item.get("command") or "", "cwd": item.get("cwd") or ""}
  281. if item_type == "fileChange":
  282. return {"changes": [{"kind": (c.get("kind") or {}).get("type") or "update", "path": c.get("path") or ""}
  283. for c in _item_changes(item)]}
  284. if item_type in _MCP_LIKE_ITEM_TYPES:
  285. args = item.get("arguments") or {}
  286. return args if isinstance(args, dict) else {"arguments": args}
  287. return {"query": item.get("query") or ""} if item_type == "webSearch" else {}
  288. def _codex_item_to_preview(item: dict) -> Any:
  289. """Short preview for the tool.started bubble; None when nothing useful (UI tolerates None)."""
  290. item_type = item.get("type") or ""
  291. if item_type in _PREVIEW_FIELDS:
  292. return (item.get(_PREVIEW_FIELDS[item_type]) or "")[:120] or None
  293. if item_type == "fileChange":
  294. paths = [c.get("path") for c in _item_changes(item) if c.get("path")]
  295. return (", ".join(paths[:3]) + (f", +{len(paths) - 3} more" if len(paths) > 3 else "")) if paths else None
  296. if item_type in _MCP_LIKE_ITEM_TYPES:
  297. args = item.get("arguments") or {}
  298. if isinstance(args, dict) and args:
  299. with suppress(TypeError, ValueError):
  300. return json.dumps(args, ensure_ascii=False)[:120]
  301. return None
  302. def _codex_item_completion_payload(item: dict) -> tuple[str, bool]:
  303. """(result_text, is_error) for a completed tool item — display-facing text for live tool cards (the
  304. persisted history keeps the projector's ``{exit_code, output}`` envelope instead)."""
  305. item_type = item.get("type") or ""
  306. if item_type == "commandExecution":
  307. out, exit_code = item.get("aggregatedOutput") or "", item.get("exitCode")
  308. is_error = bool(exit_code is not None and exit_code != 0)
  309. return (f"[exit {exit_code}]\n{out}" if is_error else out), is_error
  310. if item_type == "fileChange":
  311. status = item.get("status") or "unknown"
  312. n = len(item.get("changes") or [])
  313. return f"apply_patch status={status}, {n} change(s)", status not in {"completed", "applied", "success"}
  314. if item_type == "mcpToolCall":
  315. if error := item.get("error"):
  316. return f"[error] {json.dumps(error, ensure_ascii=False)[:1000]}", True
  317. result = item.get("result")
  318. return (json.dumps(result, ensure_ascii=False)[:4000] if result is not None else ""), False
  319. if item_type == "dynamicToolCall":
  320. content_items, success = item.get("contentItems") or [], item.get("success", True)
  321. has_items = isinstance(content_items, list) and content_items
  322. return (json.dumps(content_items, ensure_ascii=False)[:4000] if has_items else f"success={success}"), not bool(success)
  323. return "", False
  324. def _stable_call_id(item: dict, name: str) -> str:
  325. """Deterministic tool_call id mirroring CodexEventProjector (live TUI card correlates with projected history)."""
  326. from agent.transports.codex_event_projector import _deterministic_call_id
  327. item_type = item.get("type") or ""
  328. tool = item.get("tool") or "unknown"
  329. prefix = {"mcpToolCall": f"mcp__{item.get('server') or 'mcp'}__{tool}", "dynamicToolCall": f"dyn_{tool}"}.get(item_type)
  330. return _deterministic_call_id(prefix or _STABLE_ID_PREFIXES.get(item_type, name), item.get("id") or "")
  331. def make_codex_app_server_event_bridge(agent) -> Callable[[dict], None]:
  332. """Build the ``on_event`` callback for ``CodexAppServerSession(on_event=...)``.
  333. Tool items fire ``tool_progress_callback`` plus the stable-ID ``tool_start_callback`` /
  334. ``tool_complete_callback`` card hooks; deltas go to ``_fire_stream_delta`` / ``_fire_reasoning_delta``;
  335. a completed agentMessage goes to ``_emit_interim_assistant_message`` (the gateway's ``already_streamed``
  336. check dedupes against streamed deltas). Current-turn progress refreshes the activity clock even
  337. without display hooks. Every callback is guarded so a buggy display hook cannot tear down the turn loop."""
  338. # item_id -> (tool_name, args, started_monotonic); duration even when codex omits durationMs.
  339. started: dict[str, tuple[str, dict, float]] = {}
  340. def agent_cb(attr: str, fail_msg: str, *fail_args: Any, args: tuple = (), kwargs: dict | None = None) -> None:
  341. _call_guarded(getattr(agent, attr, None), fail_msg, *fail_args, args=args, kwargs=kwargs)
  342. def _fire_tool_started(item: dict) -> None:
  343. item_id, name = item.get("id") or "", _codex_item_to_tool_name(item)
  344. args = _codex_item_to_args(item)
  345. if item_id:
  346. started[item_id] = (name, args, time.monotonic())
  347. agent_cb("tool_progress_callback", "tool_progress_callback raised on tool.started for %s", name,
  348. args=("tool.started", name, _codex_item_to_preview(item), args),
  349. kwargs={"tool_call_id": _stable_call_id(item, name)} if getattr(agent, "_yaovi_embedded", False) else None)
  350. # Stable-ID tool card (TUI/desktop) fires alongside the progress bubble.
  351. agent_cb("tool_start_callback", "tool_start_callback raised for %s", name,
  352. args=(_stable_call_id(item, name), name, args))
  353. def _fire_tool_completed(item: dict) -> None:
  354. name = _codex_item_to_tool_name(item)
  355. prior = started.pop(item_id, None) if (item_id := item.get("id") or "") else None
  356. # Prefer codex's durationMs; else our started timestamp; else None (some codex
  357. # versions only emit completed for fast items).
  358. codex_ms = item.get("durationMs")
  359. has_codex_ms = isinstance(codex_ms, (int, float)) and codex_ms >= 0
  360. duration: Any = codex_ms / 1000.0 if has_codex_ms else (time.monotonic() - prior[2] if prior else None)
  361. result, is_error = _codex_item_completion_payload(item)
  362. agent_cb("tool_progress_callback", "tool_progress_callback raised on tool.completed for %s", name,
  363. args=("tool.completed", name, None, None),
  364. kwargs={"duration": duration, "is_error": is_error, "result": result,
  365. **({"tool_call_id": _stable_call_id(item, name)} if getattr(agent, "_yaovi_embedded", False) else {})})
  366. args = prior[1] if prior is not None else _codex_item_to_args(item)
  367. agent_cb("tool_complete_callback", "tool_complete_callback raised for %s", name,
  368. args=(_stable_call_id(item, name), name, args, result))
  369. def _fire_delta(params: dict, attr: str) -> None:
  370. text = _delta_text(params)
  371. # Single-writer guard (#65991): a superseded stream must not pollute the turn's accumulated text
  372. # (which also feeds the interim-visible-text de-dup comparison), even when a caller reaches this
  373. # directly (the tool-suppressed content path) rather than through _fire_stream_delta.
  374. if text:
  375. agent_cb(attr, f"{attr} raised", args=(text,))
  376. def _fire_agent_message_completed(item: dict) -> None:
  377. text = item.get("text") or ""
  378. # display.show_commentary=false keeps mid-turn narration off the interim path too (codex_responses contract).
  379. if isinstance(text, str) and text.strip() and getattr(agent, "show_commentary", True):
  380. agent_cb("_emit_interim_assistant_message", "_emit_interim_assistant_message raised",
  381. args=({"role": "assistant", "content": text},))
  382. # Each agentMessage item is its own delivered message: the completed item was just compared
  383. # against ITS deltas, so drop them before the next item's deltas arrive. Otherwise the buffer
  384. # holds "commentary + final", the final agentMessage no longer prefix-matches, and it is
  385. # re-delivered with already_streamed=False as a second copy (#74248 boundary 2).
  386. agent._current_streamed_assistant_text = ""
  387. def _on_item(params: dict, completed: bool) -> None:
  388. item = params.get("item")
  389. if not isinstance(item, dict):
  390. return
  391. item_type = item.get("type") or ""
  392. if item_type in _CODEX_TOOL_ITEM_TYPES:
  393. (_fire_tool_completed if completed else _fire_tool_started)(item)
  394. elif completed and item_type == "agentMessage":
  395. _fire_agent_message_completed(item)
  396. handlers: dict[str, Callable[[dict], None]] = {
  397. method: functools.partial(_fire_delta, attr=attr) for method, attr in _CODEX_TEXT_DELTA_METHODS
  398. }
  399. handlers["item/started"] = lambda p: _on_item(p, completed=False)
  400. handlers["item/completed"] = lambda p: _on_item(p, completed=True)
  401. def on_event(note: dict) -> None:
  402. if not isinstance(note, dict):
  403. return
  404. method = note.get("method") or ""
  405. params = note.get("params")
  406. params = params if isinstance(params, dict) else {}
  407. # The session has already filtered foreign thread/turn notifications. Count
  408. # progress even without UI callbacks (or when commentary is hidden), but
  409. # never let empty deltas or transport keepalives mask a stalled turn.
  410. item = params.get("item")
  411. is_delta = method in _CODEX_PROGRESS_DELTA_METHODS and bool(_delta_text(params))
  412. is_item = (
  413. method in {"item/started", "item/completed"} and isinstance(item, dict)
  414. and item.get("type") in _CODEX_PROGRESS_ITEM_TYPES
  415. )
  416. if is_delta or is_item:
  417. agent_cb("_touch_activity", "_touch_activity raised", args=(f"codex app-server: {method}",))
  418. handler = handlers.get(method)
  419. if handler is not None:
  420. handler(params)
  421. return on_event
  422. # --- Codex app-server turn ----------------------------------------------------
  423. def _close_codex_session(agent) -> None:
  424. """Drop the session so the next turn respawns codex instead of reusing a dead client."""
  425. with suppress(Exception):
  426. agent._codex_session.close()
  427. agent._codex_session = None
  428. def _consume_user_interrupt(agent, active: bool = True) -> tuple[bool, Any]:
  429. """(user_interrupted, interrupt_message); clears the agent-level interrupt so a hard
  430. stop cannot poison the next turn (mirrors the conversation-loop finalizer)."""
  431. interrupted = bool(active and getattr(agent, "_interrupt_requested", False))
  432. message = getattr(agent, "_interrupt_message", None) if interrupted else None
  433. if interrupted:
  434. agent.clear_interrupt()
  435. return interrupted, message
  436. def _codex_developer_instructions(agent) -> str:
  437. """The prompt composition the standard loop sends as its system message (turn_context order)."""
  438. developer_instructions = getattr(agent, "_cached_system_prompt", None) or ""
  439. if getattr(agent, "ephemeral_system_prompt", None):
  440. developer_instructions = (developer_instructions + "\n\n" + agent.ephemeral_system_prompt).strip()
  441. return developer_instructions
  442. # Durable codex thread binding: ``sessions.model_config.codex_thread_id`` (hermes_state), written after the
  443. # turn's projected rows were committed, read by the next AIAgent built for the same Hermes session so an
  444. # API-server restart (or the per-request agents of /api/sessions/{id}/chat) resumes the model-side thread
  445. # instead of starting an empty one while Hermes' own transcript continues (#100531).
  446. _CODEX_THREAD_ID_KEY = "codex_thread_id"
  447. _CODEX_THREAD_RESUME_NOTICE = "Codex thread could not be resumed; starting a new one."
  448. def _stored_codex_thread_id(agent) -> str | None:
  449. db, session_id = getattr(agent, "_session_db", None), getattr(agent, "session_id", None)
  450. if db is None or not session_id:
  451. return None
  452. thread_id = db.get_session_model_config_value(session_id, _CODEX_THREAD_ID_KEY)
  453. return thread_id if isinstance(thread_id, str) and thread_id else None
  454. def _store_codex_thread_id(agent, thread_id: str | None) -> None:
  455. """Merge (``None`` clears) the binding into the session row; a failed write only logs — the turn is done."""
  456. db, session_id = getattr(agent, "_session_db", None), getattr(agent, "session_id", None)
  457. if db is None or not session_id:
  458. return
  459. _call_guarded(db.patch_session_model_config, "codex thread id could not be stored on the session row",
  460. args=(session_id, {_CODEX_THREAD_ID_KEY: thread_id}))
  461. def _start_codex_thread(agent) -> str:
  462. """``ensure_started`` with the fail-closed resume policy: a stored thread that codex cannot hand back
  463. (unknown id, rollout locked by a killed app-server, different thread) is dropped from the session row,
  464. the user is told once on the status rail, and a fresh thread starts on the same client."""
  465. from agent.transports.codex_app_server_session import CodexThreadResumeError
  466. try:
  467. return agent._codex_session.ensure_started()
  468. except CodexThreadResumeError as exc:
  469. logger.warning("%s; starting a new codex thread (session=%s)", exc.message, getattr(agent, "session_id", None))
  470. _store_codex_thread_id(agent, None)
  471. agent._emit_diagnostic_status(_CODEX_THREAD_RESUME_NOTICE)
  472. return agent._codex_session.ensure_started()
  473. def _codex_model_provider(agent) -> str | None:
  474. """codex's ``[model_providers.<id>]`` for a named custom provider (``providers.<name>``); None means codex's
  475. own provider. Only the stable id is sent and codex resolves base_url/env_key itself, so Hermes' credential
  476. never enters the JSON-RPC payload (#75186)."""
  477. if str(getattr(agent, "provider", "") or "").strip().lower() != "custom":
  478. return None
  479. from hermes_cli.runtime_provider_custom import codex_model_provider_id
  480. return codex_model_provider_id(str(getattr(agent, "requested_provider", "") or ""))
  481. def _codex_wire_model(agent, model_provider: str | None) -> str | None:
  482. """The slug codex should run. ``-900k`` picker variants are Hermes-side aliases the backend rejects
  483. ("not supported when using Codex with a ChatGPT account"); codex applies the catalog's extended window
  484. to the base slug itself. On codex's own provider the slug is bare (``openai/gpt-5.5`` -> ``gpt-5.5``),
  485. as for openai-codex; a named custom provider's model ids are its own and pass through."""
  486. from agent.model_metadata import strip_codex_context_variant_suffix
  487. model = strip_codex_context_variant_suffix(getattr(agent, "model", None)) or None
  488. if model and model_provider is None:
  489. from hermes_cli.model_normalize import normalize_model_for_provider
  490. model = normalize_model_for_provider(model, "openai-codex")
  491. return model
  492. def _codex_turn_effort(agent, model: str | None) -> str | None:
  493. """``turn/start.effort``: only an explicit Hermes reasoning setting overrides codex's own default, clamped
  494. to the route's vocabulary like the Responses path (a level the model lacks fails the turn with 400
  495. "Unsupported value"). ``ultra`` stays ``ultra`` where the model reaches ``max``: codex runs it as its
  496. harness mode. Disabled reasoning goes out as ``none`` where the route accepts it."""
  497. reasoning_config = getattr(agent, "reasoning_config", None)
  498. # Guard first: _resolve_reasoning fills an unset config with "medium", which would override codex's default.
  499. if not isinstance(reasoning_config, dict) or not (
  500. reasoning_config.get("enabled") is False or reasoning_config.get("effort")):
  501. return None
  502. from agent.codex_responses_adapter import classify_responses_route
  503. from agent.reasoning_effort import route_supported_efforts
  504. from agent.transports.codex import _resolve_reasoning
  505. route = classify_responses_route(agent)
  506. effort, _enabled = _resolve_reasoning(model or "", {
  507. "reasoning_config": reasoning_config, "provider": getattr(agent, "provider", None),
  508. "base_url": getattr(agent, "base_url", None), "is_codex_backend": route.is_codex_backend,
  509. "is_xai_responses": route.is_xai_responses,
  510. })
  511. # ``ultra`` is codex's harness mode, not an inference level: keep it where the route accepts it
  512. # instead of the ``max`` the Responses clamp maps it to.
  513. if effort == "max" and reasoning_config.get("effort") == "ultra" and "ultra" in route_supported_efforts(
  514. getattr(agent, "provider", None), model, "codex_app_server"):
  515. return "ultra"
  516. return effort
  517. def _codex_turn_service_tier(agent) -> str | None:
  518. """``turn/start.serviceTier``: the tier Hermes' own Responses path would request this turn (a static
  519. ``/fast`` tier pinned in request_overrides, or an open ``auto``/``cold`` window), in codex's words: the
  520. OpenAI ``priority`` tier is codex's ``fast``. A tier codex has no word for (``ultrafast``) is not sent."""
  521. from agent.fast_mode import CODEX_TIER_WORDS, effective_request_overrides
  522. return CODEX_TIER_WORDS.get(effective_request_overrides(agent).get("service_tier"))
  523. def _ensure_codex_session(agent, messages: list[dict[str, Any]] | None = None) -> None:
  524. """Lazily spawn one CodexAppServerSession per AIAgent (reused across turns, closed by the _cleanup hook).
  525. A live session whose thread was started with a different prompt composition (TUI/Desktop ``/personality``
  526. or a prompt mirror mutate the agent in place) is retired first so the new thread carries the current one.
  527. Only the FIRST session of an AIAgent resumes the stored codex thread: a retired/recreated one keeps
  528. today's fresh-thread behaviour and overwrites the binding once its turn is committed. ``messages`` is the
  529. turn's transcript (current user row last); a thread started from scratch is seeded with the prior turns."""
  530. developer_instructions = _codex_developer_instructions(agent)
  531. model_provider = _codex_model_provider(agent)
  532. if getattr(agent, "_codex_session", None) is not None:
  533. # Only a session whose recorded composition differs is stale; one attached without a record is kept.
  534. # The provider is fixed per thread (turn/start can switch the model, not the provider), so an
  535. # in-place switch to or from a named custom provider retires the thread too.
  536. recorded = getattr(agent, "_codex_session_prompt", None)
  537. if recorded is None or (recorded == developer_instructions
  538. and getattr(agent, "_codex_session_model_provider", None) == model_provider):
  539. return
  540. _close_codex_session(agent)
  541. resume_thread_id = None if getattr(agent, "_codex_session_prompt", None) is not None else _stored_codex_thread_id(agent)
  542. from agent.runtime_cwd import resolve_agent_cwd
  543. from agent.transports.codex_app_server_session import CodexAppServerSession, _ServerRequestRouting
  544. from hermes_cli.codex_runtime_switch import get_configured_codex_binary
  545. from hermes_cli.config import load_config
  546. # Approval callback: Hermes' standard prompt flow when a CLI thread installed one. `hermes chat -q`, cron and
  547. # unattended platforms can have one registered with nobody to answer it, so they get none and fail closed at once.
  548. approval_callback = None
  549. with suppress(Exception):
  550. from tools.approval_context import _no_user_can_answer
  551. from tools.terminal_tool import _get_approval_callback
  552. if not _no_user_can_answer():
  553. approval_callback = _get_approval_callback()
  554. # Gateway/cron have no UI for codex approval requests, so exec/apply_patch fail closed by default. Only an
  555. # explicit approval bypass (approvals.mode: off, /yolo, --yolo, HERMES_YOLO_MODE) hands policy to codex's sandbox.
  556. auto_approve_requests = False
  557. try:
  558. from tools.approval import is_approval_bypass_active
  559. auto_approve_requests = is_approval_bypass_active()
  560. except Exception:
  561. logger.debug("codex app-server: approval-bypass lookup failed; keeping fail-closed default", exc_info=True)
  562. # Bridge codex JSON-RPC notifications (item/started, item/completed, item/agentMessage/delta, ...) into
  563. # Hermes' gateway UI callbacks (tool_progress_callback, _fire_stream_delta,
  564. # _emit_interim_assistant_message). Without this, Discord/Telegram users see no live tool-progress or
  565. # interim commentary while codex_app_server is running — only the final answer (#33200). Supersedes the
  566. # narrower item/started-only bridge from #38835.
  567. # Hermes owns the prompt: the same composition the standard loop sends as its system message
  568. # (cached per-session prompt + ephemeral additions such as channel overrides) rides along ONCE per
  569. # thread as developerInstructions. A retired/recreated session re-sends the current composition.
  570. # A thread started from scratch (no resumable codex thread) also receives the session's prior turns
  571. # once, so a /model switch into codex or a retired thread does not start blind (#74712, #26035).
  572. # The recorded composition stays the bare prompt: the seed must not make the next turn retire the thread.
  573. agent._codex_session_prompt = developer_instructions
  574. agent._codex_session_model_provider = model_provider
  575. from agent.codex_runtime_history_seed import render_history_seed
  576. history_seed = render_history_seed(messages) or None
  577. # The model always rides along: codex's home is shared, while the Hermes model is per profile/session,
  578. # so omitting it ran codex's own default instead of the selection.
  579. agent._codex_session = CodexAppServerSession(
  580. cwd=getattr(agent, "session_cwd", None) or str(resolve_agent_cwd()), approval_callback=approval_callback,
  581. codex_bin=get_configured_codex_binary(load_config()),
  582. request_routing=_ServerRequestRouting(auto_approve_exec=auto_approve_requests, auto_approve_apply_patch=auto_approve_requests),
  583. on_event=make_codex_app_server_event_bridge(agent),
  584. developer_instructions=developer_instructions or None,
  585. model=_codex_wire_model(agent, model_provider), model_provider=model_provider,
  586. resume_thread_id=resume_thread_id, history_seed=history_seed,
  587. )
  588. def _persist_projected_messages(agent, turn, messages: list[dict[str, Any]]) -> bool:
  589. """Splice the projected messages into ``messages`` and flush them to the session DB; True when the
  590. rows are durable in the session DB (the codex thread binding may then be published).
  591. Bypasses conversation_loop's per-step _persist_session(); the flush dedups via _DB_PERSISTED_MARKER so
  592. only the new codex rows are written. The agent stays the sole persister (agent_persisted=True): a
  593. gateway re-write would re-INSERT the user turn."""
  594. if not turn.projected_messages:
  595. return False
  596. from agent.message_metadata import append_message
  597. projected_messages = turn.projected_messages
  598. # Turn-start persistence owns the accepted input. Codex's leading user item
  599. # only echoes its coerced wire text; later/nonmatching events remain real.
  600. submitted_user_text = getattr(turn, "submitted_user_text", None)
  601. first = projected_messages[0]
  602. if (submitted_user_text is not None and first.get("role") == "user"
  603. and first.get("content") == submitted_user_text):
  604. projected_messages = projected_messages[1:]
  605. for projected_message in projected_messages:
  606. append_message(messages, projected_message)
  607. if getattr(agent, "_session_db", None) is None:
  608. return False
  609. flush_ok = False
  610. try:
  611. flush_ok = agent._flush_messages_to_session_db(messages)
  612. except Exception:
  613. logger.warning("codex app-server projected-message flush failed", exc_info=True)
  614. if flush_ok is False:
  615. # Output already streamed and agent_persisted cannot flip to False: surface the gap loudly.
  616. logger.warning("codex app-server turn was delivered but could NOT be persisted to the session DB "
  617. "(session=%s) — this turn will be missing after restart/resume", getattr(agent, "session_id", None))
  618. return flush_ok is True
  619. def _finish_codex_turn(agent, turn, messages: list[dict[str, Any]], *, original_user_message: Any,
  620. should_review_memory: bool, final_response: Any) -> dict[str, Any]:
  621. """Post-turn bookkeeping mirroring the chat_completions loop; returns usage fields.
  622. ``final_response`` is the reply after ``transform_llm_output``: memory keeps what the user got."""
  623. # run_conversation() already bumped _turns_since_memory / _user_turn_count; only _iters_since_skill is ours.
  624. agent._iters_since_skill = getattr(agent, "_iters_since_skill", 0) + turn.tool_iterations
  625. _record_codex_app_server_compaction(agent, turn)
  626. usage_result = _record_codex_app_server_usage(agent, turn, messages=messages)
  627. # Skill nudge check AFTER iters were incremented (same as chat_completions).
  628. should_review_skills = (0 < agent._skill_nudge_interval <= agent._iters_since_skill
  629. and "skill_manage" in agent.valid_tool_names)
  630. if should_review_skills:
  631. agent._iters_since_skill = 0
  632. # External memory sync skipped on interrupt/error (no partial transcripts).
  633. if not turn.interrupted and turn.error is None:
  634. _call_guarded(getattr(agent, "_sync_external_memory_for_turn", None), "external memory sync raised", kwargs=dict(
  635. original_user_message=original_user_message, final_response=final_response, interrupted=False, messages=messages,
  636. ))
  637. # Background review fork: only when a trigger tripped AND a real final response exists.
  638. if turn.final_text and not turn.interrupted and (should_review_memory or should_review_skills):
  639. _call_guarded(getattr(agent, "_spawn_background_review", None), "background review spawn raised", kwargs=dict(
  640. messages_snapshot=list(messages), review_memory=should_review_memory, review_skills=should_review_skills,
  641. ))
  642. return usage_result
  643. def _transform_codex_reply(agent, turn, turn_id: Any) -> dict[str, Any]:
  644. """Run ``transform_llm_output`` on a codex reply before its rows are persisted, as
  645. ``finish_text_response`` does on the generic loop: the user and the stored row get the rewrite.
  646. A turn that failed after an agentMessage completed still delivers that text, so it is rewritten
  647. too; a held turn rewrites even an interrupted one, and its earlier agentMessages (commentary the
  648. user never saw) are typed hidden for every history load."""
  649. held = getattr(agent, "_hold_stream", False)
  650. if not turn.final_text or (turn.interrupted and not held):
  651. return {"final_response": turn.final_text}
  652. from agent.turn_finalizer import apply_llm_output_transform, public_pre_transform
  653. final, transformed, pre_transform = apply_llm_output_transform(agent, turn.final_text, turn_id=turn_id)
  654. rows = [m for m in turn.projected_messages or [] if m.get("role") == "assistant" and m.get("content")]
  655. final_row = next((m for m in reversed(rows) if m.get("content") == turn.final_text), None)
  656. if transformed and final_row is not None:
  657. final_row["content"] = final
  658. for message in rows if held else ():
  659. if message is not final_row and not message.get("display_kind"):
  660. message["display_kind"] = "hidden"
  661. return {"final_response": final, "response_transformed": transformed,
  662. "pre_transform_response": public_pre_transform(agent, pre_transform)}
  663. def run_codex_app_server_turn(agent, *, user_message: str, original_user_message: Any, messages: list[dict[str, Any]],
  664. effective_task_id: str, should_review_memory: bool = False,
  665. turn_id: Any = None) -> dict[str, Any]:
  666. """Hand the turn to a ``codex app-server`` subprocess and project its events into ``messages``.
  667. Returns the chat_completions result shape. The user message is ALREADY in ``messages`` — never append it again."""
  668. # Defense in depth for compression.checkpoint_required: agent init refuses the combination, but
  669. # api_mode is mutable. Explicit-True check matches compress_context().
  670. if getattr(agent, "compression_checkpoint_required", False) is True:
  671. from agent.conversation_compression_checkpoint import _checkpoint_blocked
  672. raise _checkpoint_blocked("codex_app_server owns the authoritative thread and compacts it "
  673. "without a truthful pre-compaction transcript boundary")
  674. _ensure_codex_session(agent, messages)
  675. try:
  676. _start_codex_thread(agent)
  677. wire_model = _codex_wire_model(agent, getattr(agent, "_codex_session_model_provider", None))
  678. turn = agent._codex_session.run_turn(
  679. user_input=user_message,
  680. model=wire_model, reasoning_effort=_codex_turn_effort(agent, wire_model),
  681. service_tier=_codex_turn_service_tier(agent))
  682. except Exception as exc:
  683. logger.exception("codex app-server turn failed")
  684. _close_codex_session(agent)
  685. return _turn_result(
  686. _consume_user_interrupt(agent), messages, api_calls=0, completed=False, error=str(exc),
  687. final_response=f"Codex app-server turn failed: {exc}. Fall back to default runtime with `/codex-runtime auto`.",
  688. )
  689. interrupt = _consume_user_interrupt(agent, turn.interrupted)
  690. # Wedged client (turn deadline blown, OAuth refresh died, subprocess exited): retire it. Post-tool
  691. # silence alone no longer retires — it only logs a warning (#112928).
  692. if getattr(turn, "should_retire", False):
  693. logger.warning("codex app-server session retired (turn error: %s)", turn.error)
  694. _close_codex_session(agent)
  695. reply = _transform_codex_reply(agent, turn, turn_id)
  696. # The binding is published only once the transcript it belongs to is durable, and never for a
  697. # retired thread (the next agent would only resume into the same wedge).
  698. if _persist_projected_messages(agent, turn, messages) and not getattr(turn, "should_retire", False):
  699. _store_codex_thread_id(agent, turn.thread_id)
  700. usage_result = _finish_codex_turn(
  701. agent, turn, messages, original_user_message=original_user_message, should_review_memory=should_review_memory,
  702. final_response=reply["final_response"],
  703. )
  704. return _turn_result(
  705. interrupt, messages, api_calls=1, completed=not turn.interrupted and turn.error is None, error=turn.error,
  706. # We flushed the projected rows ourselves (agent_persisted); the gateway must skip its own DB write.
  707. agent_persisted=True, codex_thread_id=turn.thread_id, codex_turn_id=turn.turn_id, **reply, **usage_result,
  708. )
  709. def _turn_result(interrupt: tuple[bool, Any], messages: list[dict[str, Any]], *, api_calls: int, completed: bool,
  710. error: Any, final_response: Any, **extra: Any) -> dict[str, Any]:
  711. """Result shape shared with the chat_completions path (``partial`` == ``not completed``)."""
  712. user_interrupted, interrupt_message = interrupt
  713. return {
  714. "final_response": final_response, "messages": messages, "api_calls": api_calls,
  715. "completed": completed, "partial": not completed, "interrupted": user_interrupted,
  716. **({"interrupt_message": interrupt_message} if interrupt_message else {}),
  717. "error": error, **extra,
  718. }
  719. # --- Event-driven Responses streaming -----------------------------------------
  720. # The SDK's ``responses.stream(...)`` helper rebuilds a typed Response from ``response.completed.response.output``
  721. # and crashes when it is null. We consume raw ``responses.create(stream=True)`` SSE events and assemble the final
  722. # response from ``output_item.done``, so the terminal ``output`` may be null / [] / a string / absent.
  723. def _event_field(event: Any, name: str, default: Any = None) -> Any:
  724. """Field access for attr-style (SDK objects) and dict (raw JSON) events/items."""
  725. value = getattr(event, name, None)
  726. if value is None and isinstance(event, dict):
  727. value = event.get(name, default)
  728. return value if value is not None else default
  729. _CODEX_PROGRESS_DELTA_TYPES = frozenset({
  730. "response.output_text.delta", "response.reasoning_summary_text.delta", "response.text.delta",
  731. "response.audio.delta", "response.function_call_arguments.delta", "response.reasoning_text.delta",
  732. "response.refusal.delta",
  733. })
  734. def _codex_event_has_content(event: Any) -> bool:
  735. """Whether a Codex Responses event carries substantive forward progress.
  736. Lifecycle/keepalive frames and empty structural deltas prove transport
  737. liveness, but do not mean the model has begun producing its response.
  738. """
  739. event_type = _event_field(event, "type")
  740. if event_type in _CODEX_PROGRESS_DELTA_TYPES:
  741. return bool(_event_field(event, "delta"))
  742. if event_type == "response.output_item.added":
  743. item = _event_field(event, "item")
  744. return "function_call" in str(_event_field(item, "type") or "") and any(
  745. bool(_event_field(item, field)) for field in ("id", "call_id", "name", "arguments"))
  746. return False
  747. def _raise_stream_error(event: Any) -> None:
  748. """Raise ``_StreamErrorEvent`` from a ``type=error`` SSE frame. The spec puts code/message/param at the
  749. top level, but the SDK and several proxies nest them under ``error``; read top-level first, then the envelope."""
  750. from run_agent import _StreamErrorEvent
  751. nested = _event_field(event, "error")
  752. def _error_field(name: str) -> Any:
  753. value = _event_field(event, name)
  754. return _event_field(nested, name) if value is None and nested is not None else value
  755. raw_message = _error_field("message")
  756. message = (str(raw_message) if raw_message is not None else "stream emitted error event").strip() or "stream emitted error event"
  757. raise _StreamErrorEvent(message, code=_error_field("code"), param=_error_field("param"))
  758. def _message_phase(item: Any) -> str | None:
  759. phase = _event_field(item, "phase", None)
  760. return phase.strip().lower() if isinstance(phase, str) else None
  761. def _output_text_of(item: Any) -> str:
  762. """Concatenated ``output_text`` parts of a message item ("" if content is not a list)."""
  763. content_parts = _event_field(item, "content", [])
  764. parts = content_parts if isinstance(content_parts, list) else []
  765. return "".join(
  766. str(_event_field(part, "text", "") or "") for part in parts if _event_field(part, "type", "") == "output_text"
  767. ).strip()
  768. class _CodexResponseAssembler:
  769. """Assemble a Response-shaped ``SimpleNamespace`` from raw Responses SSE events.
  770. Only ``usage`` / ``status`` / ``id`` / ``service_tier`` are read from the terminal frame — never ``response.output``. Output
  771. items come from ``output_item.done``, or are synthesized from text deltas, or settled from function calls
  772. announced via ``output_item.added`` but never confirmed (some backends omit per-item done events on success)."""
  773. has_tool_calls = first_delta_fired = saw_terminal = False
  774. next_output_sequence = 0
  775. active_message_phase: str | None = None
  776. # Reasoning summary parts carry no separator; a summary_index change is where the blank line belongs.
  777. active_summary_index: Any = None
  778. terminal_status: str = "completed"
  779. terminal_usage = terminal_response_id = terminal_incomplete_details = terminal_error = None
  780. terminal_service_tier = None # the tier the backend SERVED (may differ from the one requested)
  781. # terminal_status defaults to "completed", so settlement needs an explicitly observed response.completed frame.
  782. saw_response_completed = False
  783. # An assistant answer message item was announced (output_item.added, not commentary/analysis): its text deltas
  784. # are real answer text even when its done event never arrives. Bare output_text deltas carry no such proof.
  785. saw_answer_item_added = False
  786. def __init__(self, *, model, on_text_delta, on_reasoning_delta, on_commentary_message, on_first_delta):
  787. self.model, self.on_text_delta, self.on_reasoning_delta = model, on_text_delta, on_reasoning_delta
  788. self.on_commentary_message, self.on_first_delta = on_commentary_message, on_first_delta
  789. self.output_items: list[Any] = []
  790. # output_index / first-observed sequence per output item, in lockstep, so settled pending calls merge
  791. # back in stream order.
  792. self.output_indexes, self.output_sequences = [], []
  793. self.text_deltas, self.commentary_text_deltas = [], []
  794. # pending_function_calls: announced-but-unconfirmed function calls keyed by item id. announced_output_order:
  795. # first-observed (sequence, output_index) per announced item id so a later .done keeps its announced position.
  796. self.pending_function_calls: dict[str, dict[str, Any]] = {}
  797. self.announced_output_order: dict[str, tuple] = {}
  798. # Phase per announced message item id: a delta that names its item keeps that item's phase even after a
  799. # later item (a function call) was announced, so commentary is never fused into the answer text.
  800. self.message_phases: dict[str, str | None] = {}
  801. def _safe(self, cb: Callable | None, label: str, *args: Any) -> None:
  802. _call_guarded(cb, f"Codex stream {label} raised", args=args)
  803. def _on_item_added(self, event: Any, event_type: str) -> None:
  804. item = _event_field(event, "item")
  805. item_type = _event_field(item, "type", "")
  806. self.active_message_phase = _message_phase(item) if item_type == "message" else None
  807. if item_type == "message" and self.active_message_phase not in ("commentary", "analysis"):
  808. self.saw_answer_item_added = True
  809. if self.active_message_phase == "commentary":
  810. self.commentary_text_deltas = []
  811. # Record first-observed ordering for EVERY announced item; .done must reuse it or a mixed
  812. # announced/pending stream without output_index values reorders the calls.
  813. item_id = str(_event_field(item, "id", ""))
  814. if item_id and item_type == "message":
  815. self.message_phases[item_id] = self.active_message_phase
  816. if item_id and item_id not in self.announced_output_order:
  817. self.announced_output_order[item_id] = (self.next_output_sequence, _event_field(event, "output_index"))
  818. self.next_output_sequence += 1
  819. if "function_call" in str(item_type):
  820. self.has_tool_calls = True
  821. if item_id:
  822. announced_sequence, announced_index = self.announced_output_order[item_id]
  823. self.pending_function_calls[item_id] = {
  824. "item": item, "arguments": str(_event_field(item, "arguments", "") or ""),
  825. "output_index": announced_index, "sequence": announced_sequence,
  826. }
  827. def _on_text_delta(self, event: Any, event_type: str) -> None:
  828. delta_text = _event_field(event, "delta", "")
  829. if not delta_text:
  830. return
  831. # Harmony commentary/analysis text is mid-turn narration, never the final answer: route to the
  832. # reasoning callback, keep only the item for replay.
  833. phase = self.message_phases.get(str(_event_field(event, "item_id", "") or ""), self.active_message_phase)
  834. if phase == "commentary":
  835. self.commentary_text_deltas.append(delta_text)
  836. # Legacy fallback when no first-class commentary consumer is installed.
  837. if self.on_commentary_message is None:
  838. self._safe(self.on_reasoning_delta, "on_reasoning_delta", delta_text)
  839. elif phase == "analysis":
  840. self._safe(self.on_reasoning_delta, "on_reasoning_delta", delta_text)
  841. else:
  842. self.text_deltas.append(delta_text)
  843. if self.has_tool_calls:
  844. return
  845. if not self.first_delta_fired:
  846. self.first_delta_fired = True
  847. self._safe(self.on_first_delta, "on_first_delta")
  848. self._safe(self.on_text_delta, "on_text_delta", delta_text)
  849. def _on_refusal_delta(self, event: Any, event_type: str) -> None:
  850. # ``response.refusal.delta``: the model declined and streams its explanation on the refusal
  851. # channel instead of output_text. It is answer text — a refusal-only stream must not end
  852. # with zero content and "did not emit a terminal response". The done item's ``refusal``
  853. # part is read by the normalizer; the deltas cover backends that omit the done item.
  854. refusal_text = _event_field(event, "delta", "")
  855. if isinstance(refusal_text, str) and refusal_text:
  856. self.text_deltas.append(refusal_text)
  857. def _pending_function_key(self, item_id: str, output_index: Any) -> str | None:
  858. if item_id in self.pending_function_calls:
  859. return item_id
  860. # Some Responses backends rotate opaque item IDs between SSE events. The response-local
  861. # output_index still identifies the call, including index zero.
  862. if output_index is not None:
  863. return next((key for key, pending in self.pending_function_calls.items()
  864. if pending["output_index"] == output_index), None)
  865. return None
  866. def _on_function_call(self, event: Any, event_type: str) -> None:
  867. self.has_tool_calls = True
  868. key = self._pending_function_key(str(_event_field(event, "item_id", "")),
  869. _event_field(event, "output_index"))
  870. pending = self.pending_function_calls.get(key)
  871. if pending is None:
  872. return # the item itself lands on output_item.done
  873. if "delta" in event_type:
  874. pending["arguments"] += _event_field(event, "delta", "") or ""
  875. elif event_type.endswith("function_call_arguments.done"):
  876. # Authoritative for the accumulated string; an explicit "" (zero-arg call) counts, only a
  877. # missing field keeps the streamed deltas.
  878. if (done_args := _event_field(event, "arguments", None)) is not None:
  879. pending["arguments"] = str(done_args)
  880. def _on_reasoning_delta(self, event: Any, event_type: str) -> None:
  881. reasoning_text = _event_field(event, "delta", "")
  882. if not reasoning_text or self.on_reasoning_delta is None:
  883. return
  884. summary_index = _event_field(event, "summary_index")
  885. if summary_index is not None:
  886. if self.active_summary_index is not None and summary_index != self.active_summary_index:
  887. reasoning_text = f"\n\n{reasoning_text}"
  888. self.active_summary_index = summary_index
  889. self._safe(self.on_reasoning_delta, "on_reasoning_delta", reasoning_text)
  890. def _on_item_done(self, event: Any, event_type: str) -> None:
  891. done_item = _event_field(event, "item")
  892. if done_item is None:
  893. return
  894. self.output_items.append(done_item)
  895. # Reuse the announced position when known (fresh tail sequence for unannounced items); the .done
  896. # event's own output_index wins over the announced one.
  897. done_id = str(_event_field(done_item, "id", ""))
  898. announced_sequence, announced_index = self.announced_output_order.get(done_id, (None, None))
  899. pending_keys = []
  900. if "function_call" in str(_event_field(done_item, "type", "")):
  901. done_call_id = _event_field(done_item, "call_id", "")
  902. if isinstance(done_call_id, str) and done_call_id.strip():
  903. # A stable call_id is exclusive: conflicting item/index identities
  904. # must not evict unrelated calls, even when no alias matches.
  905. pending_keys = [key for key, pending in self.pending_function_calls.items()
  906. if _event_field(pending["item"], "call_id") == done_call_id]
  907. # An announcement that carried no call_id cannot contradict this one, so the
  908. # positional match argument events use is the only evidence left for it.
  909. if not pending_keys:
  910. pending_key = self._pending_function_key(done_id, _event_field(event, "output_index"))
  911. if pending_key is not None and not _event_field(
  912. self.pending_function_calls[pending_key]["item"], "call_id"):
  913. pending_keys = [pending_key]
  914. announced_sequence, announced_index = None, None
  915. else:
  916. # Without a stable call_id, retain item-id then index association,
  917. # as used by argument events (which never carry a call_id).
  918. pending_key = self._pending_function_key(done_id, _event_field(event, "output_index"))
  919. if pending_key is not None:
  920. pending_keys = [pending_key]
  921. if pending_keys:
  922. announced_alias = min((self.pending_function_calls[key] for key in pending_keys),
  923. key=lambda pending: pending["sequence"])
  924. announced_sequence, announced_index = announced_alias["sequence"], announced_alias["output_index"]
  925. if announced_sequence is None:
  926. announced_sequence, self.next_output_sequence = self.next_output_sequence, self.next_output_sequence + 1
  927. self.output_indexes.append(_event_field(event, "output_index", announced_index))
  928. self.output_sequences.append(announced_sequence)
  929. # The done payload is authoritative for every pending alias of this call.
  930. for pending_key in pending_keys:
  931. self.pending_function_calls.pop(pending_key, None)
  932. if _message_phase(done_item) == "commentary" and self.on_commentary_message is not None:
  933. commentary_text = "".join(self.commentary_text_deltas).strip() or _output_text_of(done_item)
  934. if commentary_text:
  935. self._safe(self.on_commentary_message, "on_commentary_message", commentary_text)
  936. self.commentary_text_deltas = []
  937. def _on_terminal(self, event: Any, event_type: str) -> bool:
  938. self.saw_terminal = True
  939. resp_obj = _event_field(event, "response")
  940. if resp_obj is not None:
  941. self.terminal_usage, self.terminal_response_id = _event_field(resp_obj, "usage"), _event_field(resp_obj, "id")
  942. self.terminal_service_tier = _event_field(resp_obj, "service_tier")
  943. rstatus = _event_field(resp_obj, "status")
  944. if isinstance(rstatus, str):
  945. self.terminal_status = rstatus
  946. if event_type == "response.incomplete":
  947. self.terminal_incomplete_details = _event_field(resp_obj, "incomplete_details")
  948. elif event_type == "response.failed":
  949. self.terminal_error = _event_field(resp_obj, "error")
  950. self.saw_response_completed = self.saw_response_completed or event_type == "response.completed"
  951. self.terminal_status = self.terminal_status or event_type.removeprefix("response.")
  952. return True
  953. # Exact-type handlers first, then substring-matched ones in priority order. ``error`` frames
  954. # carry the provider's real failure reason; raise so the credential pool + classifier see the body.
  955. _EXACT_HANDLERS = {
  956. "error": lambda self, event, event_type: _raise_stream_error(event),
  957. "response.output_item.added": _on_item_added, "response.output_item.done": _on_item_done,
  958. "response.completed": _on_terminal, "response.incomplete": _on_terminal, "response.failed": _on_terminal,
  959. "response.refusal.delta": _on_refusal_delta,
  960. }
  961. _FUZZY_HANDLERS = (
  962. (lambda t: "output_text.delta" in t, _on_text_delta), (lambda t: "function_call" in t, _on_function_call),
  963. (lambda t: "reasoning" in t and "delta" in t, _on_reasoning_delta),
  964. )
  965. def feed(self, event: Any) -> bool:
  966. """Process one event; True when the stream hit a terminal frame."""
  967. event_type = _event_field(event, "type", "")
  968. event_type = event_type if isinstance(event_type, str) else ""
  969. handler = self._EXACT_HANDLERS.get(event_type) or next((h for m, h in self._FUZZY_HANDLERS if m(event_type)), None)
  970. return bool(handler(self, event, event_type)) if handler is not None else False
  971. def _settled_output(self) -> list[Any]:
  972. """Merge .done items with settled pending calls, keeping stream order."""
  973. indexed = list(zip(self.output_indexes, self.output_sequences, self.output_items))
  974. for pending in self.pending_function_calls.values():
  975. item = pending["item"]
  976. indexed.append((pending.get("output_index"), pending["sequence"], SimpleNamespace(
  977. type="function_call", id=_event_field(item, "id", None), call_id=_event_field(item, "call_id", None),
  978. name=_event_field(item, "name", None), status="completed",
  979. # Empty/whitespace arguments become "{}" so zero-delta calls stay executable; malformed
  980. # non-empty JSON passes through untouched.
  981. arguments=(pending["arguments"] or "").strip() or "{}",
  982. )))
  983. # output_index is optional: protocol order only when every entry has one, else wire order.
  984. if all(entry[0] is not None for entry in indexed):
  985. with suppress(TypeError): # non-comparable index values: keep wire order
  986. indexed.sort(key=lambda entry: entry[0])
  987. else:
  988. indexed.sort(key=lambda entry: entry[1])
  989. return [entry[2] for entry in indexed]
  990. def result(self) -> SimpleNamespace:
  991. # Successful completion orders all output, even when no calls remain pending.
  992. # Done items stay authoritative; only successful streams settle missing done events.
  993. output: list[Any] = self._settled_output() if self.saw_response_completed else list(self.output_items)
  994. # Plain text deltas with no output at all: synthesize one message item. Beside tool calls and commentary
  995. # items, only an ANNOUNCED answer message whose done event never came is recovered, or the text said around
  996. # a call is dropped from the turn (#143893); bare output_text beside commentary stays narration (length).
  997. # The recovery needs a terminal frame: a stream cut mid-turn (no terminal, a call still pending) must raise
  998. # below and retry, never end the turn on the cut-off sentence with the call silently lost.
  999. if self.text_deltas and (
  1000. (not output and not self.has_tool_calls)
  1001. or (self.saw_terminal and self.saw_answer_item_added and not any(
  1002. _event_field(item, "type", "") == "message"
  1003. and _message_phase(item) not in ("commentary", "analysis") for item in output))):
  1004. content = [SimpleNamespace(type="output_text", text="".join(self.text_deltas))]
  1005. message = SimpleNamespace(type="message", role="assistant", status="completed", content=content)
  1006. first_call = next((i for i, item in enumerate(output)
  1007. if "function_call" in str(_event_field(item, "type", ""))), len(output))
  1008. output.insert(first_call, message)
  1009. # No terminal frame AND no usable content = truncated / rejected stream.
  1010. if not self.saw_terminal and not output:
  1011. raise RuntimeError("Codex Responses stream did not emit a terminal response")
  1012. return SimpleNamespace(
  1013. output=output, output_text="".join(self.text_deltas), usage=self.terminal_usage, status=self.terminal_status,
  1014. id=self.terminal_response_id, model=self.model, incomplete_details=self.terminal_incomplete_details,
  1015. error=self.terminal_error, service_tier=self.terminal_service_tier)
  1016. def _consume_codex_event_stream(
  1017. event_iter: Any, *, model: str, on_text_delta=None, on_reasoning_delta=None, on_commentary_message=None,
  1018. on_first_delta=None, on_event=None, interrupt_check=None,
  1019. ) -> SimpleNamespace:
  1020. """Consume a Codex Responses SSE stream into a Response-shaped ``SimpleNamespace`` (see
  1021. :class:`_CodexResponseAssembler`; ``status`` is ``completed`` when the stream ended with content but no
  1022. terminal frame; ``model`` comes from kwargs).
  1023. Callbacks: ``on_text_delta`` per output_text delta, suppressed once a function_call is seen;
  1024. ``on_reasoning_delta`` for reasoning and ``phase=analysis`` deltas (also commentary without a commentary
  1025. callback); ``on_commentary_message`` once per completed ``phase=commentary`` message, before any following
  1026. tool item; ``on_first_delta`` one-shot; ``on_event`` every event before any processing; ``interrupt_check()``
  1027. True breaks the loop and may raise ``TimeoutError`` / ``InterruptedError`` for request retirement that
  1028. must not become a partial final response."""
  1029. assembler = _CodexResponseAssembler(model=model, on_text_delta=on_text_delta, on_reasoning_delta=on_reasoning_delta,
  1030. on_commentary_message=on_commentary_message, on_first_delta=on_first_delta)
  1031. for event in event_iter:
  1032. if on_event is not None:
  1033. try:
  1034. on_event(event)
  1035. except (TimeoutError, InterruptedError):
  1036. raise # watchdog / cancellation control flow must propagate
  1037. except Exception:
  1038. logger.debug("Codex stream on_event hook raised", exc_info=True)
  1039. if (interrupt_check is not None and interrupt_check()) or assembler.feed(event):
  1040. break
  1041. return assembler.result()
  1042. def _sanitize_consumer_codex_request(agent: Any, request: dict[str, Any]) -> dict[str, Any]:
  1043. """Drop fields the ChatGPT OAuth Codex endpoint rejects, at the final wire boundary (after Relay /
  1044. middleware / ``request_overrides``): a late ``prompt_cache_retention``, top-level or nested in
  1045. ``extra_body``, would otherwise HTTP 400 a valid follow-up."""
  1046. sanitized = dict(request)
  1047. # getattr: run_codex_stream is also driven with stand-in agents carrying only the attrs a path needs.
  1048. backend_predicate = getattr(agent, "_is_codex_backend", None)
  1049. if not (callable(backend_predicate) and bool(backend_predicate())):
  1050. return sanitized
  1051. dropped_from = ["top-level"] if "prompt_cache_retention" in sanitized else []
  1052. sanitized.pop("prompt_cache_retention", None)
  1053. # Copy before editing (caller's mapping must not mutate); drop when emptied.
  1054. extra_body = sanitized.get("extra_body")
  1055. if isinstance(extra_body, dict) and "prompt_cache_retention" in extra_body:
  1056. sanitized["extra_body"] = {k: v for k, v in extra_body.items() if k != "prompt_cache_retention"}
  1057. if not sanitized["extra_body"]:
  1058. sanitized.pop("extra_body")
  1059. dropped_from.append("extra_body")
  1060. if dropped_from:
  1061. logger.warning("Dropped unsupported prompt_cache_retention at consumer Codex wire boundary (model=%s, via %s).",
  1062. sanitized.get("model", getattr(agent, "model", "unknown")), ", ".join(dropped_from))
  1063. return sanitized
  1064. def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta=None):
  1065. """One streaming Responses API request over raw ``responses.create(stream=True)`` events."""
  1066. import httpx as _httpx
  1067. from openai import APIConnectionError as _APIConnectionError
  1068. from agent import relay_llm
  1069. transport_errors = (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ReadError, _httpx.ConnectError, ConnectionError)
  1070. active_client = client or agent._ensure_primary_openai_client(reason="codex_stream_direct")
  1071. max_stream_retries, model = 1, api_kwargs.get("model")
  1072. # Accumulate streamed text so callers / compat shims can read it.
  1073. agent._codex_streamed_text_parts: list = []
  1074. # Retirement token for THIS request (installed by ``interruptible_api_call``). A watchdog that kills the
  1075. # connection clears the agent-level token, so a worker still draining frames can tell it was retired.
  1076. # ``None`` = no watchdog; every check passes.
  1077. watchdog_state = _codex_watchdog_state_var.get()
  1078. request_token = (
  1079. watchdog_state.token
  1080. if watchdog_state is not None
  1081. else getattr(agent, "_active_codex_stream_request_token", None)
  1082. )
  1083. # Delta-sink claim for the CURRENT physical attempt (None until the stream opens). A newer attempt that
  1084. # claims the sink supersedes this token; that only silences OUR live callbacks — consumption continues,
  1085. # because stopping here handed the gateway a "completed" response missing its tail (#69486).
  1086. writer_token = {"value": None, "raw_stream": None, "superseded_logged": False}
  1087. def _request_is_current() -> bool:
  1088. return request_token is None or getattr(agent, "_active_codex_stream_request_token", None) is request_token
  1089. def _writer_is_current() -> bool:
  1090. token = writer_token["value"]
  1091. if token is None or stream_writer_is_current(agent, token):
  1092. return True
  1093. if not writer_token["superseded_logged"]:
  1094. writer_token["superseded_logged"] = True
  1095. logger.warning("Codex streaming attempt superseded by a newer stream; suppressing its live deltas while "
  1096. "consuming to completion so the final response is not truncated (model=%s).",
  1097. api_kwargs.get("model", "unknown"))
  1098. return False
  1099. def _fenced(fn: Callable[[Any], None]) -> Callable[[Any], None]:
  1100. """Wrap a callback so a retired request's late frames never reach the agent."""
  1101. return lambda value: fn(value) if _request_is_current() else None
  1102. def _live(fn: Callable[..., None]) -> Callable[..., None]:
  1103. """Wrap a live-display callback so a superseded writer's frames never reach the sink (retired ones neither)."""
  1104. return lambda *args: fn(*args) if _request_is_current() and _writer_is_current() else None
  1105. def _on_text_delta(text: str) -> None:
  1106. agent._codex_streamed_text_parts.append(text)
  1107. if _writer_is_current():
  1108. agent._fire_stream_delta(text)
  1109. def _on_event(event: Any) -> None: # TTFB/activity touch — once per SSE event.
  1110. now = time.time()
  1111. # Lifecycle frames can precede text, so the first accepted parsed event is the Responses
  1112. # equivalent of Chat Completions' first chunk. Preserve the per-attempt reset; the ``_fenced``
  1113. # wrapper around this callback already keeps a retired worker from overwriting a newer request.
  1114. if getattr(agent, "_last_api_first_chunk_at", None) is None:
  1115. agent._last_api_first_chunk_at = now
  1116. has_progress = _codex_event_has_content(event)
  1117. first_event = first_progress = False
  1118. if watchdog_state is not None:
  1119. with watchdog_state.lock:
  1120. first_event = watchdog_state.last_event_ts is None
  1121. watchdog_state.last_event_ts = now
  1122. if has_progress:
  1123. first_progress = watchdog_state.last_progress_ts is None
  1124. watchdog_state.last_progress_ts = now
  1125. if watchdog_state.phase_aware:
  1126. watchdog_state.retry_started_ts = None
  1127. if first_event:
  1128. logger.info("Codex stream first parsed event at %.3f (attempt=%s/%s, model=%s)",
  1129. now, attempt + 1, max_stream_retries + 1, model)
  1130. if first_progress:
  1131. logger.info("Codex stream first substantive progress at %.3f (attempt=%s/%s, model=%s)",
  1132. now, attempt + 1, max_stream_retries + 1, model)
  1133. agent._touch_activity("receiving stream response")
  1134. def _interrupt_or_superseded() -> bool:
  1135. # A retired request must NOT break out of the consume loop (that returns a partial ``final`` with
  1136. # status "completed"); raise so the watchdog's TimeoutError is seen.
  1137. if not _request_is_current():
  1138. raise TimeoutError("Codex Responses stream request retired before terminal response")
  1139. return bool(agent._interrupt_requested)
  1140. def _open_codex_stream(next_api_kwargs: dict[str, Any]):
  1141. from hermes_cli.providers import is_actual_route
  1142. if is_actual_route(
  1143. getattr(agent, "provider", ""),
  1144. str(getattr(active_client, "base_url", "") or ""),
  1145. ):
  1146. raise ValueError(
  1147. "Actual requests require Chat Completions; refusing to call /responses."
  1148. )
  1149. stream_kwargs = _sanitize_consumer_codex_request(agent, next_api_kwargs)
  1150. stream_kwargs["stream"] = True
  1151. return active_client.responses.create(**bypass_sdk_request_transform(stream_kwargs))
  1152. def _log_failure(exc: BaseException) -> None:
  1153. request_body_bytes, exception_chain = _codex_request_failure_details(exc)
  1154. logger.warning("Codex Responses request failed: serialized_request_body_bytes=%s stream_opened=%s "
  1155. "exception_chain=%s model=%s attempt=%s", "unknown" if request_body_bytes is None else request_body_bytes,
  1156. str(writer_token["value"] is not None).lower(), exception_chain, getattr(agent, "model", "unknown"),
  1157. f"{attempt + 1}/{max_stream_retries + 1}")
  1158. if writer_token["value"] is None:
  1159. # No stream ever opened: the user gets one line naming host/attempts/size (#97548).
  1160. buffer_connect_exhausted_notice(
  1161. agent, exc, attempts=attempt + 1,
  1162. base_url=getattr(active_client, "base_url", None) or getattr(agent, "base_url", ""))
  1163. def _codex_stream_created(_raw_stream: Any) -> None:
  1164. # Claim the delta sink for THIS attempt; a newer attempt supersedes this token.
  1165. writer_token["value"] = claim_stream_writer(agent)
  1166. writer_token["raw_stream"] = _raw_stream
  1167. logger.debug("Codex stream opened (attempt=%s/%s, model=%s)",
  1168. attempt + 1, max_stream_retries + 1, model)
  1169. def _drain_for_finalizer(event_stream: Any) -> None:
  1170. # The final response is already assembled. Keep the finalizer drain on THIS owner thread:
  1171. # moving the reader to a daemon thread and later closing from here releases the FD under that
  1172. # thread's SSL BIO (#127390, same class as #29507). A tiny watchdog may only shutdown() the
  1173. # socket; shutdown wakes this owner-thread read without releasing the descriptor.
  1174. budget = _stream_drain_timeout()
  1175. if budget <= 0:
  1176. return
  1177. from agent.agent_runtime_helpers import _socket_from_response
  1178. # Only the raw SDK stream carries ``.response``; any lookup failure means "not interruptible".
  1179. try:
  1180. sock = _socket_from_response(getattr(writer_token.get("raw_stream"), "response", None))
  1181. except Exception:
  1182. sock = None
  1183. if sock is None:
  1184. # Without a shutdown-capable socket a synchronous drain could become unbounded. The drain
  1185. # is only for Relay's finalizer, so skip it and let the owner-thread finally close below.
  1186. logger.debug("Codex post-terminal drain skipped: no interruptible stream socket found. %s",
  1187. agent._client_log_context())
  1188. return
  1189. timed_out = threading.Event()
  1190. def _wake_owner() -> None:
  1191. timed_out.set()
  1192. # FD-safe from a stranger thread: never close here. The owner continues the iteration,
  1193. # observes EOF/error, and performs the real close from the same thread that was reading.
  1194. from agent.agent_runtime_helpers import _shutdown_socket
  1195. _shutdown_socket(sock)
  1196. watchdog = threading.Timer(budget, _wake_owner)
  1197. watchdog.name = "codex-post-terminal-watchdog"
  1198. watchdog.daemon = True
  1199. try:
  1200. watchdog.start()
  1201. for _ignored in event_stream:
  1202. pass
  1203. except (*transport_errors, _APIConnectionError) as exc:
  1204. # A timeout-triggered shutdown is the expected wakeup, not another provider failure. Other
  1205. # transport failures still get the old diagnostic, but none may discard the completed response.
  1206. if not timed_out.is_set():
  1207. if not isinstance(exc, transport_errors):
  1208. _log_failure(exc)
  1209. logger.warning(
  1210. "Codex Responses stream transport finalization failed after a terminal response was already "
  1211. "received; returning the completed response instead of retrying. %s error=%s",
  1212. agent._client_log_context(), exc,
  1213. )
  1214. except Exception:
  1215. logger.debug("Codex Responses stream finalization failed after a terminal response", exc_info=True)
  1216. finally:
  1217. # cancel() wins if the budget has not fired; join() also waits for an already-running shutdown
  1218. # callback, preserving shutdown-before-close ordering at the exact timeout boundary. A failed
  1219. # or interrupted start() leaves no thread (ident None) to join.
  1220. watchdog.cancel()
  1221. if watchdog.ident is not None:
  1222. watchdog.join()
  1223. if timed_out.is_set():
  1224. logger.warning(
  1225. "Codex Responses stream remained open %.1fs after a terminal response "
  1226. "(agent.stream_drain_timeout); shut down its socket and completed cleanup on the owner thread. %s",
  1227. budget, agent._client_log_context(),
  1228. )
  1229. def _close_event_stream(event_stream: Any) -> None:
  1230. close_fn = getattr(event_stream, "close", None) # None while connect never succeeded
  1231. try:
  1232. if callable(close_fn):
  1233. close_fn()
  1234. except Exception:
  1235. # A failed close can leave this connection checked out of the httpx pool while the caller
  1236. # reuse-caches the client; poison the slot so close really closes the pool. ``client is None``
  1237. # is the shared primary client — never force-shut.
  1238. if client is not None:
  1239. agent._abort_request_openai_client(active_client, reason="codex_stream_close_failed")
  1240. show_commentary = getattr(agent, "show_commentary", True)
  1241. wants_commentary = getattr(agent, "interim_assistant_callback", None) is not None and show_commentary
  1242. on_commentary_message = _live(agent._fire_streamed_codex_commentary) if wants_commentary else None
  1243. call_role = ("delegated" if getattr(agent, "is_subagent", False)
  1244. else "fallback" if int(getattr(agent, "_fallback_index", 0) or 0) > 0 else "primary")
  1245. for attempt in range(max_stream_retries + 1):
  1246. if not _request_is_current():
  1247. raise TimeoutError("Codex Responses stream request retired before retry")
  1248. if agent._interrupt_requested:
  1249. raise InterruptedError("Agent interrupted before Codex stream retry")
  1250. if attempt > 0 and watchdog_state is not None and watchdog_state.phase_aware:
  1251. # One origin for the whole physical attempt: lifecycle frames may change
  1252. # diagnostics, but cannot restart the first-progress budget.
  1253. with watchdog_state.lock:
  1254. watchdog_state.retry_started_ts = time.time()
  1255. watchdog_state.last_event_ts = None
  1256. watchdog_state.last_progress_ts = None
  1257. logger.info("Codex physical stream retry at %.3f (attempt=%s/%s, model=%s)",
  1258. watchdog_state.retry_started_ts, attempt + 1, max_stream_retries + 1, model)
  1259. intercepted_events: list = []
  1260. writer_token["value"] = writer_token["raw_stream"] = event_stream = None
  1261. writer_token["superseded_logged"] = False
  1262. try:
  1263. try:
  1264. event_stream = relay_llm.stream(
  1265. dict(api_kwargs), _open_codex_stream,
  1266. session_id=str(getattr(agent, "session_id", "") or ""),
  1267. name=str(getattr(agent, "provider", "") or "codex"), model_name=str(model or ""),
  1268. finalizer=lambda: _consume_codex_event_stream(list(intercepted_events), model=model),
  1269. on_stream_created=_codex_stream_created, on_chunk=intercepted_events.append,
  1270. chunk_adapter=lambda chunk: chunk,
  1271. completed_response_predicate=lambda r: bool(hasattr(r, "output") and not hasattr(r, "__iter__")),
  1272. metadata={"api_mode": "codex_responses", "call_role": call_role, "retry_count": attempt,
  1273. "api_request_id": getattr(agent, "_current_api_request_id", None)},
  1274. defer_logical_completion=True,
  1275. )
  1276. final = _consume_codex_event_stream(
  1277. event_stream, model=model, on_text_delta=_fenced(_on_text_delta),
  1278. on_reasoning_delta=_live(agent._fire_reasoning_delta), on_commentary_message=on_commentary_message,
  1279. on_first_delta=_live(on_first_delta) if on_first_delta is not None else None,
  1280. on_event=_fenced(_on_event), interrupt_check=_interrupt_or_superseded,
  1281. )
  1282. except transport_errors as exc:
  1283. if attempt >= max_stream_retries:
  1284. _log_failure(exc)
  1285. raise
  1286. logger.debug(
  1287. "Codex Responses stream connect failed (attempt %s/%s); retrying. %s error=%s" if event_stream is None
  1288. else "Codex Responses stream transport failed mid-iteration (attempt %s/%s); retrying. %s error=%s",
  1289. attempt + 1, max_stream_retries + 1, agent._client_log_context(), exc,
  1290. )
  1291. if not intercepted_events: # zero-event attempt: never resend a pathological payload silently
  1292. api_kwargs = _prune_zero_event_retry_payload(api_kwargs, attempt + 1, max_stream_retries + 1)
  1293. continue
  1294. except RuntimeError:
  1295. # "No terminal response"; Relay may still hold a finalizer-assembled response.
  1296. if event_stream is not None and event_stream.final_response is not None:
  1297. return event_stream.final_response
  1298. raise
  1299. except _APIConnectionError as exc:
  1300. # The SDK wraps every connect/receive failure (``raise APIConnectionError from err``), so the
  1301. # raw ``transport_errors`` branch above never sees a pre-stream failure. Before the stream
  1302. # opened nothing is billed, so one fresh physical request is safe (#103673); once the writer
  1303. # token is claimed the inference may already be billed, so mid-stream failures still raise.
  1304. if (attempt < max_stream_retries and writer_token["value"] is None
  1305. and isinstance(exc.__cause__, _httpx.TransportError)):
  1306. logger.debug(
  1307. "Codex Responses pre-stream connect failed (attempt %s/%s); retrying. %s error=%s",
  1308. attempt + 1, max_stream_retries + 1, agent._client_log_context(), exc,
  1309. )
  1310. continue
  1311. _log_failure(exc)
  1312. raise
  1313. if not agent._interrupt_requested:
  1314. _drain_for_finalizer(event_stream)
  1315. if final.status in {"incomplete", "failed"}:
  1316. logger.warning("Codex Responses stream terminal status=%s "
  1317. "(incomplete_details=%s, error=%s, streamed_chars=%d). %s",
  1318. final.status, final.incomplete_details, final.error,
  1319. sum(len(p) for p in agent._codex_streamed_text_parts), agent._client_log_context())
  1320. return final
  1321. finally:
  1322. # relay_llm.stream is ManagedLlmStream, whose close() always owns the provider stream; only
  1323. # when construction itself failed (event_stream None) may a raw stream be left to close here.
  1324. if event_stream is None:
  1325. _close_event_stream(writer_token.get("raw_stream"))
  1326. else:
  1327. _close_event_stream(event_stream)
  1328. __all__ = [
  1329. "_consume_codex_event_stream",
  1330. "make_codex_app_server_event_bridge",
  1331. "run_codex_app_server_turn",
  1332. "run_codex_stream",
  1333. ]