codex_app_server_session.py 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849
  1. """Session adapter for codex app-server runtime.
  2. Owns one Codex thread per Hermes session: drives ``turn/start``, consumes
  3. streaming notifications via CodexEventProjector, bridges server-initiated
  4. approval requests, translates cancellation, and returns a TurnResult that
  5. AIAgent.run_conversation() splices into ``messages``. Synchronous: the client's
  6. reader threads feed queues that this adapter polls, like the chat_completions loop.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import logging
  11. import os
  12. import threading
  13. import time
  14. from dataclasses import dataclass, field
  15. from typing import Any, Callable, Optional
  16. from agent.codex_responses_adapter import _format_responses_error
  17. from agent.redact import redact_sensitive_text
  18. from agent.transports.codex_app_server import (
  19. CodexAppServerClient, CodexAppServerError, CodexAppServerTransportError,
  20. )
  21. from agent.transports.codex_event_projector import CodexEventProjector, ProjectionResult
  22. from agent.transports.hermes_tools_mcp_server import HERMES_TOOLS_MCP_SERVER_NAME
  23. logger = logging.getLogger(__name__)
  24. _STDERR_TAIL_LINES = 12 # stderr tail on generic errors: legible, yet enough for a config/auth diagnostic
  25. _TIER_UNKNOWN = object() # thread resumed: the tier it carries is not known
  26. # Hermes' tools.terminal.security_mode -> Codex permissions profile id.
  27. # Missing config -> workspace-write (Codex's own default).
  28. _HERMES_TO_CODEX_PERMISSION_PROFILE = {
  29. "auto": "workspace-write", "approval-required": "read-only-with-approval",
  30. "unrestricted": "full-access", "yolo": "full-access", # yolo: backstop alias used by some skills/tests
  31. }
  32. @dataclass
  33. class TurnResult:
  34. """Result of one user→assistant→tool turn through the codex app-server."""
  35. final_text: str = ""
  36. projected_messages: list[dict] = field(default_factory=list)
  37. tool_iterations: int = 0
  38. interrupted: bool = False
  39. error: Optional[str] = None # non-recoverable turn error
  40. turn_id: Optional[str] = None
  41. thread_id: Optional[str] = None
  42. # Exact turn/start text distinguishes the input echo from a new user event.
  43. submitted_user_text: Optional[str] = None
  44. token_usage_last: Optional[dict[str, Any]] = None
  45. model_context_window: Optional[int] = None
  46. compacted: bool = False
  47. # Codex likely wedged (turn timeout, dead subprocess, token refresh failure): caller respawns next turn.
  48. should_retire: bool = False
  49. # Some codex versions stream ``<turn_aborted>`` as raw agentMessage text when an
  50. # interrupt/upstream error tears the turn down without emitting turn/completed.
  51. _TURN_ABORTED_MARKERS = ("<turn_aborted>", "<turn_aborted/>")
  52. def _first_scope_id(*lookups: tuple[Any, str, str]) -> Any:
  53. """``src.get(a) or src.get(b)`` over successive dict sources until one is not None."""
  54. for src, primary, fallback in lookups:
  55. if isinstance(src, dict):
  56. observed = src.get(primary) or src.get(fallback)
  57. if observed is not None:
  58. return observed
  59. return None
  60. def _notification_scope_ids(note: dict) -> tuple[Optional[str], Optional[str]]:
  61. """Extract the thread/turn identity carried by a notification (top-level, then turn/item)."""
  62. params = (note.get("params") or {}) if isinstance(note, dict) else None
  63. if not isinstance(params, dict):
  64. return None, None
  65. turn, item = params.get("turn") or {}, params.get("item") or {}
  66. return (
  67. _first_scope_id((params, "threadId", "thread_id"), (turn, "threadId", "thread_id"), (item, "threadId", "thread_id")),
  68. _first_scope_id((params, "turnId", "turn_id"), (turn, "id", "turnId"), (item, "turnId", "turn_id")),
  69. )
  70. def _notification_belongs_to_turn(note: dict, *, thread_id: Optional[str], turn_id: Optional[str]) -> bool:
  71. """Whether a multiplexed notification belongs to this turn.
  72. One connection can carry parent and hosted subagent threads; an explicitly
  73. foreign thread/turn event must not mutate this transcript. Unscoped
  74. notifications remain accepted for protocol compatibility.
  75. """
  76. if not isinstance(note, dict):
  77. return False
  78. observed = _notification_scope_ids(note)
  79. return not any(
  80. expected is not None and seen is not None and str(seen) != str(expected)
  81. for expected, seen in zip((thread_id, turn_id), observed)
  82. )
  83. _TEXT_PART_TYPES = frozenset({"text", "input_text"})
  84. _IMAGE_PART_TYPES = frozenset({"image", "image_url", "input_image"})
  85. _IMAGE_URL_SCHEMES = ("data:", "http://", "https://")
  86. def _image_part_to_turn_input(item: dict) -> Optional[dict]:
  87. """Map one Hermes image part onto the app-server ``UserInput`` shape.
  88. ``turn/start`` accepts ``{type: image, url}`` (data:/http URLs) and ``{type: localImage, path}``
  89. natively (protocol schema ``v2/UserInput``), so nothing here is flattened into a text marker.
  90. """
  91. ref = item.get("image_url") or item.get("url") or item.get("path") or item.get("image")
  92. if isinstance(ref, dict):
  93. ref = ref.get("url") or ref.get("path")
  94. ref = (ref or "").strip() if isinstance(ref, str) else ""
  95. if not ref:
  96. return None
  97. if ref.startswith(_IMAGE_URL_SCHEMES):
  98. return {"type": "image", "url": ref}
  99. ref = ref.removeprefix("file://")
  100. return {"type": "localImage", "path": ref}
  101. def _build_turn_input(user_input: Any) -> tuple[list[dict], str]:
  102. """Build the ``turn/start`` ``input`` list plus the text the wire will echo back.
  103. Text parts stay text; image parts ride natively (#51053 — a text marker in their place left the
  104. model blind to the attachment). Returns ``(input_items, submitted_text)``.
  105. """
  106. if isinstance(user_input, str):
  107. return [{"type": "text", "text": user_input}], user_input
  108. if not isinstance(user_input, list):
  109. text = "" if user_input is None else str(user_input)
  110. return [{"type": "text", "text": text}], text
  111. texts: list[str] = []
  112. images: list[dict] = []
  113. for item in user_input:
  114. if not isinstance(item, dict):
  115. if item.strip() if isinstance(item, str) else item is not None:
  116. texts.append(str(item))
  117. elif item.get("type") in _TEXT_PART_TYPES:
  118. texts.append(str(item.get("text") or item.get("content") or ""))
  119. elif item.get("type") in _IMAGE_PART_TYPES:
  120. mapped = _image_part_to_turn_input(item)
  121. if mapped is not None:
  122. images.append(mapped)
  123. text = "\n\n".join(t for t in texts if t).strip()
  124. if not text and images:
  125. text = "What do you see in this image?"
  126. items: list[dict] = [{"type": "text", "text": text}] if text or not images else []
  127. return items + images, text
  128. # Strong credential-failure signals: trusted whether they appear in the primary
  129. # JSON-RPC error or in ambient app-server stderr.
  130. _OAUTH_REFRESH_FAILURE_HINTS = (
  131. "invalid_grant", "invalid grant", "refresh token", "refresh_token", "token refresh", "token_refresh",
  132. "token has expired", "expired_token", "token_expired", "expired token", "not authenticated", "unauthenticated",
  133. "re-authenticate", "reauthenticate", "please log in", "please login", "no auth profile",
  134. )
  135. # Generic auth words are authoritative only in the primary error. codex writes
  136. # independent ChatGPT plugin prewarm failures ("HTTP 401 Unauthorized") to stderr,
  137. # so there they must not mask an unrelated RPC error or timeout (#75167).
  138. _PRIMARY_ONLY_OAUTH_HINTS = ("401 unauthorized", "unauthorized", "oauth", "auth profile")
  139. _OAUTH_REAUTH_HINT = (
  140. "Codex authentication failed — your ChatGPT/Codex login looks expired or invalid. Run `codex login` to refresh, "
  141. "then retry. (Fall back to default runtime with `/codex-runtime auto` if the issue persists.)"
  142. )
  143. def _classify_oauth_failure(primary: str = "", *, stderr: str = "") -> Optional[str]:
  144. """Re-auth hint when ``primary`` (the operation's own error) or ``stderr`` proves the codex login is broken."""
  145. primary_l = (primary or "").lower()
  146. stderr_l = (stderr or "").lower()
  147. if any(n in primary_l for n in _OAUTH_REFRESH_FAILURE_HINTS + _PRIMARY_ONLY_OAUTH_HINTS):
  148. return _OAUTH_REAUTH_HINT
  149. return _OAUTH_REAUTH_HINT if any(n in stderr_l for n in _OAUTH_REFRESH_FAILURE_HINTS) else None
  150. @dataclass
  151. class _ServerRequestRouting:
  152. """Default approval policies when no interactive approval_callback is wired in (tests, cron)."""
  153. auto_approve_exec: bool = False
  154. auto_approve_apply_patch: bool = False
  155. class CodexThreadResumeError(CodexAppServerError):
  156. """``thread/resume`` did not hand back the stored thread (unknown/garbage id, rollout still locked by
  157. a killed app-server, or codex answered with a different thread). The caller decides the policy."""
  158. def __init__(self, thread_id: str, detail: str) -> None:
  159. super().__init__(code=-32600, message=f"codex thread {thread_id[:8]} could not be resumed: {detail}")
  160. self.thread_id = thread_id
  161. def _extract_thread_id(result: dict) -> Optional[str]:
  162. """Different codex versions serialize the id under thread.id / sessionId / threadId."""
  163. thread_obj = result.get("thread") or {}
  164. return thread_obj.get("id") or thread_obj.get("sessionId") or result.get("sessionId") or result.get("threadId")
  165. class CodexAppServerSession:
  166. """One Codex thread per Hermes session, lifetime owned by AIAgent. Not thread-safe: one caller at a time."""
  167. def __init__(
  168. self, *, cwd: Optional[str] = None, codex_bin: str = "codex",
  169. codex_home: Optional[str] = None, permission_profile: Optional[str] = None,
  170. approval_callback: Optional[Callable[..., str]] = None,
  171. on_event: Optional[Callable[[dict], None]] = None,
  172. request_routing: Optional[_ServerRequestRouting] = None,
  173. client_factory: Optional[Callable[..., CodexAppServerClient]] = None,
  174. model: Optional[str] = None, model_provider: Optional[str] = None,
  175. developer_instructions: Optional[str] = None, resume_thread_id: Optional[str] = None,
  176. history_seed: Optional[str] = None,
  177. ) -> None:
  178. self._cwd = cwd or os.getcwd()
  179. self._codex_bin = codex_bin
  180. self._codex_home = codex_home
  181. # A codex thread id persisted by an earlier process for this Hermes session: the first
  182. # ``ensure_started`` issues ``thread/resume`` for it instead of ``thread/start``.
  183. self._resume_thread_id = resume_thread_id
  184. # ``thread/start.model``: the Hermes-selected slug, for every provider. ``.modelProvider``: a named
  185. # custom provider's id in codex's own ``[model_providers.<id>]`` table; only the id travels, codex
  186. # reads base_url/env_key itself.
  187. self._model = (model or "").strip() or None
  188. self._model_provider = (model_provider or "").strip() or None
  189. # The tier this thread last received on turn/start; None = codex's own default, never overridden.
  190. self._service_tier_sent: Any = None
  191. # Hermes' composed system prompt (SOUL.md, memory, channel overrides). Sent ONCE per thread as
  192. # ``thread/start.developerInstructions``: codex keeps its own base instructions (tool guidance) and
  193. # inserts this as the first developer message of every model request. ``baseInstructions`` would
  194. # REPLACE codex's base and ``instructions`` is accepted but ignored (verified against codex 0.147).
  195. self._developer_instructions = developer_instructions
  196. # Hermes' prior transcript, appended to developerInstructions ONLY when a thread is started from
  197. # scratch: a resumed thread already holds the conversation (agent/codex_runtime_history_seed.py).
  198. self._history_seed = history_seed
  199. self._permission_profile = permission_profile or _HERMES_TO_CODEX_PERMISSION_PROFILE.get(
  200. os.environ.get("HERMES_TERMINAL_SECURITY_MODE", "auto"), "workspace-write"
  201. )
  202. self._approval_callback = approval_callback
  203. self._on_event = on_event # Display hook (kawaii spinner ticks etc.)
  204. self._routing = request_routing or _ServerRequestRouting()
  205. self._client_factory = client_factory or CodexAppServerClient
  206. self._client: Optional[CodexAppServerClient] = None
  207. self._thread_id: Optional[str] = None
  208. self._interrupt_event = threading.Event()
  209. self._active_turn_id: Optional[str] = None
  210. self._active_turn_lock = threading.Lock()
  211. # In-progress fileChange items by id (item/started -> item/completed):
  212. # approval params don't carry the changeset, so this feeds the prompt summary.
  213. self._pending_file_changes: dict[str, str] = {}
  214. self._closed = False
  215. def ensure_started(self) -> str:
  216. """Spawn, handshake, and ``thread/start`` (or ``thread/resume`` for a stored id); idempotent, returns
  217. the codex thread id. A failed resume raises :class:`CodexThreadResumeError` once; the next call
  218. starts a fresh thread on the same handshaken client."""
  219. if self._thread_id is not None:
  220. return self._thread_id
  221. if self._client is None:
  222. self._client = self._client_factory(codex_bin=self._codex_bin, codex_home=self._codex_home)
  223. self._client.initialize(client_name="hermes", client_title="Hermes Agent", client_version=_get_hermes_version())
  224. # Permissions are NOT sent on thread/start: codex gates ``thread/start.permissions``
  225. # behind experimentalApi + a matching ``[permissions]`` table in ~/.codex/config.toml.
  226. # Hermes supplies the agent identity through its own system prompt; ``personality: "none"`` strips
  227. # codex's built-in "# Personality" section from the base instructions so it cannot compete (#72104).
  228. params: dict[str, Any] = {"cwd": self._cwd, "personality": "none"}
  229. if self._developer_instructions and self._developer_instructions.strip():
  230. params["developerInstructions"] = self._developer_instructions
  231. if self._model_provider:
  232. params["modelProvider"] = self._model_provider
  233. if self._model:
  234. params["model"] = self._model
  235. if self._resume_thread_id:
  236. wanted, self._resume_thread_id = self._resume_thread_id, None # one attempt per stored id
  237. thread_id = self._resume_thread(wanted, params)
  238. # A resumed thread may carry a tier an earlier process selected (CLI ``/fast`` rebuilds the agent),
  239. # so its first turn sends Hermes' tier even when that is a clearing null.
  240. self._service_tier_sent = _TIER_UNKNOWN
  241. logger.info("codex app-server thread resumed: id=%s cwd=%s", thread_id[:8], self._cwd)
  242. else:
  243. if self._history_seed:
  244. params["developerInstructions"] = "\n\n".join(
  245. part for part in (params.get("developerInstructions"), self._history_seed) if part)
  246. result = self._client.request("thread/start", params, timeout=15)
  247. thread_id = _extract_thread_id(result)
  248. if not thread_id:
  249. raise CodexAppServerError(
  250. code=-32603, message=f"codex thread/start returned no thread id (payload keys: {sorted(result.keys())})",
  251. )
  252. self._service_tier_sent = None # a new thread runs codex's own configured tier
  253. logger.info("codex app-server thread started: id=%s profile=%s cwd=%s", thread_id[:8], self._permission_profile, self._cwd)
  254. self._thread_id = thread_id
  255. return thread_id
  256. def _resume_thread(self, wanted: str, params: dict[str, Any]) -> str:
  257. """``thread/resume`` for the stored id; the same thread/start params ride along so the resumed thread
  258. carries the CURRENT prompt composition and provider (accepted by the resume schema, codex 0.147)."""
  259. assert self._client is not None
  260. try:
  261. result = self._client.request("thread/resume", {"threadId": wanted, **params}, timeout=15)
  262. except CodexAppServerError as exc:
  263. raise CodexThreadResumeError(wanted, exc.message) from exc
  264. thread_id = _extract_thread_id(result)
  265. if thread_id != wanted:
  266. raise CodexThreadResumeError(wanted, f"app-server answered with thread {str(thread_id)[:8]!r}")
  267. return wanted
  268. def close(self) -> None:
  269. if self._closed:
  270. return
  271. self._closed = True
  272. with self._active_turn_lock:
  273. self._active_turn_id = None
  274. if self._client is not None:
  275. with contextlib.suppress(Exception): # pragma: no cover - best-effort cleanup
  276. self._client.close()
  277. self._client = None
  278. self._thread_id = None
  279. def request_interrupt(self) -> None:
  280. """Idempotent: signal the active turn loop to issue turn/interrupt and unwind."""
  281. self._interrupt_event.set()
  282. def request_steer(self, text: str) -> bool:
  283. """Append user guidance to the active Codex turn via ``turn/steer``."""
  284. cleaned = str(text or "").strip()
  285. if not cleaned:
  286. return False
  287. with self._active_turn_lock:
  288. turn_id, thread_id, client = self._active_turn_id, self._thread_id, self._client
  289. if not turn_id or not thread_id or client is None:
  290. return False
  291. try:
  292. response = client.request(
  293. "turn/steer",
  294. {"threadId": thread_id, "input": [{"type": "text", "text": cleaned}], "expectedTurnId": turn_id}, timeout=10,
  295. )
  296. except (CodexAppServerError, TimeoutError):
  297. logger.debug("turn/steer rejected for active Codex turn", exc_info=True)
  298. return False
  299. accepted_turn_id = response.get("turnId") if isinstance(response, dict) else None
  300. return accepted_turn_id in {None, turn_id}
  301. def _format_error_with_stderr(self, prefix: str, exc: Any = "", *, tail_lines: int = _STDERR_TAIL_LINES) -> str:
  302. """User-facing error string plus the force-redacted stderr tail (keeps secrets out of chat output)."""
  303. exc_str = "" if exc is None else str(exc)
  304. base = f"{prefix}: {exc_str}" if exc_str else prefix
  305. try:
  306. tail = self._client.stderr_tail(tail_lines) if self._client is not None else []
  307. except Exception: # pragma: no cover - diagnostic best-effort
  308. return base
  309. joined = "\n".join(line.rstrip() for line in tail if line)
  310. if not joined.strip():
  311. return base
  312. return f"{base}\ncodex stderr (last {len(tail)} lines):\n{redact_sensitive_text(joined, force=True)}"
  313. def _stderr_blob(self, n: int) -> str:
  314. client = self._client
  315. return "" if client is None else "\n".join(client.stderr_tail(n))
  316. @staticmethod
  317. def _retire(result: TurnResult, error: str) -> None:
  318. """Record a terminal error and flag the session for respawn on the next turn."""
  319. result.error = error
  320. result.should_retire = True
  321. def _set_classified_error(self, result: TurnResult, prefix: str, classify_text: str, detail: Any) -> None:
  322. """OAuth failures -> re-auth hint AND retire (token store broken though JSON-RPC is fine); else stderr tail."""
  323. hint = _classify_oauth_failure(classify_text, stderr=self._stderr_blob(40))
  324. if hint is not None:
  325. self._retire(result, hint)
  326. else:
  327. result.error = self._format_error_with_stderr(prefix, detail)
  328. def _start_for(self, result: TurnResult) -> bool:
  329. """ensure_started(); startup failures become a retiring TurnResult.error instead of raw exceptions."""
  330. try:
  331. self.ensure_started()
  332. except (CodexAppServerError, TimeoutError) as exc:
  333. self._retire(result, self._format_error_with_stderr("codex app-server startup failed", exc))
  334. return False
  335. assert self._client is not None and self._thread_id is not None
  336. result.thread_id = self._thread_id
  337. return True
  338. def _request_for(self, result: TurnResult, method: str, params: dict, label: str) -> Optional[dict]:
  339. """Issue ``method``; on failure fill ``result.error`` and return None. A timeout always retires."""
  340. try:
  341. return self._client.request(method, params, timeout=10)
  342. except CodexAppServerTransportError as exc:
  343. self._retire(result, self._format_error_with_stderr(f"{label} failed", exc))
  344. except CodexAppServerError as exc:
  345. self._set_classified_error(result, f"{label} failed", exc.message, exc)
  346. except TimeoutError as exc:
  347. hint = _classify_oauth_failure(stderr=self._stderr_blob(40))
  348. self._retire(result, hint or self._format_error_with_stderr(f"{label} timed out", exc))
  349. return None
  350. def _subprocess_died(self, result: TurnResult, client: Optional[CodexAppServerClient]) -> bool:
  351. """Bail out early (rather than waiting on the deadline) when codex exited or close() ran.
  352. ``client`` is the loop's snapshot: close() on another thread nulls ``self._client``
  353. mid-turn (session expiry), which must end the turn, not raise AttributeError. A
  354. ``None`` snapshot means close() already landed before the loop started.
  355. """
  356. if client is None or self._closed or self._client is not client:
  357. result.interrupted = True
  358. self._retire(result, "codex app-server session closed while the turn was in flight")
  359. return True
  360. if client.is_alive():
  361. return False
  362. hint = _classify_oauth_failure(stderr=self._stderr_blob(60))
  363. self._retire(result, hint or self._format_error_with_stderr("codex app-server subprocess exited unexpectedly", tail_lines=20))
  364. return True
  365. def _absorb_notification(
  366. self, result: TurnResult, projector: CodexEventProjector, note: dict
  367. ) -> tuple[ProjectionResult, bool]:
  368. """Fan one in-scope notification out to display, accounting, file-change tracking and the projector.
  369. Returns (projection, aborted); aborted = agent text carried a terminal ``<turn_aborted>`` marker.
  370. """
  371. if self._on_event is not None:
  372. try:
  373. self._on_event(note)
  374. except Exception: # pragma: no cover - display callback
  375. logger.debug("on_event callback raised", exc_info=True)
  376. _apply_accounting_notification(result, note)
  377. self._track_pending_file_change(note)
  378. projection = projector.project(note)
  379. if projection.messages:
  380. result.projected_messages.extend(projection.messages)
  381. if projection.is_tool_iteration:
  382. result.tool_iterations += 1
  383. aborted = False
  384. if projection.final_text is not None:
  385. # Multiple agentMessage items per turn: the last one is canonical.
  386. result.final_text = projection.final_text
  387. aborted = _has_turn_aborted_marker(projection.final_text)
  388. if aborted:
  389. result.interrupted = True
  390. result.error = result.error or "codex reported turn_aborted"
  391. return projection, aborted
  392. def run_turn(
  393. self, user_input: Any, *, model: Optional[str] = None, reasoning_effort: Optional[str] = None,
  394. service_tier: Optional[str] = None, turn_timeout: float = 600.0,
  395. notification_poll_timeout: float = 0.25, post_tool_quiet_timeout: float = 90.0,
  396. ) -> TurnResult:
  397. """Send a user message and block until turn/completed, bridging approvals and projecting items.
  398. model / reasoning_effort: sent on ``turn/start`` (codex applies them to this and later turns), so an
  399. in-place ``/model`` or effort change reaches a thread that was started with other settings.
  400. service_tier: sent only when it differs from what this thread last received, so codex's own configured
  401. tier is left alone until Hermes selects one, and ``None`` after a selection clears it.
  402. post_tool_quiet_timeout: if codex emits a tool completion and then goes quiet for this many seconds
  403. without emitting another item or `turn/completed`, log a warning (once per tool result) and keep
  404. waiting. Wire silence is not evidence of a wedged process: after a large tool output codex can
  405. reason for minutes without emitting a single event while the app-server still answers RPCs
  406. (#112928). Only subprocess death or ``turn_timeout`` retires the session.
  407. """
  408. result = TurnResult()
  409. if self._start_for(result):
  410. # Do not clear first: a hard stop arriving during ensure_started() must
  411. # be honored before launching a Codex turn.
  412. if self._interrupt_event.is_set():
  413. result.interrupted = True
  414. else:
  415. input_items, result.submitted_user_text = _build_turn_input(user_input)
  416. params: dict[str, Any] = {"threadId": self._thread_id, "input": input_items}
  417. if model:
  418. params["model"] = model
  419. if reasoning_effort is not None:
  420. params["effort"] = reasoning_effort
  421. if service_tier != self._service_tier_sent:
  422. params["serviceTier"] = service_tier
  423. ts = self._request_for(result, "turn/start", params, "turn/start")
  424. if ts is not None:
  425. self._service_tier_sent = service_tier
  426. self._run_started_turn(result, ts, turn_timeout, notification_poll_timeout, post_tool_quiet_timeout)
  427. self._interrupt_event.clear()
  428. return result
  429. def _run_started_turn(
  430. self, result: TurnResult, ts: dict, turn_timeout: float, notification_poll_timeout: float,
  431. post_tool_quiet_timeout: float,
  432. ) -> None:
  433. """Drive an accepted ``turn/start`` to completion: quiet warning, approvals, projection."""
  434. projector = CodexEventProjector()
  435. client = self._client
  436. result.turn_id = (ts.get("turn") or {}).get("id")
  437. with self._active_turn_lock:
  438. self._active_turn_id = result.turn_id
  439. # Post-tool quiet timer: armed on each tool completion, cleared by any other activity.
  440. # Observability only — it never interrupts or retires (see run_turn docstring).
  441. last_tool_completion_at: Optional[float] = None
  442. def warn_if_quiet() -> bool:
  443. nonlocal last_tool_completion_at
  444. if last_tool_completion_at is None or (time.monotonic() - last_tool_completion_at) <= post_tool_quiet_timeout:
  445. return False
  446. last_tool_completion_at = None
  447. logger.warning(
  448. "codex has emitted no events for %.0fs after a tool result; still waiting (turn deadline %.0fs)",
  449. post_tool_quiet_timeout, turn_timeout,
  450. )
  451. return False
  452. def on_server_request(sreq: dict) -> bool:
  453. nonlocal last_tool_completion_at
  454. # Drain pending notifications first (bounded) so _pending_file_changes is
  455. # current for the approval decision and display events still reach on_event.
  456. turn_complete = False
  457. for _ in range(8):
  458. pending = client.take_notification(timeout=0)
  459. if pending is None:
  460. break
  461. if not _notification_belongs_to_turn(pending, thread_id=self._thread_id, turn_id=result.turn_id):
  462. logger.debug("ignoring foreign codex notification while draining server request: method=%s", pending.get("method"))
  463. continue
  464. proj, aborted = self._absorb_notification(result, projector, pending)
  465. if proj.is_tool_iteration:
  466. last_tool_completion_at = time.monotonic()
  467. turn_complete = turn_complete or aborted
  468. self._handle_server_request(sreq)
  469. # An approval round-trip is live signal — don't let it trip the quiet warning.
  470. last_tool_completion_at = None
  471. return turn_complete
  472. def on_note(note: dict, method: str) -> bool:
  473. nonlocal last_tool_completion_at
  474. projection, aborted = self._absorb_notification(result, projector, note)
  475. if projection.is_tool_iteration:
  476. last_tool_completion_at = time.monotonic()
  477. elif projection.messages or projection.final_text is not None:
  478. last_tool_completion_at = None
  479. if method != "turn/completed":
  480. return aborted
  481. turn_obj = (note.get("params") or {}).get("turn") or {}
  482. turn_status = turn_obj.get("status")
  483. if turn_status and turn_status not in {"completed", "interrupted"} and turn_obj.get("error"):
  484. err_msg = _format_responses_error(turn_obj["error"], str(turn_status))
  485. self._set_classified_error(result, f"turn ended status={turn_status}", err_msg, err_msg)
  486. return True
  487. self._drive_turn(
  488. result, turn_timeout=turn_timeout, notification_poll_timeout=notification_poll_timeout,
  489. timeout_label="turn", before_poll=warn_if_quiet, on_server_request=on_server_request,
  490. on_note=on_note, accept_final_text_at_deadline=True,
  491. )
  492. with self._active_turn_lock:
  493. self._active_turn_id = None
  494. def _drive_turn(
  495. self, result: TurnResult, *, turn_timeout: float, notification_poll_timeout: float,
  496. timeout_label: str, on_server_request: Callable[[dict], bool],
  497. on_note: Callable[[dict, str], bool], before_poll: Optional[Callable[[], bool]] = None,
  498. pre_scope_filter: Optional[Callable[[dict, str], bool]] = None,
  499. accept_final_text_at_deadline: bool = False,
  500. ) -> None:
  501. """Shared poll loop for run_turn / compact_thread until turn/completed or deadline.
  502. Per iteration: interrupt -> subprocess death -> ``before_poll`` (quiet warning) ->
  503. server requests (answered first so codex isn't blocked) -> one notification,
  504. filtered by ``pre_scope_filter`` then turn scope, handed to ``on_note``. Hooks
  505. return True to complete the turn. Deadline without completion interrupts and
  506. retires the session.
  507. """
  508. deadline = time.monotonic() + turn_timeout
  509. turn_complete = False
  510. client = self._client
  511. while time.monotonic() < deadline and not turn_complete:
  512. if self._interrupt_event.is_set():
  513. self._issue_interrupt(result.turn_id)
  514. result.interrupted = True
  515. break
  516. if self._subprocess_died(result, client) or client is None: # `is None` narrows only; already retired
  517. break
  518. if before_poll is not None and before_poll():
  519. break
  520. sreq = client.take_server_request(timeout=0)
  521. if sreq is not None:
  522. try:
  523. turn_complete = on_server_request(sreq)
  524. except CodexAppServerTransportError as exc:
  525. self._retire(result, self._format_error_with_stderr("codex app-server request response failed", exc))
  526. break
  527. continue
  528. note = client.take_notification(timeout=notification_poll_timeout)
  529. if note is None:
  530. continue
  531. method = note.get("method", "")
  532. if pre_scope_filter is not None and not pre_scope_filter(note, method):
  533. continue
  534. if not _notification_belongs_to_turn(note, thread_id=self._thread_id, turn_id=result.turn_id):
  535. logger.debug("ignoring foreign codex notification: method=%s", method)
  536. continue
  537. turn_complete = on_note(note, method)
  538. if accept_final_text_at_deadline and not turn_complete and not result.interrupted and result.final_text and result.error is None:
  539. logger.warning(
  540. "codex app-server turn reached deadline after a completed assistant message but before "
  541. "turn/completed; accepting the assistant text as the terminal response"
  542. )
  543. turn_complete = True
  544. if not turn_complete and not result.interrupted:
  545. self._issue_interrupt(result.turn_id)
  546. result.interrupted = True
  547. if not result.error:
  548. result.error = self._format_error_with_stderr(f"{timeout_label} timed out after {turn_timeout}s")
  549. result.should_retire = True
  550. def compact_thread(
  551. self, *, turn_timeout: float = 600.0, notification_poll_timeout: float = 0.25
  552. ) -> TurnResult:
  553. """Trigger Codex-native history compaction for the current thread.
  554. ``thread/compact/start`` returns immediately with no turn id; progress streams
  555. as normal turn/item notifications, so wait for the matching ``turn/completed``.
  556. """
  557. result = TurnResult()
  558. if not self._start_for(result):
  559. return result
  560. self._interrupt_event.clear()
  561. projector = CodexEventProjector()
  562. if self._request_for(result, "thread/compact/start", {"threadId": self._thread_id}, "thread/compact/start") is None:
  563. return result
  564. def pre_scope_filter(note: dict, method: str) -> bool:
  565. if result.turn_id is not None:
  566. return True
  567. observed_thread_id, observed_turn_id = _notification_scope_ids(note)
  568. if method == "turn/started":
  569. if observed_thread_id is not None and str(observed_thread_id) != str(self._thread_id):
  570. logger.debug("ignoring foreign compact turn/started: thread=%s", observed_thread_id)
  571. return False
  572. if observed_turn_id is None:
  573. logger.debug("ignoring compact turn/started without a turn id")
  574. return False
  575. result.turn_id = str(observed_turn_id)
  576. elif observed_turn_id is not None or method in {"item/completed", "turn/completed"}:
  577. # Before the new turn/started, terminal/projectable events are stale or unattributable.
  578. logger.debug("ignoring codex notification before compact turn start: method=%s", method)
  579. return False
  580. return True
  581. def on_note(note: dict, method: str) -> bool:
  582. _, aborted = self._absorb_notification(result, projector, note)
  583. if method not in {"turn/started", "turn/completed"}:
  584. return aborted
  585. turn_obj = (note.get("params") or {}).get("turn") or {}
  586. result.turn_id = turn_obj.get("id") or result.turn_id
  587. if method == "turn/started":
  588. return aborted
  589. turn_status = turn_obj.get("status")
  590. if turn_status == "interrupted":
  591. result.interrupted = True
  592. result.error = result.error or "compact turn interrupted"
  593. elif turn_status and turn_status != "completed":
  594. err_msg = _format_responses_error(turn_obj.get("error"), str(turn_status))
  595. self._set_classified_error(result, f"compact turn ended status={turn_status}", err_msg, err_msg)
  596. return True
  597. def on_server_request(sreq: dict) -> bool:
  598. self._handle_server_request(sreq)
  599. return False
  600. self._drive_turn(
  601. result, turn_timeout=turn_timeout, notification_poll_timeout=notification_poll_timeout,
  602. timeout_label="compact turn", on_server_request=on_server_request, on_note=on_note,
  603. pre_scope_filter=pre_scope_filter,
  604. )
  605. return result
  606. def _issue_interrupt(self, turn_id: Optional[str]) -> None:
  607. client = self._client
  608. if client is None or self._thread_id is None or turn_id is None:
  609. return
  610. try:
  611. client.request("turn/interrupt", {"threadId": self._thread_id, "turnId": turn_id}, timeout=5)
  612. except CodexAppServerError as exc:
  613. # "no active turn to interrupt" is fine — already done.
  614. logger.debug("turn/interrupt non-fatal: %s", exc)
  615. except TimeoutError:
  616. logger.warning("turn/interrupt timed out")
  617. def _handle_server_request(self, req: dict) -> None:
  618. """Answer a codex server request (approval / elicitation) via Hermes' approval flow.
  619. Permission escalations are always declined (the user chose their profile in
  620. ~/.codex/config.toml); unknown methods get a JSON-RPC error so codex doesn't hang.
  621. """
  622. client = self._client
  623. if client is None:
  624. return
  625. method = req.get("method", "")
  626. rid = req.get("id")
  627. params = req.get("params") or {}
  628. handler = self._SERVER_REQUEST_HANDLERS.get(method)
  629. if handler is None:
  630. logger.warning("Unknown codex server request: %s", method)
  631. client.respond_error(rid, code=-32601, message=f"Unsupported method: {method}")
  632. return
  633. client.respond(rid, handler(self, params))
  634. def _respond_elicitation(self, params: dict) -> dict:
  635. """MCP elicitation: auto-accept our own hermes-tools server (opted in by enabling the runtime;
  636. exposes nothing codex's shell can't do); decline others so the user opts in via codex's own flow."""
  637. action = "accept" if (params.get("serverName") or "") == HERMES_TOOLS_MCP_SERVER_NAME else "decline"
  638. return {"action": action, "content": None, "_meta": None}
  639. _SERVER_REQUEST_HANDLERS: dict[str, Callable[..., dict]] = {
  640. "item/commandExecution/requestApproval": lambda self, p: {"decision": self._decide_exec_approval(p)},
  641. "item/fileChange/requestApproval": lambda self, p: {"decision": self._decide_apply_patch_approval(p)},
  642. "item/permissions/requestApproval": lambda self, p: {"decision": "decline"},
  643. "mcpServer/elicitation/request": _respond_elicitation,
  644. }
  645. def _run_approval_callback(self, auto_approve: bool, prompt: Callable[[], tuple[str, str]], log_label: str) -> str:
  646. """Protocol routing only: auto-approve, fail-closed without a callback, else ask via ``prompt()``.
  647. Approval mode/timeout resolution lives upstream (codex_runtime.py derives the
  648. auto flags; the callback runs the shared gate). Do not re-read config here.
  649. """
  650. if auto_approve:
  651. return "accept"
  652. if self._approval_callback is None:
  653. return "decline"
  654. command, description = prompt()
  655. try:
  656. choice = self._approval_callback(command, description, allow_permanent=False)
  657. return _approval_choice_to_codex_decision(choice)
  658. except Exception:
  659. logger.exception("approval_callback raised on %s", log_label)
  660. return "decline"
  661. def _decide_exec_approval(self, params: dict) -> str:
  662. def prompt() -> tuple[str, str]:
  663. # ``cwd`` is Optional on codex's side; fall back so the prompt is never empty.
  664. description = f"Codex requests exec in {params.get('cwd') or self._cwd or '<unknown>'}"
  665. if params.get("reason"):
  666. description += f" — {params['reason']}"
  667. return params.get("command") or "", description
  668. return self._run_approval_callback(self._routing.auto_approve_exec, prompt, "exec request")
  669. def _decide_apply_patch_approval(self, params: dict) -> str:
  670. def prompt() -> tuple[str, str]:
  671. # Params carry reason + grantRoot only; the changeset comes from _track_pending_file_change.
  672. reason, grant_root = params.get("reason"), params.get("grantRoot")
  673. change_summary = self._pending_file_changes.get(params.get("itemId") or "") or None
  674. parts = [p for p in (reason, change_summary, grant_root and f"grants write to {grant_root}") if p]
  675. detail = change_summary or reason
  676. return (
  677. f"apply_patch: {detail}" if detail else "apply_patch",
  678. "; ".join(parts) if parts else "Codex requests to apply a patch",
  679. )
  680. return self._run_approval_callback(self._routing.auto_approve_apply_patch, prompt, "apply_patch")
  681. def _track_pending_file_change(self, note: dict) -> None:
  682. """Track fileChange items (item/started -> item/completed) so the apply_patch prompt can show the changeset."""
  683. method = note.get("method", "")
  684. item = (note.get("params") or {}).get("item") or {}
  685. item_id = item.get("id") or ""
  686. if item.get("type") != "fileChange" or not item_id:
  687. return
  688. if method == "item/completed":
  689. self._pending_file_changes.pop(item_id, None)
  690. elif method == "item/started":
  691. self._pending_file_changes[item_id] = _summarize_file_changes(item.get("changes") or [])
  692. def _summarize_file_changes(raw_changes: list) -> str:
  693. """One-line ``"<n> add, <m> update: a.py, b.py, +k more"`` summary of a fileChange item's changes."""
  694. if not raw_changes:
  695. return "1 change pending"
  696. changes = [ch for ch in raw_changes if isinstance(ch, dict)]
  697. kinds: dict[str, int] = {}
  698. for ch in changes:
  699. kind = (ch.get("kind") or {}).get("type") or "update"
  700. kinds[kind] = kinds.get(kind, 0) + 1
  701. paths: list[str] = [ch["path"] for ch in changes if ch.get("path")]
  702. counts = ", ".join(f"{n} {k}" for k, n in sorted(kinds.items()))
  703. preview = ", ".join(paths[:3])
  704. if len(paths) > 3:
  705. preview += f", +{len(paths) - 3} more"
  706. return f"{counts}: {preview}" if preview else counts
  707. def _apply_accounting_notification(result: TurnResult, note: dict) -> None:
  708. """Capture token usage (thread/tokenUsage/updated, not turn/completed) and compaction
  709. boundaries (a contextCompaction item on recent builds, deprecated thread/compacted on older)."""
  710. if not isinstance(note, dict):
  711. return
  712. method = note.get("method") or ""
  713. params = note.get("params") or {}
  714. if not isinstance(params, dict):
  715. return
  716. if method == "thread/tokenUsage/updated":
  717. token_usage = params.get("tokenUsage") or {}
  718. if isinstance(token_usage, dict):
  719. last, window = token_usage.get("last"), token_usage.get("modelContextWindow")
  720. if isinstance(last, dict):
  721. result.token_usage_last = dict(last)
  722. if isinstance(window, int) and window > 0:
  723. result.model_context_window = window
  724. return
  725. item = params.get("item") if method in {"item/started", "item/completed"} else None
  726. if method == "thread/compacted" or (isinstance(item, dict) and item.get("type") == "contextCompaction"):
  727. result.compacted = True
  728. result.thread_id = params.get("threadId") or result.thread_id
  729. result.turn_id = params.get("turnId") or result.turn_id
  730. # Hermes approval choice -> codex decision (app-server-protocol v2). "deny" and
  731. # "timeout" both decline — codex has no "prompt expired" wire value.
  732. _APPROVAL_CHOICE_TO_DECISION = {"once": "accept", "session": "acceptForSession", "always": "acceptForSession"}
  733. def _approval_choice_to_codex_decision(choice: str) -> str:
  734. """Map a Hermes approval choice onto codex's approval decision wire value."""
  735. return _APPROVAL_CHOICE_TO_DECISION.get(choice, "decline")
  736. def _has_turn_aborted_marker(text: str) -> bool:
  737. """True if ``text`` carries a raw ``<turn_aborted>`` marker (terminal without turn/completed)."""
  738. return bool(text) and any(marker in text for marker in _TURN_ABORTED_MARKERS)
  739. def _get_hermes_version() -> str:
  740. """Best-effort Hermes version string for codex's userAgent line."""
  741. try:
  742. from importlib.metadata import version
  743. return version("hermes-agent")
  744. except Exception: # pragma: no cover
  745. return "0.0.0"