codex_app_server.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. """Codex app-server JSON-RPC client (newline-delimited JSON-RPC 2.0 over stdio, codex 0.125+).
  2. ``initialize`` handshake, then ``thread/start`` + ``turn/start`` with streaming
  3. ``item/*`` notifications until ``turn/completed``. Wire-level speaker only —
  4. projection, approvals and transcript handling live in sibling modules.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import json
  9. import os
  10. import queue
  11. import re
  12. import subprocess
  13. import threading
  14. from dataclasses import dataclass
  15. from typing import Any, Optional
  16. from agent.deadline import kill_process_tree
  17. from agent.transports.hermes_tools_mcp_server import HERMES_TOOLS_MCP_SERVER_NAME
  18. from tools.environments.local import hermes_subprocess_env
  19. MIN_CODEX_VERSION = (0, 125, 0)
  20. @dataclass
  21. class CodexAppServerError(RuntimeError):
  22. """Raised on JSON-RPC errors from the app-server."""
  23. code: int
  24. message: str
  25. data: Optional[Any] = None
  26. def __str__(self) -> str: # pragma: no cover - trivial
  27. return f"codex app-server error {self.code}: {self.message}"
  28. class CodexAppServerTransportError(CodexAppServerError):
  29. """The JSON-RPC transport is gone: a write failed or close() drained the request.
  30. Distinct from server-reported errors so session boundaries can retire the
  31. session without swallowing unrelated ``RuntimeError`` programming defects.
  32. """
  33. def __str__(self) -> str: # pragma: no cover - trivial
  34. return self.message
  35. _TRANSPORT_LOST_CODE = -32000
  36. def _snapshot_descendants(pid: int) -> list[Any]:
  37. """psutil handles for ``pid``'s current descendants ([] when psutil is unavailable)."""
  38. try:
  39. import psutil
  40. return psutil.Process(int(pid)).children(recursive=True)
  41. except Exception:
  42. return []
  43. def _reap_snapshotted(descendants: list[Any]) -> None:
  44. """SIGTERM the snapshotted descendants (deepest first), then SIGKILL survivors after a
  45. bounded wait. psutil identity checks make a recycled PID a no-op."""
  46. if not descendants:
  47. return
  48. import psutil
  49. live = []
  50. for child in reversed(descendants):
  51. with contextlib.suppress(Exception):
  52. if child.is_running():
  53. child.terminate()
  54. live.append(child)
  55. try:
  56. _, alive = psutil.wait_procs(live, timeout=1.0)
  57. except Exception:
  58. alive = live
  59. for child in alive:
  60. with contextlib.suppress(Exception):
  61. child.kill()
  62. class CodexAppServerClient:
  63. """Minimal synchronous JSON-RPC 2.0 client for ``codex app-server`` over stdio.
  64. One reader thread routes replies to pending queues and notifications / server
  65. requests to queues; another captures stderr. Deliberately NOT async:
  66. AIAgent.run_conversation() is synchronous and cancels via ``turn/interrupt``.
  67. """
  68. def __init__(
  69. self, codex_bin: str = "codex", codex_home: Optional[str] = None,
  70. extra_args: Optional[list[str]] = None, env: Optional[dict[str, str]] = None,
  71. ) -> None:
  72. self._codex_bin = codex_bin
  73. # codex needs LLM provider creds but must not receive Tier-1 Hermes secrets (gateway/GitHub/infra tokens).
  74. # codex app-server is a model-driving CLI executor: it runs a model-chosen agentic loop that
  75. # executes shell commands, so it legitimately needs LLM provider credentials
  76. # (inherit_credentials=True) to authenticate against the model endpoint. But the previous
  77. # `os.environ.copy()` also handed it every Tier-1 Hermes secret — gateway bot tokens, GitHub auth,
  78. # Modal/Daytona infra tokens, the dashboard session token, AUXILIARY_* side-LLM keys,
  79. # GATEWAY_RELAY_* auth — none of which a coding subprocess has any use for. Route through the
  80. # centralized helper so Tier-1 + dynamic-internal secrets are always stripped while provider creds
  81. # still flow, matching the ACP client (#29157 sibling spawn-site gap).
  82. spawn_env = hermes_subprocess_env(inherit_credentials=True)
  83. if env:
  84. spawn_env.update(env)
  85. if codex_home:
  86. spawn_env["CODEX_HOME"] = codex_home
  87. cmd = [codex_bin, "app-server", *(extra_args or [])]
  88. from agent.delegation_context import (
  89. DELEGATED_CHILD_ENV_MARKER, KANBAN_ENV_KEYS,
  90. delegated_child_subprocess_env, is_dispatcher_owned_worker_context,
  91. )
  92. # Native shell children remain unowned. Only Hermes' managed MCP tool
  93. # endpoint acts for this worker; grant it scope via its existing per-server
  94. # environment (the entry the runtime migration registers), never by granting
  95. # the whole executor process ownership.
  96. owned_task = os.environ.get("HERMES_KANBAN_TASK") and is_dispatcher_owned_worker_context()
  97. if owned_task:
  98. for key in (*KANBAN_ENV_KEYS, "HERMES_KANBAN_DB", "HERMES_KANBAN_BOARD"):
  99. if key in os.environ:
  100. cmd += ["-c", f"mcp_servers.{HERMES_TOOLS_MCP_SERVER_NAME}.env.{key}={json.dumps(os.environ[key])}"]
  101. cmd += ["-c", f'mcp_servers.{HERMES_TOOLS_MCP_SERVER_NAME}.env.{DELEGATED_CHILD_ENV_MARKER}=""']
  102. spawn_env = delegated_child_subprocess_env(spawn_env)
  103. # Kanban workers must write handoff/status to the board DB outside the
  104. # workspace: keep the sandbox on, add the Kanban root as writable.
  105. if owned_task:
  106. kanban_db = spawn_env.get("HERMES_KANBAN_DB")
  107. default_root = os.path.join(spawn_env.get("HERMES_HOME", os.path.expanduser("~/.hermes")), "kanban")
  108. kanban_root = os.path.dirname(kanban_db) if kanban_db else spawn_env.get("HERMES_KANBAN_ROOT", default_root)
  109. cmd += [
  110. "-c", 'sandbox_mode="workspace-write"',
  111. "-c", f'sandbox_workspace_write.writable_roots=["{kanban_root}"]',
  112. "-c", "sandbox_workspace_write.network_access=false",
  113. ]
  114. # Codex emits tracing to stderr; default WARN keeps it quiet for users.
  115. spawn_env.setdefault("RUST_LOG", "warn")
  116. # Hide the console the codex child would otherwise flash on Windows (#56747).
  117. # Hide-only — stdio pipes stay intact for the app-server wire.
  118. # See #56747.
  119. from hermes_cli._subprocess_compat import windows_hide_flags
  120. self._proc = subprocess.Popen(
  121. cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  122. bufsize=0, env=spawn_env, creationflags=windows_hide_flags(),
  123. )
  124. self._next_id = 1
  125. self._pending: dict[int, queue.Queue] = {} # request id -> single-slot reply queue
  126. self._pending_lock = threading.Lock()
  127. self._notifications: queue.Queue = queue.Queue()
  128. self._server_requests: queue.Queue = queue.Queue()
  129. self._stderr_lines: list[str] = []
  130. self._stderr_lock = threading.Lock()
  131. self._closed = False
  132. self._initialized = False
  133. self._reader = threading.Thread(target=self._read_stdout, daemon=True)
  134. self._stderr_reader = threading.Thread(target=self._read_stderr, daemon=True)
  135. self._reader.start()
  136. self._stderr_reader.start()
  137. def initialize(
  138. self, client_name: str = "hermes", client_title: str = "Hermes Agent",
  139. client_version: str = "0.1", capabilities: Optional[dict] = None, timeout: float = 10.0,
  140. ) -> dict:
  141. """Send ``initialize`` + ``initialized``; return the server's InitializeResponse."""
  142. if self._initialized:
  143. raise RuntimeError("already initialized")
  144. params = {
  145. "clientInfo": {"name": client_name, "title": client_title, "version": client_version},
  146. "capabilities": capabilities or {},
  147. }
  148. result = self.request("initialize", params, timeout=timeout)
  149. self.notify("initialized")
  150. self._initialized = True
  151. return result
  152. def close(self, timeout: float = 3.0) -> None:
  153. """Close stdin and wait for the subprocess TREE to exit, escalating to kill.
  154. Codex app-server owns stdio MCP descendants that may sit in their own process
  155. groups (``setsid``). Once the root exits they reparent and a parent walk can no
  156. longer find them, so descendants are snapshotted BEFORE the root is retired and
  157. the proven identities (PID + create time) are swept afterwards."""
  158. if self._closed:
  159. return
  160. self._closed = True
  161. self._fail_pending_requests("codex app-server client is closing")
  162. descendants = _snapshot_descendants(self._proc.pid)
  163. with contextlib.suppress(Exception):
  164. if self._proc.stdin and not self._proc.stdin.closed:
  165. self._proc.stdin.close()
  166. try:
  167. self._proc.terminate()
  168. self._proc.wait(timeout=timeout)
  169. except subprocess.TimeoutExpired:
  170. with contextlib.suppress(Exception):
  171. kill_process_tree(self._proc.pid)
  172. self._proc.wait(timeout=1.0)
  173. finally:
  174. _reap_snapshotted(descendants)
  175. def _fail_pending_requests(self, reason: str) -> None:
  176. """Unblock every thread currently sitting in request() instead of
  177. leaving them to ride out their own per-call timeout (up to 30s by
  178. default) after the transport they're waiting on has already died.
  179. Mirrors _read_stdout's own pop-then-deliver dispatch under the same
  180. lock, so a reply that lands at the exact same moment still wins the
  181. race cleanly instead of being dropped or double-delivered."""
  182. with self._pending_lock:
  183. pending_items = list(self._pending.items())
  184. self._pending.clear()
  185. if not pending_items:
  186. return
  187. synthetic = {"error": {"code": _TRANSPORT_LOST_CODE, "message": reason}, "transportLost": True}
  188. for _rid, pending in pending_items:
  189. pending.put_nowait(synthetic)
  190. def __enter__(self) -> CodexAppServerClient:
  191. return self
  192. def __exit__(self, *exc: object) -> None:
  193. self.close()
  194. def request(self, method: str, params: Optional[dict] = None, timeout: float = 30.0) -> dict:
  195. """Send a request and block for ``result``; raise CodexAppServerError on ``error``."""
  196. rid, self._next_id = self._next_id, self._next_id + 1
  197. q: queue.Queue = queue.Queue(maxsize=1)
  198. with self._pending_lock:
  199. self._pending[rid] = q
  200. try:
  201. self._send({"id": rid, "method": method, "params": params or {}})
  202. except CodexAppServerTransportError:
  203. with self._pending_lock:
  204. self._pending.pop(rid, None)
  205. raise
  206. try:
  207. msg = q.get(timeout=timeout)
  208. except queue.Empty:
  209. with self._pending_lock:
  210. self._pending.pop(rid, None)
  211. raise TimeoutError(f"codex app-server method {method!r} timed out after {timeout}s")
  212. if "error" in msg:
  213. err = msg["error"]
  214. cls = CodexAppServerTransportError if msg.get("transportLost") else CodexAppServerError
  215. raise cls(code=err.get("code", -1), message=err.get("message", ""), data=err.get("data"))
  216. return msg.get("result", {})
  217. def notify(self, method: str, params: Optional[dict] = None) -> None:
  218. """Send a JSON-RPC notification (no id, no response expected)."""
  219. self._send({"method": method, "params": params or {}})
  220. def respond(self, request_id: Any, result: dict) -> None:
  221. """Reply to a server-initiated request (e.g. approval prompts)."""
  222. self._send({"id": request_id, "result": result})
  223. def respond_error(self, request_id: Any, code: int, message: str, data: Optional[Any] = None) -> None:
  224. """Reply to a server-initiated request with an error."""
  225. err: dict[str, Any] = {"code": code, "message": message}
  226. if data is not None:
  227. err["data"] = data
  228. self._send({"id": request_id, "error": err})
  229. @staticmethod
  230. def _take(q: queue.Queue, timeout: float) -> Optional[dict]:
  231. try:
  232. return q.get_nowait() if timeout <= 0 else q.get(timeout=timeout)
  233. except queue.Empty:
  234. return None
  235. def take_notification(self, timeout: float = 0.0) -> Optional[dict]:
  236. """Pop the next streaming notification, or None on timeout (0 = non-blocking)."""
  237. return self._take(self._notifications, timeout)
  238. def take_server_request(self, timeout: float = 0.0) -> Optional[dict]:
  239. """Pop the next server-initiated request (e.g. exec/applyPatch approval)."""
  240. return self._take(self._server_requests, timeout)
  241. def stderr_tail(self, n: int = 20) -> list[str]:
  242. """Return last n lines of codex's stderr (for error reports)."""
  243. if self._proc.poll() is not None:
  244. # The reader can still hold the crash text when the exit is first observed (#121467).
  245. self._stderr_reader.join(timeout=1.0)
  246. with self._stderr_lock:
  247. return list(self._stderr_lines[-n:])
  248. def is_alive(self) -> bool:
  249. return self._proc.poll() is None
  250. def _send(self, obj: dict) -> None:
  251. if self._closed:
  252. raise CodexAppServerTransportError(code=_TRANSPORT_LOST_CODE, message="codex app-server client is closed")
  253. if self._proc.stdin is None:
  254. raise CodexAppServerTransportError(code=_TRANSPORT_LOST_CODE, message="codex app-server stdin not available")
  255. try:
  256. self._proc.stdin.write((json.dumps(obj) + "\n").encode("utf-8"))
  257. self._proc.stdin.flush()
  258. except (OSError, ValueError) as exc: # BrokenPipe, EINVAL on a torn-down pipe, write on closed file
  259. raise CodexAppServerTransportError(
  260. code=_TRANSPORT_LOST_CODE, message=f"codex app-server stdin closed unexpectedly: {exc}",
  261. ) from exc
  262. def _append_stderr(self, line: str) -> None:
  263. with self._stderr_lock:
  264. self._stderr_lines.append(line)
  265. if len(self._stderr_lines) > 500: # bound memory
  266. self._stderr_lines = self._stderr_lines[-500:]
  267. def _read_stdout(self) -> None:
  268. if self._proc.stdout is None:
  269. return
  270. try:
  271. for line in iter(self._proc.stdout.readline, b""):
  272. line = line.strip()
  273. if not line:
  274. continue
  275. try:
  276. msg = json.loads(line)
  277. except json.JSONDecodeError:
  278. # Non-JSON stdout is unexpected; surface it via the stderr buffer.
  279. self._append_stderr(f"<non-json on stdout> {line[:200]!r}")
  280. continue
  281. self._dispatch(msg)
  282. except Exception as exc:
  283. self._append_stderr(f"<stdout reader error> {exc}")
  284. finally:
  285. # EOF (codex died) or a reader failure: nobody will ever answer the
  286. # requests still waiting, so fail them now instead of letting each
  287. # ride out its per-call timeout.
  288. self._fail_pending_requests("codex app-server stdout closed")
  289. def _dispatch(self, msg: dict) -> None:
  290. if "id" in msg and ("result" in msg or "error" in msg): # reply
  291. with self._pending_lock:
  292. pending = self._pending.pop(msg["id"], None)
  293. if pending is not None:
  294. with contextlib.suppress(queue.Full): # pragma: no cover - defensive
  295. pending.put_nowait(msg)
  296. elif "method" in msg: # server-initiated request (has id) or notification
  297. (self._server_requests if "id" in msg else self._notifications).put(msg)
  298. def _read_stderr(self) -> None:
  299. if self._proc.stderr is None:
  300. return
  301. with contextlib.suppress(Exception): # pragma: no cover
  302. for line in iter(self._proc.stderr.readline, b""):
  303. self._append_stderr(line.decode("utf-8", "replace").rstrip())
  304. def parse_codex_version(output: str) -> Optional[tuple[int, int, int]]:
  305. """Parse ``codex --version`` output ("codex-cli 0.130.0 ...") into (major, minor, patch)."""
  306. match = re.search(r"(\d+)\.(\d+)\.(\d+)", output or "")
  307. return tuple(int(g) for g in match.groups()) if match else None
  308. def check_codex_binary(
  309. codex_bin: str = "codex", min_version: tuple[int, int, int] = MIN_CODEX_VERSION
  310. ) -> tuple[bool, str]:
  311. """Verify codex CLI is installed and meets minimum version. Returns (ok, message)."""
  312. try:
  313. proc = subprocess.run(
  314. [codex_bin, "--version"], capture_output=True, text=True, encoding='utf-8', errors='replace',
  315. timeout=10, stdin=subprocess.DEVNULL,
  316. check=False,
  317. )
  318. except FileNotFoundError:
  319. return False, f"codex CLI not found at {codex_bin!r}. Install with: npm i -g @openai/codex"
  320. except subprocess.TimeoutExpired:
  321. return False, "codex --version timed out"
  322. if proc.returncode != 0:
  323. return False, f"codex --version exited {proc.returncode}: {proc.stderr.strip()}"
  324. version = parse_codex_version(proc.stdout)
  325. if version is None:
  326. return False, f"could not parse codex version from: {proc.stdout!r}"
  327. have = ".".join(map(str, version))
  328. if version < min_version:
  329. return False, f"codex {have} is older than required {'.'.join(map(str, min_version))}. Run: npm i -g @openai/codex"
  330. return True, have