code_kernel.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868
  1. """Session-persistent Python kernels for execute_code: one child per (owner, mode,
  2. interpreter, cwd, tool-set), one code cell per call, state survives across calls.
  3. Constraints, in order: (1) SAME security envelope as per-call (``_build_child_env``
  4. scrubbing, ``_rpc_server_loop`` token + per-cell tool budget, ANSI strip + secret
  5. redaction) — only lifetime widens. (2) A wedged kernel dies, never hangs the agent:
  6. timeout/interrupt kills the process tree and drops the registry entry; state loss is
  7. deliberate (a cell cannot be interrupted in place safely). (3) Env frozen at spawn:
  8. later passthrough is invisible until ``reset=true`` (the result names the kernel).
  9. Wire protocol: one JSON request per stdin line ``{"id", "code"}``; replies framed on
  10. stdout as ``<SENTINEL> <byte-length>\\n<json>`` with a per-kernel random SENTINEL from
  11. the env. Bytes outside frames are raw fd output attributed to the running cell (calls
  12. are serialized per kernel). A forged frame can only fake its own cell result.
  13. Also hosts what ``tools.code_kernel_remote`` shares: owner resolution, registry, cell core.
  14. """
  15. from __future__ import annotations
  16. import atexit
  17. import glob
  18. import json
  19. import logging
  20. import os
  21. import queue
  22. import secrets
  23. import shutil
  24. import socket
  25. import subprocess
  26. import sys
  27. import tempfile
  28. import threading
  29. import time
  30. import uuid
  31. from pathlib import Path
  32. from typing import Any, Callable, Dict, List, Optional, Tuple
  33. logger = logging.getLogger(__name__)
  34. _IS_WINDOWS = sys.platform == "win32"
  35. # Runner-side cap on captured python-level output; the host re-applies its own MAX_STDOUT cap.
  36. _RUNNER_CAPTURE_BYTES = 1_000_000
  37. # Shared by both generated runners (which define _CAPTURE_LIMIT first): exec one request in the
  38. # persistent GLOBALS namespace, build the payload. `__name__` is `__main__` as on the per-call path.
  39. RUNNER_CELL_SOURCE = '''\
  40. GLOBALS = {"__name__": "__main__", "__builtins__": __builtins__}
  41. def _clip(text):
  42. return (text, False) if len(text) <= _CAPTURE_LIMIT else (text[:_CAPTURE_LIMIT], True)
  43. def run_cell(request, execution_count):
  44. """Exec one cell; returns (response payload, FULL stdout text)."""
  45. out, err = io.StringIO(), io.StringIO()
  46. status, trace = "ok", ""
  47. try:
  48. with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
  49. exec(compile(request["code"], "<cell>", "exec"), GLOBALS)
  50. except SystemExit as exc:
  51. status, trace = "exit", "SystemExit: " + repr(exc.code)
  52. except BaseException:
  53. status, trace = "error", traceback.format_exc()
  54. stdout_text, stdout_clipped = _clip(out.getvalue())
  55. stderr_text, stderr_clipped = _clip(err.getvalue())
  56. return {
  57. "id": request.get("id", ""), "status": status,
  58. "stdout": stdout_text, "stderr": stderr_text,
  59. "stdout_clipped": stdout_clipped, "stderr_clipped": stderr_clipped,
  60. "traceback": trace, "execution_count": execution_count,
  61. }, out.getvalue()
  62. '''
  63. KERNEL_RUNNER_SOURCE = f'''\
  64. """Auto-generated Hermes session-kernel runner. One exec cell per request."""
  65. import contextlib
  66. import io
  67. import json
  68. import os
  69. import sys
  70. import threading
  71. import traceback
  72. _SENTINEL = os.environ["HERMES_KERNEL_SENTINEL"]
  73. _CAPTURE_LIMIT = {_RUNNER_CAPTURE_BYTES}
  74. _SPILL_DIR = os.environ.get("HERMES_KERNEL_SPILL_DIR", "")
  75. _SPILL_CAP = {5_000_000}
  76. _PARENT_PROCESS_HANDLE = os.environ.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", "")
  77. def _start_parent_process_watchdog():
  78. """Exit when the exact Windows parent process object is signaled.
  79. The inherited SYNCHRONIZE handle names a process object, not a reusable
  80. PID. Missing or invalid handles fail open so watchdog setup can never kill
  81. an otherwise healthy kernel.
  82. """
  83. global _PARENT_PROCESS_HANDLE
  84. raw_handle = _PARENT_PROCESS_HANDLE
  85. _PARENT_PROCESS_HANDLE = ""
  86. if sys.platform != "win32" or not raw_handle:
  87. return
  88. try:
  89. import ctypes
  90. from ctypes import wintypes
  91. handle = int(raw_handle)
  92. if handle <= 0:
  93. return
  94. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  95. kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
  96. kernel32.WaitForSingleObject.restype = wintypes.DWORD
  97. kernel32.SetHandleInformation.argtypes = [
  98. wintypes.HANDLE,
  99. wintypes.DWORD,
  100. wintypes.DWORD,
  101. ]
  102. kernel32.SetHandleInformation.restype = wintypes.BOOL
  103. kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
  104. kernel32.CloseHandle.restype = wintypes.BOOL
  105. # This process needs the handle, but user code spawned by a cell must
  106. # not pass it any further. If Windows refuses to clear inheritance,
  107. # disable the watchdog rather than leak the handle into cell children.
  108. if not kernel32.SetHandleInformation(handle, 0x00000001, 0):
  109. kernel32.CloseHandle(handle)
  110. return
  111. except (ImportError, OSError, TypeError, ValueError):
  112. return
  113. def _wait():
  114. try:
  115. result = kernel32.WaitForSingleObject(handle, 0xFFFFFFFF)
  116. finally:
  117. kernel32.CloseHandle(handle)
  118. if result == 0x00000000: # WAIT_OBJECT_0: the parent exited
  119. os._exit(0)
  120. threading.Thread(target=_wait, name="hermes-parent-watchdog", daemon=True).start()
  121. _start_parent_process_watchdog()
  122. _real_stdout = sys.stdout
  123. {RUNNER_CELL_SOURCE}
  124. def _spill(text, spill_name):
  125. """Best-effort: write the FULL clipped stdout to disk, return its path or ""."""
  126. if not _SPILL_DIR:
  127. return ""
  128. try:
  129. spill_path = os.path.join(_SPILL_DIR, spill_name)
  130. with open(spill_path, "w", encoding="utf-8", errors="replace") as f:
  131. f.write(text[:_SPILL_CAP])
  132. if len(text) > _SPILL_CAP:
  133. f.write("\\n\\n[... spill capped ...]")
  134. return spill_path
  135. except Exception:
  136. return ""
  137. def _reply(payload):
  138. body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
  139. _real_stdout.buffer.write(("\\n" + _SENTINEL + " " + str(len(body)) + "\\n").encode("utf-8"))
  140. _real_stdout.buffer.write(body)
  141. _real_stdout.buffer.flush()
  142. def main():
  143. execution_count = 0
  144. for line in sys.stdin:
  145. line = line.strip()
  146. if not line:
  147. continue
  148. try:
  149. request = json.loads(line)
  150. except ValueError:
  151. continue
  152. execution_count += 1
  153. payload, full_stdout = run_cell(request, execution_count)
  154. payload["stdout_spill_path"] = (
  155. _spill(full_stdout, "cell_%06d_stdout.txt" % execution_count)
  156. if payload["stdout_clipped"] else ""
  157. )
  158. _reply(payload)
  159. if payload["status"] == "exit":
  160. break
  161. if __name__ == "__main__":
  162. main()
  163. '''
  164. class CellAuthority:
  165. """The approval/context identity of exactly one execute_code cell.
  166. Interpreter state persists across cells; RPC authority must not. Each cell installs a
  167. fresh authority captured from the CALLING thread at cell start (what
  168. ``propagate_context_to_thread`` captures for a per-call RPC thread) and retires it when
  169. the cell settles, so a late tool call (leaked background thread, raced client write) is
  170. refused instead of running under a stale approval/session/turn identity.
  171. """
  172. def __init__(self, task_id: str):
  173. import contextvars
  174. self.task_id = task_id
  175. self.ctx = contextvars.copy_context()
  176. self.active = True
  177. # ((getter, setter), captured value) per thread-local prompt callback (approval, sudo, vault unlock…)
  178. self._callbacks: list = []
  179. try:
  180. from tools.thread_context import _callback_api
  181. self._callbacks = [(pair, pair[0]()) for pair in _callback_api()]
  182. except Exception:
  183. # Fail-closed like propagate_context_to_thread: no callbacks → dangerous approvals deny.
  184. self._callbacks = []
  185. def retire(self) -> None:
  186. self.active = False
  187. def dispatch(self, tool_name: str, tool_args: dict) -> str:
  188. """Run one tool call under THIS cell's context and callbacks."""
  189. from tools.registry import tool_error
  190. if not self.active:
  191. return tool_error("No active execute_code cell: the cell this kernel call "
  192. "belonged to has settled, so its tool authority is retired.")
  193. return self.ctx.run(self._invoke, tool_name, tool_args)
  194. def _invoke(self, tool_name: str, tool_args: dict) -> str:
  195. from model_tools import handle_function_call
  196. previous = None
  197. if self._callbacks:
  198. try:
  199. previous = [(setter, getter()) for (getter, setter), _cb in self._callbacks]
  200. for (_getter, setter), cb in self._callbacks:
  201. setter(cb)
  202. except Exception:
  203. previous = None
  204. try:
  205. return handle_function_call(tool_name, tool_args, task_id=self.task_id)
  206. finally:
  207. if previous is not None:
  208. try:
  209. for setter, cb in previous:
  210. setter(cb)
  211. except Exception:
  212. pass
  213. class _BoundedBuffer:
  214. """Byte chunks capped at a total size; ``drain`` returns text and resets."""
  215. def __init__(self):
  216. self.chunks: list[bytes] = []
  217. self.total = 0
  218. def append(self, data: bytes, cap: int) -> None:
  219. keep = data[: max(0, cap - self.total)]
  220. if keep:
  221. self.chunks.append(keep)
  222. self.total += len(keep)
  223. def drain(self) -> str:
  224. chunks, self.chunks, self.total = self.chunks, [], 0
  225. return b"".join(chunks).decode("utf-8", errors="replace")
  226. class SessionKernel:
  227. """One live kernel process plus its RPC server and reader threads."""
  228. def __init__(self, key: tuple):
  229. self.key, self.owner, self.lock = key, key[0], threading.Lock()
  230. self.proc: Optional[subprocess.Popen] = None
  231. self.tmpdir = self.rpc_token = self.sentinel = ""
  232. self.sock_path: Optional[str] = None
  233. self.server_sock: Optional[socket.socket] = None
  234. self.stop_event = threading.Event()
  235. self.tool_call_log: list = []
  236. self.cell_log_start = 0
  237. self.tool_call_counter: list[int] = [0]
  238. # Cells currently attached (bumped under the registry lock on selection, dropped when the
  239. # cell settles). Reaping/cap-eviction skip attached kernels: tearing one down mid-spawn
  240. # rmtree'd the staging dir under the spawner and killed live cells.
  241. self.attached: int = 0
  242. # Owned by a live delegate_task child: exempt from LRU eviction (the child's teardown disposes it).
  243. self.pinned: bool = False
  244. self.response_q: queue.Queue[dict] = queue.Queue()
  245. self.raw, self.stderr = _BoundedBuffer(), _BoundedBuffer()
  246. self.execution_count, self.last_used = 0, time.monotonic()
  247. self.cell_authority: Optional[CellAuthority] = None
  248. def alive(self) -> bool:
  249. return self.proc is not None and self.proc.poll() is None
  250. def dead(self) -> bool:
  251. """True only once a spawned process has exited. ``proc is None`` is mid-spawn, not dead:
  252. parallel cells for one owner race the first ``_spawn``, and treating the pending kernel as
  253. dead made every racer replace it, orphaning the winner's process outside the registry."""
  254. return self.proc is not None and self.proc.poll() is not None
  255. def teardown(self) -> None:
  256. self.stop_event.set()
  257. if self.alive():
  258. from tools.code_execution_tool import _kill_process_group
  259. _kill_process_group(self.proc, escalate=True)
  260. if self.proc is not None and not _IS_WINDOWS:
  261. from hermes_cli.process_identity import release_process_group_once_empty
  262. release_process_group_once_empty(self.proc.pid, proc=self.proc)
  263. sock, self.server_sock = self.server_sock, None
  264. try:
  265. if sock is not None:
  266. sock.close()
  267. if self.sock_path:
  268. os.unlink(self.sock_path)
  269. except OSError:
  270. pass
  271. if self.tmpdir:
  272. import shutil
  273. shutil.rmtree(self.tmpdir, ignore_errors=True)
  274. class KernelRegistry:
  275. """Key -> kernel map plus its lock (shared with the remote registry). Kernels are popped
  276. under the lock and torn down outside it — teardown may block on the child or the transport."""
  277. def __init__(self, teardown: Callable[[Any], None]):
  278. self.kernels: dict[tuple, Any] = {}
  279. self.lock, self._teardown = threading.Lock(), teardown
  280. def shutdown(self, owner: Optional[str] = None, *, owner_matches: Optional[Callable[[str], bool]] = None) -> None:
  281. """Tear down every kernel, every kernel one owner (key[0]) holds, or every kernel whose owner
  282. satisfies ``owner_matches``."""
  283. with self.lock:
  284. doomed = [self.kernels.pop(key) for key in list(self.kernels)
  285. if (owner is None and owner_matches is None) or key[0] == owner
  286. or (owner_matches is not None and owner_matches(key[0]))]
  287. for kernel in doomed:
  288. self._teardown(kernel)
  289. def discard(self, key: tuple, kernel: Any) -> None:
  290. """Drop *kernel*'s registry entry (only if it is still the one registered under *key* —
  291. never a replacement) and tear the kernel down."""
  292. with self.lock:
  293. if self.kernels.get(key) is kernel:
  294. self.kernels.pop(key, None)
  295. self._teardown(kernel)
  296. _REGISTRY = KernelRegistry(lambda kernel: kernel.teardown())
  297. _KERNELS: dict[tuple, SessionKernel] = _REGISTRY.kernels
  298. # Bounded lifecycle defaults (config: code_execution.max_session_kernels / kernel_idle_timeout).
  299. # A long-lived gateway must never accumulate one live child per finished conversation:
  300. # stable owner id, owner-teardown disposal, idle reaping, max-live bound.
  301. # See #88637.
  302. DEFAULT_MAX_SESSION_KERNELS = 4
  303. DEFAULT_KERNEL_IDLE_TIMEOUT = 1800
  304. def _lifecycle_limits() -> tuple[int, int]:
  305. from tools.code_execution_tool import _load_config
  306. config = _load_config()
  307. def limit(key: str, default: int) -> int:
  308. try:
  309. return max(1, int(config.get(key, default)))
  310. except (TypeError, ValueError):
  311. return default
  312. return limit("max_session_kernels", DEFAULT_MAX_SESSION_KERNELS), limit("kernel_idle_timeout", DEFAULT_KERNEL_IDLE_TIMEOUT)
  313. _CHILD_OWNER_QUALIFIER = "::child::"
  314. def _resolve_owner(task_id: str) -> str:
  315. """The stable identity a session kernel belongs to: the conversation's approval session key
  316. (context-propagated, stable across turns, distinct per session). ``run_agent`` mints a fresh
  317. task id per turn, so a task-keyed kernel would neither survive the next turn nor be torn down
  318. with anything; the task id is only the last-resort owner (embeds/tests without a session).
  319. Delegated children INHERIT the parent's approval session key — without the ``::child::``
  320. qualifier a child's execute_code would attach to the parent's kernel and read its state
  321. (verified live, both directions). Children get their own kernels keyed by delegation session id.
  322. """
  323. try:
  324. from tools.approval_context import get_current_session_key
  325. session_key = get_current_session_key(default="")
  326. except Exception:
  327. session_key = ""
  328. owner = session_key or (task_id or "")
  329. try:
  330. from agent.delegation_context import is_delegated_child_context
  331. if is_delegated_child_context():
  332. from gateway.session_context import get_session_env
  333. child_id = get_session_env("HERMES_SESSION_ID", "") or (task_id or "")
  334. owner = f"{owner}{_CHILD_OWNER_QUALIFIER}{child_id}"
  335. except Exception:
  336. pass
  337. return owner
  338. def shutdown_all_kernels() -> None:
  339. """Kill every session kernel. Registered via atexit; also used by tests."""
  340. _REGISTRY.shutdown()
  341. def shutdown_kernels_for_owner(owner: str) -> None:
  342. """Dispose every kernel a session owns — wired into ``tools.approval.clear_session``
  343. so kernels die at the same boundary that clears approval/yolo state (/new, session close).
  344. See #88637.
  345. """
  346. if owner:
  347. _REGISTRY.shutdown(owner)
  348. def delegated_child_owner_matcher(child_session_id: str) -> Callable[[str], bool]:
  349. """Predicate for the kernels a delegate_task child owns (``_resolve_owner`` qualifies a child's
  350. owner with its delegation session id). Shared with the remote registry."""
  351. suffix = f"{_CHILD_OWNER_QUALIFIER}{child_session_id}"
  352. return lambda owner: owner.endswith(suffix)
  353. def shutdown_kernels_for_delegated_child(child_session_id: str) -> None:
  354. """Dispose a finished child's kernels (local and remote). A child's kernel lives exactly as long as the
  355. child: pinned against LRU eviction while it runs, torn down here — otherwise finished children's
  356. kernels squatted the process-wide cap for ``kernel_idle_timeout`` and evicted LIVE children's kernels,
  357. which then lost their state mid-task with no signal but ``reused: false``."""
  358. if not child_session_id:
  359. return
  360. matcher = delegated_child_owner_matcher(child_session_id)
  361. _REGISTRY.shutdown(owner_matches=matcher)
  362. from tools.code_kernel_remote import shutdown_remote_kernels_where
  363. shutdown_remote_kernels_where(matcher)
  364. atexit.register(shutdown_all_kernels)
  365. def _rpc_forever(kernel: SessionKernel, max_tool_calls: int,
  366. sandbox_tools: frozenset) -> None:
  367. """Serve tool RPC for the kernel's whole life: ``_rpc_server_loop`` returns on disconnect or
  368. its 300s idle timeout, and a kernel idles longer between cells, so re-accept until teardown
  369. (the client stub reconnects: HERMES_RPC_PERSISTENT). The serving thread carries NO frozen
  370. authority — every dispatch routes through the CURRENT cell's ``CellAuthority``."""
  371. from tools.code_execution_rpc import _rpc_server_loop
  372. from tools.registry import tool_error
  373. def _dispatch(tool_name: str, tool_args: dict) -> str:
  374. authority = kernel.cell_authority
  375. if authority is None:
  376. return tool_error("No active execute_code cell: this kernel has no cell authority installed.")
  377. return authority.dispatch(tool_name, tool_args)
  378. while not kernel.stop_event.is_set():
  379. _rpc_server_loop(kernel.server_sock, "", kernel.tool_call_log, kernel.tool_call_counter,
  380. max_tool_calls, sandbox_tools, kernel.stop_event, kernel.rpc_token,
  381. dispatch=_dispatch)
  382. def _stdout_reader(kernel: SessionKernel) -> None:
  383. """Split the child's stdout into protocol frames and raw passthrough."""
  384. from tools.code_execution_tool import MAX_STDOUT_BYTES
  385. assert kernel.proc is not None and kernel.proc.stdout is not None
  386. stream = kernel.proc.stdout
  387. marker = ("\n" + kernel.sentinel + " ").encode("utf-8")
  388. def raw(data: bytes) -> None:
  389. kernel.raw.append(data, MAX_STDOUT_BYTES)
  390. buf = b""
  391. while True:
  392. # read1 returns as soon as any bytes arrive; a plain read(n) on a BufferedReader
  393. # blocks until n bytes or EOF and would sit on a complete small frame forever.
  394. chunk = stream.read1(4096)
  395. if not chunk:
  396. if buf:
  397. raw(buf)
  398. kernel.response_q.put({"status": "kernel-eof"})
  399. return
  400. buf += chunk
  401. while True:
  402. index = buf.find(marker)
  403. if index < 0:
  404. # Keep a marker-sized tail (marker may be split across reads); the rest is raw.
  405. spill = buf[: -len(marker)] if len(buf) > len(marker) else b""
  406. if spill:
  407. raw(spill)
  408. buf = buf[len(spill):]
  409. break
  410. if index:
  411. raw(buf[:index])
  412. rest = buf[index + len(marker):]
  413. newline = rest.find(b"\n")
  414. if newline < 0:
  415. buf = buf[index:]
  416. break
  417. try:
  418. length = int(rest[:newline])
  419. except ValueError:
  420. # Not a real frame header (user output containing the marker bytes): raw.
  421. raw(marker)
  422. buf = rest
  423. continue
  424. body = rest[newline + 1:]
  425. while len(body) < length:
  426. more = stream.read1(length - len(body))
  427. if not more:
  428. kernel.response_q.put({"status": "kernel-eof"})
  429. return
  430. body += more
  431. try:
  432. kernel.response_q.put(json.loads(body[:length].decode("utf-8", errors="replace")))
  433. except ValueError:
  434. kernel.response_q.put({"status": "protocol-error"})
  435. buf = body[length:]
  436. def _stderr_reader(kernel: SessionKernel) -> None:
  437. from tools.code_execution_tool import MAX_STDERR_BYTES
  438. assert kernel.proc is not None and kernel.proc.stderr is not None
  439. while chunk := kernel.proc.stderr.read1(4096):
  440. kernel.stderr.append(chunk, MAX_STDERR_BYTES)
  441. def _bind_rpc_socket(kernel: SessionKernel) -> str:
  442. """Bind the tool-RPC listener: loopback TCP on Windows, 0600 UDS elsewhere."""
  443. if _IS_WINDOWS:
  444. kernel.sock_path = None
  445. server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  446. server_sock.bind(("127.0.0.1", 0))
  447. host, port = server_sock.getsockname()[:2]
  448. rpc_endpoint = f"tcp://{host}:{port}"
  449. else:
  450. from hermes_constants import socket_safe_tmpdir
  451. sock_tmpdir = socket_safe_tmpdir()
  452. rpc_endpoint = kernel.sock_path = os.path.join(sock_tmpdir, f"hermes_rpc_{uuid.uuid4().hex}.sock")
  453. server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  454. server_sock.bind(kernel.sock_path)
  455. os.chmod(kernel.sock_path, 0o600)
  456. server_sock.listen(1)
  457. kernel.server_sock = server_sock
  458. return rpc_endpoint
  459. def _parent_process_handle(child_env: dict[str, str]):
  460. """Windows: open an inheritable SYNCHRONIZE handle to this process for the kernel's parent-death
  461. watchdog. Returns (handle, CloseHandle, startupinfo) or (None, None, None); fails open."""
  462. handle = close = startupinfo = None
  463. try:
  464. import ctypes
  465. from ctypes import wintypes
  466. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  467. kernel32.GetCurrentProcessId.argtypes = []
  468. kernel32.GetCurrentProcessId.restype = wintypes.DWORD
  469. kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
  470. kernel32.OpenProcess.restype = wintypes.HANDLE
  471. kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
  472. kernel32.CloseHandle.restype = wintypes.BOOL
  473. close = kernel32.CloseHandle
  474. # SYNCHRONIZE; inherited only by the explicitly allow-listed child.
  475. handle = kernel32.OpenProcess(0x00100000, True, kernel32.GetCurrentProcessId())
  476. if handle:
  477. child_env["HERMES_KERNEL_PARENT_PROCESS_HANDLE"] = str(int(handle))
  478. startupinfo = subprocess.STARTUPINFO()
  479. startupinfo.lpAttributeList = {"handle_list": [int(handle)]}
  480. except (AttributeError, ImportError, OSError, TypeError, ValueError):
  481. if handle and close is not None:
  482. close(handle)
  483. child_env.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", None)
  484. handle = close = startupinfo = None
  485. return handle, close, startupinfo
  486. def _spawn(kernel: SessionKernel, *, child_python: str, child_cwd: str,
  487. sandbox_tools: frozenset, max_tool_calls: int, task_id: str = "") -> None:
  488. from tools.code_execution_env import _build_child_env
  489. from tools.code_execution_tool import generate_hermes_tools_module
  490. kernel.tmpdir = tempfile.mkdtemp(prefix="hermes_kernel_")
  491. kernel.rpc_token = secrets.token_urlsafe(32)
  492. kernel.sentinel = "@@HERMES-KERNEL-" + secrets.token_urlsafe(16) + "@@"
  493. rpc_endpoint = _bind_rpc_socket(kernel)
  494. for name, src in (("hermes_tools.py", generate_hermes_tools_module(list(sandbox_tools))),
  495. ("hermes_kernel_runner.py", KERNEL_RUNNER_SOURCE)):
  496. Path(kernel.tmpdir, name).write_text(src, encoding="utf-8")
  497. child_env = _build_child_env(rpc_endpoint=rpc_endpoint, rpc_token=kernel.rpc_token,
  498. tmpdir=kernel.tmpdir, child_python=child_python)
  499. child_env["HERMES_KERNEL_SENTINEL"] = kernel.sentinel
  500. # Full clipped stdout spills to the kernel's tmpdir so the agent can read_file the middle.
  501. child_env["HERMES_KERNEL_SPILL_DIR"] = kernel.tmpdir
  502. # Generated client reconnects after the RPC server's 300s idle timeout between cells.
  503. child_env["HERMES_RPC_PERSISTENT"] = "1"
  504. # Owner death: Windows inherits a SYNCHRONIZE handle to this process for the runner's watchdog.
  505. parent_handle, close_handle, startupinfo = _parent_process_handle(child_env) if _IS_WINDOWS else (None, None, None)
  506. # POSIX: spawn_owned makes the kernel lead its own group, which everything a cell spawns joins,
  507. # and hands that group to the owner-death supervisor: the whole group dies the moment this
  508. # process dies, by any cause. A kernel-only death pipe let a cell's subprocesses outlive the host.
  509. from hermes_cli.process_identity import spawn_owned
  510. try:
  511. kernel.proc = spawn_owned(
  512. [child_python, os.path.join(kernel.tmpdir, "hermes_kernel_runner.py")], purpose="code-kernel",
  513. # Strict mode passes an empty cwd: the kernel's staging dir plays the per-call tmpdir's role.
  514. cwd=child_cwd or kernel.tmpdir, env=child_env,
  515. stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE,
  516. creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0,
  517. close_fds=True, startupinfo=startupinfo,
  518. )
  519. finally:
  520. if parent_handle and close_handle is not None:
  521. close_handle(parent_handle)
  522. # Deliberately NOT propagate_context_to_thread: that would freeze the spawning cell's
  523. # context/callbacks into the server thread for life. Authority is rebound per cell.
  524. for target, args in ((_rpc_forever, (kernel, max_tool_calls, sandbox_tools)),
  525. (_stdout_reader, (kernel,)), (_stderr_reader, (kernel,))):
  526. threading.Thread(target=target, args=args, daemon=True).start()
  527. _ensure_background_reaper()
  528. def _pop_idle_expired(now: float, idle_timeout: float) -> list[SessionKernel]:
  529. """Pop (caller holds ``_REGISTRY.lock``) every kernel idle past *idle_timeout*. Kernels with
  530. attached cells are skipped: the last cell out tears them down."""
  531. return [_KERNELS.pop(k) for k in list(_KERNELS)
  532. if _KERNELS[k].attached == 0 and now - _KERNELS[k].last_used > idle_timeout]
  533. def _acquire_kernel(key: tuple, reset: bool, *, pinned: bool = False) -> tuple[SessionKernel, bool]:
  534. """Look up or register the kernel for *key*; returns (kernel, state_reset). Every entry also
  535. sweeps idle-expired kernels and enforces the process-wide LRU cap (doomed kernels are popped
  536. under the lock, torn down outside it), so a long-lived host stays bounded. ``pinned`` kernels
  537. (live delegate_task children) are exempt from the cap: their lifetime is the child's, ended by
  538. ``shutdown_kernels_for_delegated_child``, so the cap has nothing to bound for them."""
  539. cap, idle_timeout = _lifecycle_limits()
  540. with _REGISTRY.lock:
  541. expired = _pop_idle_expired(time.monotonic(), idle_timeout)
  542. kernel = _KERNELS.get(key)
  543. state_reset = kernel is not None and (reset or kernel.dead())
  544. if state_reset:
  545. dropped = _KERNELS.pop(key)
  546. if dropped.attached == 0:
  547. expired.append(dropped)
  548. kernel = None
  549. if kernel is None:
  550. kernel = _KERNELS[key] = SessionKernel(key)
  551. kernel.pinned = pinned
  552. kernel.last_used = time.monotonic()
  553. kernel.attached += 1
  554. unpinned = [k for k in _KERNELS if not _KERNELS[k].pinned]
  555. by_age = sorted((k for k in unpinned if k != key and _KERNELS[k].attached == 0),
  556. key=lambda k: _KERNELS[k].last_used)
  557. expired.extend(_KERNELS.pop(k) for k in by_age[: max(0, len(unpinned) - cap)])
  558. for doomed in expired:
  559. doomed.teardown()
  560. return kernel, state_reset
  561. # The acquire-path sweep above only fires on the NEXT kernel request. A host that stays
  562. # alive but stops executing anything (a pids-exhausted container whose tool dispatch is
  563. # fail-closed) never acquires again, so idle kernels and their thread pools survive
  564. # indefinitely (#117169). One low-frequency daemon thread reapplies the same criteria on
  565. # its own schedule, independent of tool traffic, and also sweeps staging dirs that
  566. # outlived a host which died without cleanup (SIGKILL / container restart).
  567. _STALE_STAGING_DIR_AGE = 7 * 86400
  568. _REAPER_INTERVAL_FLOOR, _REAPER_INTERVAL_CEIL = 30.0, 300.0
  569. _REAPER_STARTED = False
  570. def _sweep_stale_staging_dirs(now: Optional[float] = None) -> int:
  571. """Remove ``hermes_kernel_*`` staging dirs untouched for over a week. A live host
  572. rmtrees each dir within one idle timeout of the kernel's last use, so a week-old
  573. dir belongs to a host that died before its cleanup could run; younger dirs are left
  574. alone because a concurrently running host's live kernel may own one. rmtree never
  575. follows symlinks, so a planted link is rejected rather than chased."""
  576. now = time.time() if now is None else now
  577. removed = 0
  578. for path in glob.glob(os.path.join(tempfile.gettempdir(), "hermes_kernel_*")):
  579. try:
  580. if now - os.path.getmtime(path) > _STALE_STAGING_DIR_AGE:
  581. # No ignore_errors: a rejected symlink (or a half-removed dir) must not
  582. # count as swept — it stays for the next pass instead.
  583. shutil.rmtree(path)
  584. removed += 1
  585. except OSError:
  586. continue
  587. return removed
  588. def _reap_once() -> None:
  589. """One background pass: the acquire-path idle criteria, then the stale-dir sweep."""
  590. _, idle_timeout = _lifecycle_limits()
  591. with _REGISTRY.lock:
  592. expired = _pop_idle_expired(time.monotonic(), idle_timeout)
  593. for doomed in expired:
  594. doomed.teardown()
  595. _sweep_stale_staging_dirs()
  596. def _ensure_background_reaper() -> None:
  597. """Start the reaper once per process (on the first kernel spawn)."""
  598. global _REAPER_STARTED
  599. with _REGISTRY.lock:
  600. if _REAPER_STARTED:
  601. return
  602. _REAPER_STARTED = True
  603. threading.Thread(target=_background_reaper, daemon=True,
  604. name="hermes-kernel-idle-reaper").start()
  605. def _background_reaper() -> None:
  606. while True:
  607. _, idle_timeout = _lifecycle_limits()
  608. time.sleep(min(_REAPER_INTERVAL_CEIL,
  609. max(_REAPER_INTERVAL_FLOOR, idle_timeout / 6.0)))
  610. try:
  611. _reap_once()
  612. except Exception:
  613. logger.exception("kernel idle reaper pass failed; retrying next interval")
  614. def _await_cell(kernel: SessionKernel, timeout: int, is_interrupted) -> tuple[str, dict[str, Any]]:
  615. """Wait for the cell's reply; returns (host status, payload)."""
  616. deadline = time.monotonic() + timeout if timeout else None
  617. while True:
  618. if is_interrupted():
  619. return "interrupted", {}
  620. if deadline is not None and time.monotonic() > deadline:
  621. return "timeout", {}
  622. try:
  623. payload = kernel.response_q.get(timeout=0.05)
  624. except queue.Empty:
  625. continue
  626. if payload.get("status") in ("kernel-eof", "protocol-error"):
  627. return "error", payload
  628. return "success", payload
  629. def _with_stderr(stdout_text: str, stderr_text: str) -> str:
  630. return stdout_text + "\n--- stderr ---\n" + stderr_text
  631. def _cell_result(kernel: SessionKernel, key: tuple, status: str, payload: dict[str, Any], *,
  632. timeout: int, sandbox_tools: frozenset, reused: bool,
  633. state_reset: bool, exec_start: float) -> dict[str, Any]:
  634. """Assemble the tool result for one settled cell (disposing the kernel where the contract says so)."""
  635. from tools.code_execution_tool import _sandbox_failure_hint, _truncate_stdout_text
  636. from agent.redact import redact_sensitive_text
  637. from tools.ansi_strip import strip_ansi
  638. def clean(text: str) -> str:
  639. return redact_sensitive_text(strip_ansi(text), code_file=True)
  640. if status in ("timeout", "interrupted"):
  641. # No safe way to interrupt one cell in place: kill the kernel, report the loss, respawn next call.
  642. _REGISTRY.discard(key, kernel)
  643. duration = round(time.monotonic() - exec_start, 2)
  644. kernel.execution_count = int(payload.get("execution_count", kernel.execution_count + 1))
  645. stderr_raw = kernel.stderr.drain()
  646. stdout_text, stdout_metadata = _truncate_stdout_text(clean(str(payload.get("stdout", "")) + kernel.raw.drain()))
  647. cell_stderr = clean(str(payload.get("stderr", "")) + stderr_raw)
  648. cell_status = payload.get("status", "")
  649. result: dict[str, Any] = {
  650. "status": status, "output": stdout_text, "exit_code": 0,
  651. "tool_calls_made": kernel.tool_call_counter[0], "duration_seconds": duration,
  652. "kernel": {"mode": "session", "reused": reused,
  653. "execution_count": kernel.execution_count, "state_reset": state_reset},
  654. }
  655. result.update(stdout_metadata)
  656. from tools.code_execution_rpc import tool_errors_since
  657. tool_errors = tool_errors_since(kernel.tool_call_log, kernel.cell_log_start)
  658. if tool_errors:
  659. result["tool_errors"] = tool_errors
  660. # Cell-side spill (runner clipped before replying): same read_file recipe as the host-side spill.
  661. cell_spill = str(payload.get("stdout_spill_path", "") or "")
  662. if cell_spill and payload.get("stdout_clipped"):
  663. result["stdout_spill_path"] = cell_spill
  664. result["warning"] = (
  665. f"Cell stdout exceeded the inline cap; head shown. FULL output saved to {cell_spill} "
  666. f'— page it with read_file(path="{cell_spill}", offset=...) instead of re-running. '
  667. "(Kernel state persists: printing a narrower slice next call is often cheaper.)"
  668. )
  669. if status == "timeout":
  670. message = (f"Cell timed out after {timeout}s; the session kernel was killed and its "
  671. "state was lost. The next execute_code call starts a fresh kernel.")
  672. result.update(exit_code=-1, error=message,
  673. output=(stdout_text + "\n\n⏰ " + message) if stdout_text else ("⏰ " + message))
  674. elif status == "interrupted":
  675. from tools.code_execution_tool import _format_interrupted_output
  676. result.update(exit_code=-1, output=_format_interrupted_output(stdout_text),
  677. error="Interrupted; the session kernel was killed and its state was lost.")
  678. elif cell_status == "error":
  679. trace = clean(str(payload.get("traceback", "")))
  680. result.update(status="error", exit_code=1, error=trace or "Cell raised an exception.",
  681. output=_with_stderr(stdout_text, cell_stderr + trace) if (cell_stderr or trace) else stdout_text)
  682. hint = _sandbox_failure_hint(trace, enabled_tools=sandbox_tools)
  683. if hint:
  684. result["hint"] = hint
  685. elif cell_status == "exit":
  686. # The cell called sys.exit(): honor it as end-of-kernel.
  687. _REGISTRY.discard(key, kernel)
  688. result["kernel"]["ended"] = True
  689. if cell_stderr:
  690. result["output"] = _with_stderr(stdout_text, cell_stderr)
  691. elif status == "error":
  692. _REGISTRY.discard(key, kernel)
  693. result.update(exit_code=-1, error="The session kernel died while running the cell"
  694. + (": " + stderr_raw.strip() if stderr_raw.strip() else "."))
  695. elif cell_stderr:
  696. result["output"] = _with_stderr(stdout_text, cell_stderr)
  697. return result
  698. def execute_in_session_kernel(
  699. code: str, *, task_id: str, mode: str, child_python: str, child_cwd: str,
  700. sandbox_tools: frozenset, timeout: int, max_tool_calls: int, reset: bool, is_interrupted,
  701. ) -> str:
  702. """Run one cell in the (owner, mode, python, cwd, tools) session kernel. The owner is the
  703. session key (``_resolve_owner``), not the per-turn task id, so state survives across turns."""
  704. key = (_resolve_owner(task_id) or "", mode, child_python, child_cwd, tuple(sorted(sandbox_tools)))
  705. exec_start = time.monotonic()
  706. from agent.delegation_context import is_delegated_child_context
  707. kernel, state_reset = _acquire_kernel(key, reset, pinned=is_delegated_child_context())
  708. try:
  709. return _run_cell(kernel, key, code, task_id=task_id, child_python=child_python, child_cwd=child_cwd,
  710. sandbox_tools=sandbox_tools, timeout=timeout, max_tool_calls=max_tool_calls,
  711. is_interrupted=is_interrupted, exec_start=exec_start, state_reset=state_reset)
  712. finally:
  713. with _REGISTRY.lock:
  714. kernel.attached -= 1
  715. kernel.last_used = time.monotonic()
  716. # Dropped from the registry (reset/dead/reaped) while cells were still attached:
  717. # the last one out owns the teardown.
  718. orphaned = kernel.attached == 0 and _KERNELS.get(key) is not kernel
  719. if orphaned:
  720. kernel.teardown()
  721. def _run_cell(kernel: SessionKernel, key: tuple, code: str, *, task_id: str, child_python: str,
  722. child_cwd: str, sandbox_tools: frozenset, timeout: int, max_tool_calls: int,
  723. is_interrupted, exec_start: float, state_reset: bool) -> str:
  724. reused = kernel.proc is not None
  725. # Captured on the calling thread BEFORE the cell runs (the snapshot a per-call RPC thread
  726. # would get) and installed on the kernel so RPC dispatches under THIS cell's identity.
  727. authority = CellAuthority(task_id)
  728. with kernel.lock:
  729. try:
  730. if kernel.proc is None:
  731. _spawn(kernel, task_id=task_id, child_python=child_python, child_cwd=child_cwd,
  732. sandbox_tools=sandbox_tools, max_tool_calls=max_tool_calls)
  733. assert kernel.proc is not None and kernel.proc.stdin is not None
  734. # Per-cell tool budget: the RPC loop enforces counter < max; reset without restarting.
  735. kernel.tool_call_counter[0] = 0
  736. kernel.cell_log_start = len(kernel.tool_call_log)
  737. kernel.raw.drain(), kernel.stderr.drain() # raw output leaked between cells belongs to no cell
  738. kernel.cell_authority = authority
  739. kernel.proc.stdin.write((json.dumps({"id": uuid.uuid4().hex, "code": code}) + "\n").encode("utf-8"))
  740. kernel.proc.stdin.flush()
  741. status, payload = _await_cell(kernel, timeout, is_interrupted)
  742. result = _cell_result(
  743. kernel, key, status, payload,
  744. timeout=timeout, sandbox_tools=sandbox_tools, reused=reused,
  745. state_reset=state_reset, exec_start=exec_start,
  746. )
  747. return json.dumps(result, ensure_ascii=False)
  748. except Exception as exc: # pragma: no cover - defensive parity with per-call
  749. from tools.code_execution_tool import _error_result
  750. logger.error("session kernel failed: %s: %s", type(exc).__name__, exc, exc_info=True)
  751. _REGISTRY.discard(key, kernel)
  752. return _error_result(str(exc), tool_calls_made=kernel.tool_call_counter[0],
  753. duration=round(time.monotonic() - exec_start, 2))
  754. finally:
  755. # The cell has settled on every path: its tool authority retires with it, so
  756. # nothing the cell left running can dispatch under it.
  757. authority.retire()