code_kernel_remote.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433
  1. """Session-persistent kernels for REMOTE terminal backends (docker/ssh/modal).
  2. Remote backends offer one primitive — ``env.execute(cmd)``, run-to-completion
  3. — so the three things the local kernel gets from owning a child are rebuilt:
  4. a detached runner (``nohup ... &``, PID recorded, ``kill -0`` probed per cell);
  5. a file-based CELL protocol in the kernel dir (``cell_req_NNNNNN.json`` /
  6. ``cell_res_NNNNNN.json``), sibling to the unchanged file-based TOOL-RPC protocol
  7. (req_/res_) whose host-side ``_rpc_poll_loop`` starts per cell with the calling
  8. thread's context (= per-cell tool authority); and death detection — a failed
  9. liveness probe reads as *kernel died: state lost* and the next call respawns,
  10. never a hung poll (every wait is bounded by the cell timeout).
  11. Same invariants as local: owner = approval session key with the ``::child::``
  12. qualifier (one resolver in tools.code_kernel), same generated tool stubs, same
  13. output post-processing in the caller. ``reset=true`` kills and respawns. Spawn
  14. failure fails OPEN to the per-call path with a note.
  15. """
  16. from __future__ import annotations
  17. import atexit
  18. import json
  19. import logging
  20. import secrets
  21. import shlex
  22. import threading
  23. import time
  24. import uuid
  25. from dataclasses import dataclass, field
  26. from typing import Any, Callable, Dict, List, Optional, Tuple
  27. from tools.code_kernel import RUNNER_CELL_SOURCE, KernelRegistry
  28. logger = logging.getLogger(__name__)
  29. # Host poll interval for a cell result file; each poll is one env.execute round-trip
  30. # (0.1-0.4s on ssh/docker), so this is a floor, not a rate.
  31. _CELL_POLL_INTERVAL = 0.5
  32. # The remote runner: a forever-loop that polls for cell request files, execs them in one
  33. # persistent namespace, writes response files. Pure files + stdlib only (transport-agnostic);
  34. # cells and tool-RPC share the kernel dir under distinct prefixes.
  35. REMOTE_KERNEL_RUNNER_SOURCE = '''\
  36. """Auto-generated Hermes REMOTE session-kernel runner (file cell protocol)."""
  37. import contextlib
  38. import io
  39. import json
  40. import os
  41. import sys
  42. import time
  43. import traceback
  44. KDIR = os.environ["HERMES_KERNEL_DIR"]
  45. CELLS = os.path.join(KDIR, "cells")
  46. _CAPTURE_LIMIT = {capture_limit}
  47. IDLE_EXIT_SECONDS = {idle_exit}
  48. {cell_source}
  49. def main():
  50. execution_count = 0
  51. last_activity = time.time()
  52. while True:
  53. pending = sorted(
  54. f for f in os.listdir(CELLS)
  55. if f.startswith("cell_req_") and f.endswith(".json")
  56. )
  57. if not pending:
  58. if time.time() - last_activity > IDLE_EXIT_SECONDS:
  59. return # self-reap: nobody is talking to us anymore
  60. time.sleep(0.2)
  61. continue
  62. for name in pending:
  63. req_path = os.path.join(CELLS, name)
  64. try:
  65. with open(req_path, "r", encoding="utf-8") as f:
  66. request = json.load(f)
  67. except Exception:
  68. # Partially-written request (ship in progress): retry next tick.
  69. continue
  70. os.remove(req_path)
  71. last_activity = time.time()
  72. execution_count += 1
  73. payload, _ = run_cell(request, execution_count)
  74. res_name = name.replace("cell_req_", "cell_res_")
  75. tmp = os.path.join(CELLS, res_name + ".tmp")
  76. # Cell results carry the executed code's output: owner-only, even if
  77. # the process umask is permissive.
  78. fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
  79. with os.fdopen(fd, "w", encoding="utf-8") as f:
  80. json.dump(payload, f, ensure_ascii=False)
  81. os.replace(tmp, os.path.join(CELLS, res_name))
  82. if payload["status"] == "exit":
  83. return
  84. if __name__ == "__main__":
  85. main()
  86. '''
  87. def _sh(env, cmd: str, timeout: int = 15) -> str:
  88. """Run *cmd* on the remote from ``/`` and return its output text."""
  89. result = env.execute(cmd, cwd="/", timeout=timeout)
  90. return (result.get("output", "") if isinstance(result, dict) else "") or ""
  91. @dataclass
  92. class RemoteKernel:
  93. """Host-side record of one detached remote kernel process."""
  94. env: Any
  95. env_type: str
  96. kernel_dir: str
  97. pid: str
  98. rpc_token: str
  99. owner: str
  100. last_used: float = field(default_factory=time.monotonic)
  101. execution_count: int = 0
  102. cell_seq: int = 0
  103. # Cells currently running on this kernel. Reap/evict skip attached
  104. # kernels: killing one mid-cell tears the runner out from under a live
  105. # poll loop (same guard as tools.code_kernel, hermes-agent#101861).
  106. attached: int = 0
  107. # Owned by a live delegate_task child: exempt from LRU eviction (the child's teardown disposes it).
  108. pinned: bool = False
  109. def sh(self, cmd: str, timeout: int = 15) -> str:
  110. return _sh(self.env, cmd, timeout)
  111. def is_alive(self) -> bool:
  112. """Bounded liveness probe: kill -0 through the transport. Any transport
  113. failure counts as dead — a dropped ssh connection and a dead runner are
  114. indistinguishable from here, and both have the same correct answer (respawn)."""
  115. try:
  116. return "ALIVE" in self.sh(f"kill -0 {shlex.quote(self.pid)} 2>/dev/null && echo ALIVE")
  117. except Exception:
  118. return False
  119. def kill(self) -> None:
  120. """Best-effort kill of the runner and its subprocesses, then rm -rf."""
  121. q_pid = shlex.quote(self.pid)
  122. for cmd, failure in (
  123. # Kill the runner's children if the shell gave it a group, then the PID itself.
  124. (f"pkill -TERM -P {q_pid} 2>/dev/null; kill {q_pid} 2>/dev/null; true",
  125. "remote kernel kill failed (transport?)"),
  126. (f"rm -rf {shlex.quote(self.kernel_dir)}", "remote kernel dir cleanup failed"),
  127. ):
  128. try:
  129. self.sh(cmd)
  130. except Exception:
  131. logger.debug(failure, exc_info=True)
  132. def _kernel_key(owner: str, env_type: str, task_env_id: str, sandbox_tools: frozenset) -> tuple:
  133. """The hermes_tools stub module is generated from ``sandbox_tools`` once, at spawn, so a kernel
  134. is only reusable by calls with the SAME tool set; a different set gets its own kernel."""
  135. return (owner, "remote", env_type, task_env_id, tuple(sorted(sandbox_tools)))
  136. # Registry + lock shared-shape with code_kernel; teardown runs outside the lock.
  137. _REGISTRY = KernelRegistry(lambda kernel: kernel.kill())
  138. _REMOTE_KERNELS: dict[tuple, RemoteKernel] = _REGISTRY.kernels
  139. def shutdown_all_remote_kernels() -> None:
  140. _REGISTRY.shutdown()
  141. def shutdown_remote_kernels_for_owner(owner: str) -> None:
  142. """Session-boundary disposal — wired to the same clear_session hook as
  143. local kernels, so /new and session close reap both kinds."""
  144. if owner:
  145. _REGISTRY.shutdown(owner)
  146. def shutdown_remote_kernels_where(owner_matches: Callable[[str], bool]) -> None:
  147. """Dispose every remote kernel whose owner satisfies the predicate (a finished child's kernels)."""
  148. _REGISTRY.shutdown(owner_matches=owner_matches)
  149. def _reap_unlocked(idle_timeout: int) -> list[RemoteKernel]:
  150. """Pop idle-expired, unattached remote kernels; caller tears them down outside the lock. The
  151. runner self-exits after the same idle window, so this clears the HOST-side entry — without it
  152. the map grew one entry per never-revisited (owner, env_type, task_env_id) for the gateway's life."""
  153. now = time.monotonic()
  154. doomed = [key for key, kernel in _REMOTE_KERNELS.items()
  155. if kernel.attached == 0 and now - kernel.last_used > idle_timeout]
  156. return [_REMOTE_KERNELS.pop(key) for key in doomed]
  157. def _evict_over_cap_unlocked(keep: tuple) -> list[RemoteKernel]:
  158. """Pop least-recently-used unattached remote kernels beyond the process-wide cap (the same
  159. ``max_session_kernels`` bound as local kernels, applied independently to this map)."""
  160. from tools.code_kernel import _lifecycle_limits
  161. cap, _ = _lifecycle_limits()
  162. unpinned = [key for key in _REMOTE_KERNELS if not _REMOTE_KERNELS[key].pinned]
  163. if len(unpinned) <= cap:
  164. return []
  165. by_age = sorted((key for key in unpinned if key != keep and _REMOTE_KERNELS[key].attached == 0),
  166. key=lambda key: _REMOTE_KERNELS[key].last_used)
  167. return [_REMOTE_KERNELS.pop(key) for key in by_age[: len(unpinned) - cap]]
  168. atexit.register(shutdown_all_remote_kernels)
  169. def _spawn_remote_kernel(env, env_type: str, owner: str, task_env_id: str,
  170. sandbox_tools: frozenset, *, idle_exit: int) -> Optional[RemoteKernel]:
  171. """Start a detached kernel runner on the remote. None on failure (dir removed)."""
  172. from tools.code_execution_rpc import _execute_checked, _private_dirs_cmd
  173. from tools.code_execution_tool import (
  174. MAX_STDOUT_BYTES, _ship_file_to_remote, _env_temp_dir,
  175. _ship_env_file_and_launch, generate_hermes_tools_module,
  176. )
  177. kernel_dir = f"{_env_temp_dir(env)}/hermes_rkernel_{uuid.uuid4().hex[:12]}"
  178. q_dir = shlex.quote(kernel_dir)
  179. kernel = None
  180. try:
  181. # Private dirs: the kernel dir lives under a shared temp dir and carries
  182. # the RPC token (in req files), tool results, and cell code/output.
  183. # Fail closed on setup failure rather than ship secrets into a dir that
  184. # stayed permissive.
  185. _execute_checked(env, _private_dirs_cmd(kernel_dir, f"{kernel_dir}/cells",
  186. f"{kernel_dir}/rpc"),
  187. "remote kernel dir setup", timeout=15)
  188. rpc_token = secrets.token_urlsafe(32)
  189. _ship_file_to_remote(env, f"{kernel_dir}/kernel_runner.py", REMOTE_KERNEL_RUNNER_SOURCE.format(
  190. cell_source=RUNNER_CELL_SOURCE, capture_limit=MAX_STDOUT_BYTES, idle_exit=idle_exit))
  191. _ship_file_to_remote(env, f"{kernel_dir}/hermes_tools.py",
  192. generate_hermes_tools_module(list(sandbox_tools), transport="file"))
  193. # kernel.env is removed after sourcing: the runner's env keeps the
  194. # values, so the token file need not sit at rest for the kernel's
  195. # lifetime. runner.log is pre-created 600 so the launch redirect never
  196. # lands at the remote's default umask. The inner `&` stays inside the
  197. # subshell where `$!` resolves to the runner pid.
  198. launch_cmd = _ship_env_file_and_launch(
  199. env, kernel_dir, "kernel.env",
  200. "rm -f ./kernel.env && touch runner.log && chmod 600 runner.log && "
  201. '{ nohup python3 kernel_runner.py > runner.log 2>&1 & echo "PID:$!"; }',
  202. rpc_dir=f"{kernel_dir}/rpc", rpc_token=rpc_token,
  203. HERMES_KERNEL_DIR=kernel_dir, PYTHONPATH=kernel_dir)
  204. started = _sh(env, launch_cmd, timeout=20)
  205. pid = next((line.strip()[4:].strip() for line in started.splitlines()
  206. if line.strip().startswith("PID:")), "")
  207. if not pid.isdigit():
  208. logger.warning("remote kernel spawn returned no PID: %r", started)
  209. else:
  210. candidate = RemoteKernel(env=env, env_type=env_type, kernel_dir=kernel_dir,
  211. pid=pid, rpc_token=rpc_token, owner=owner)
  212. if candidate.is_alive():
  213. kernel = candidate
  214. else:
  215. # Died instantly (missing python3 was pre-checked by the caller,
  216. # so this is unexpected) — surface the runner log.
  217. try:
  218. logger.warning("remote kernel died at spawn: %s",
  219. _sh(env, f"cat {q_dir}/runner.log", timeout=10)[:500])
  220. except Exception:
  221. pass
  222. except Exception:
  223. logger.warning("remote kernel spawn failed", exc_info=True)
  224. if kernel is None:
  225. try:
  226. _sh(env, f"rm -rf {q_dir}")
  227. except Exception:
  228. pass
  229. return kernel
  230. def _acquire_remote_kernel(env, env_type: str, owner: str, task_env_id: str,
  231. sandbox_tools: frozenset, *, reset: bool,
  232. idle_exit: int) -> tuple[Optional[RemoteKernel], bool, bool, bool]:
  233. """Find/respawn the owner's kernel: (kernel|None, reused, state_reset, state_lost); reaps
  234. idle-expired entries on the way in."""
  235. key = _kernel_key(owner, env_type, task_env_id, sandbox_tools)
  236. state_lost = state_reset = False
  237. with _REGISTRY.lock:
  238. expired = _reap_unlocked(idle_exit)
  239. kernel = _REMOTE_KERNELS.get(key)
  240. for doomed in expired:
  241. doomed.kill()
  242. if kernel is not None and reset:
  243. _REGISTRY.discard(key, kernel)
  244. kernel, state_reset = None, True
  245. if kernel is not None and not kernel.is_alive():
  246. # Transport drop, container restart, self-reaped on idle, OOM — all
  247. # the same answer: report the loss, respawn fresh (kill is then only
  248. # best-effort dir cleanup; the process is already gone).
  249. _REGISTRY.discard(key, kernel)
  250. kernel, state_lost = None, True
  251. reused = kernel is not None
  252. if kernel is None:
  253. kernel = _spawn_remote_kernel(env, env_type, owner, task_env_id, sandbox_tools, idle_exit=idle_exit)
  254. if kernel is not None:
  255. from agent.delegation_context import is_delegated_child_context
  256. kernel.pinned = is_delegated_child_context()
  257. with _REGISTRY.lock:
  258. _REMOTE_KERNELS[key] = kernel
  259. return kernel, reused, state_reset, state_lost
  260. def _run_remote_cell(kernel: RemoteKernel, code: str, timeout: int) -> tuple[str, dict[str, Any]]:
  261. """Ship one cell request and poll for its result: (cell status, payload)."""
  262. from tools.code_execution_tool import _ship_file_to_remote
  263. kernel.cell_seq += 1
  264. seq = f"{kernel.cell_seq:06d}"
  265. q_cells, q_res = shlex.quote(f"{kernel.kernel_dir}/cells"), shlex.quote(f"cell_res_{seq}.json")
  266. # One round-trip: tmp write + rename publishes the request atomically.
  267. _ship_file_to_remote(kernel.env, f"{kernel.kernel_dir}/cells/cell_req_{seq}.json",
  268. json.dumps({"id": seq, "code": code}, ensure_ascii=False), atomic=True)
  269. deadline = time.monotonic() + timeout
  270. while time.monotonic() < deadline:
  271. try:
  272. body = kernel.sh(f"cat {q_cells}/{q_res} 2>/dev/null", timeout=20).strip()
  273. except Exception:
  274. # One flaky round-trip is not kernel death; liveness decides.
  275. body = ""
  276. if body:
  277. try:
  278. payload = json.loads(body)
  279. status = payload.get("status", "error")
  280. except ValueError:
  281. payload, status = {}, "protocol-error"
  282. try:
  283. kernel.sh(f"rm -f {q_cells}/{q_res}", timeout=10)
  284. except Exception:
  285. # Best-effort: the cell already ran, so raising here would send
  286. # the caller to its per-call fallback and run the code twice.
  287. # A leftover result file is harmless (seq is monotonic).
  288. logger.debug("remote kernel: cell result cleanup failed", exc_info=True)
  289. return status, payload
  290. time.sleep(_CELL_POLL_INTERVAL)
  291. return "timeout", {}
  292. def execute_in_remote_kernel(
  293. code: str, *, env, env_type: str, task_env_id: str, sandbox_tools: frozenset,
  294. timeout: int, max_tool_calls: int, reset: bool, idle_exit: int = 1800,
  295. ) -> Optional[dict[str, Any]]:
  296. """Run one cell in the owner's remote kernel. Returns the raw cell result dict (caller
  297. post-processes output), or ``None`` when no kernel could be spawned (caller falls open to
  298. per-call). ``state_lost``/``state_reset``/``reused`` ride in the ``kernel`` sub-dict."""
  299. from tools.code_kernel import _resolve_owner
  300. owner = _resolve_owner(task_env_id)
  301. kernel, reused, state_reset, state_lost = _acquire_remote_kernel(
  302. env, env_type, owner, task_env_id, sandbox_tools, reset=reset, idle_exit=idle_exit)
  303. if kernel is None:
  304. return None # fail open to per-call
  305. key = _kernel_key(owner, env_type, task_env_id, sandbox_tools)
  306. kernel.last_used = time.monotonic()
  307. with _REGISTRY.lock:
  308. kernel.attached += 1
  309. evicted = _evict_over_cap_unlocked(keep=key)
  310. for doomed in evicted:
  311. doomed.kill()
  312. try:
  313. return _run_attached_cell(kernel, key, code, env=env, task_env_id=task_env_id,
  314. sandbox_tools=sandbox_tools, timeout=timeout, max_tool_calls=max_tool_calls,
  315. reused=reused, state_reset=state_reset, state_lost=state_lost)
  316. finally:
  317. with _REGISTRY.lock:
  318. kernel.attached -= 1
  319. kernel.last_used = time.monotonic()
  320. def _run_attached_cell(kernel: RemoteKernel, key: tuple, code: str, *, env, task_env_id: str,
  321. sandbox_tools: frozenset, timeout: int, max_tool_calls: int,
  322. reused: bool, state_reset: bool, state_lost: bool) -> dict[str, Any]:
  323. from tools.code_execution_rpc import tool_errors_since
  324. from tools.code_execution_tool import _rpc_poll_loop
  325. from tools.thread_context import propagate_context_to_thread
  326. # Clean stale tool-RPC requests from a previous cell before arming this cell's poll loop, so
  327. # a background thread the last cell leaked cannot smuggle a call into this authority window.
  328. q_rpc = shlex.quote(kernel.kernel_dir + '/rpc')
  329. try:
  330. kernel.sh(f"rm -f {q_rpc}/req_* {q_rpc}/res_*", timeout=10)
  331. except Exception:
  332. pass
  333. tool_call_counter, tool_call_log, stop_event = [0], [], threading.Event()
  334. # Per-cell RPC thread carrying THIS call's approval/session context — the remote analogue
  335. # of CellAuthority: authority lives exactly as long as the cell's poll loop.
  336. rpc_thread = threading.Thread(
  337. target=propagate_context_to_thread(_rpc_poll_loop), daemon=True,
  338. args=(env, f"{kernel.kernel_dir}/rpc", task_env_id, tool_call_log, tool_call_counter,
  339. max_tool_calls, sandbox_tools, stop_event, kernel.rpc_token))
  340. rpc_thread.start()
  341. cell_status, cell_payload = "no-result", {}
  342. try:
  343. cell_status, cell_payload = _run_remote_cell(kernel, code, timeout)
  344. except Exception:
  345. # The atomic ship is the only remote call here that can raise (the poll
  346. # and result cleanup are best-effort), so the request never reached the runner and the
  347. # caller's per-call fallback runs the code exactly once. Kill the
  348. # kernel as the timeout path does: leaving it registered would let the
  349. # next call reuse a kernel whose state silently missed this cell.
  350. _REGISTRY.discard(key, kernel)
  351. raise
  352. finally:
  353. stop_event.set()
  354. rpc_thread.join(timeout=5)
  355. kernel_info: dict[str, Any] = {"reused": reused, "remote": True}
  356. result: dict[str, Any] = {
  357. "status": "error", "stdout": cell_payload.get("stdout", ""), "stderr": cell_payload.get("stderr", ""),
  358. "traceback": cell_payload.get("traceback", ""), "tool_calls_made": tool_call_counter[0], "kernel": kernel_info,
  359. "tool_errors": tool_errors_since(tool_call_log),
  360. }
  361. if cell_status in ("timeout", "protocol-error", "no-result"):
  362. # No safe way to interrupt one cell in place (same contract as local): kill, report, respawn.
  363. _REGISTRY.discard(key, kernel)
  364. if cell_status == "timeout":
  365. result["status"] = "timeout"
  366. kernel_info.update(ended=True, state_lost=True, note=(
  367. "Cell timed out; the remote session kernel was killed and its state was lost. The next call "
  368. "starts a fresh kernel." if cell_status == "timeout"
  369. else "Remote kernel protocol failure; kernel killed, state lost."))
  370. return result
  371. if cell_status == "exit":
  372. _REGISTRY.discard(key, kernel)
  373. kernel_info["ended"] = True
  374. kernel.execution_count = kernel_info["execution_count"] = int(cell_payload.get("execution_count", 0) or 0)
  375. if cell_status in ("ok", "exit"):
  376. result["status"] = "success"
  377. result["stdout_clipped"] = bool(cell_payload.get("stdout_clipped"))
  378. result["stderr_clipped"] = bool(cell_payload.get("stderr_clipped"))
  379. if state_reset:
  380. kernel_info["state_reset"] = True
  381. if state_lost:
  382. kernel_info.update(state_lost=True, note=(
  383. "The previous remote kernel was gone (transport drop, container "
  384. "restart, or idle self-exit); state from earlier calls was lost and a fresh kernel was started."))
  385. if cell_status == "error" and result["traceback"]:
  386. result["error"] = result["traceback"].strip().splitlines()[-1]
  387. return result