modal.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. """Modal cloud execution environment using the native Modal SDK directly
  2. (``Sandbox.create()`` + ``Sandbox.exec()``) with persistent snapshots across sessions."""
  3. import asyncio
  4. import base64
  5. import io
  6. import logging
  7. import shlex
  8. import tarfile
  9. import threading
  10. from pathlib import Path
  11. from typing import Any, Callable, Optional
  12. from hermes_constants import get_hermes_home
  13. from tools.environments.base import BaseEnvironment, _load_json_store, _update_json_store
  14. from tools.environments.base_output import _ThreadedProcessHandle
  15. from tools.environments.file_sync import (
  16. FileSyncManager, iter_sync_files, quoted_mkdir_command, quoted_rm_command, unique_parent_dirs)
  17. from tools.environments.remote_common import bash_argv, ensure_lazy_dep
  18. logger = logging.getLogger(__name__)
  19. def _snapshot_store() -> Path:
  20. # Resolved per call: the multiplexed gateway serves every profile from one process, so an
  21. # import-time path would keep every profile's snapshots in the launch profile's home.
  22. return get_hermes_home() / "modal_snapshots.json"
  23. def _load_snapshots() -> dict:
  24. return _load_json_store(_snapshot_store())
  25. def _update_snapshots(update: Callable[[dict], None]) -> None:
  26. _update_json_store(_snapshot_store(), update)
  27. def _get_snapshot_restore_candidate(task_id: str) -> tuple[str | None, bool]:
  28. """Return (snapshot_id, from_legacy_key); the namespaced key wins over the legacy bare task id."""
  29. snapshots = _load_snapshots()
  30. for key, legacy in ((f"direct:{task_id}", False), (task_id, True)):
  31. snapshot_id = snapshots.get(key)
  32. if isinstance(snapshot_id, str) and snapshot_id:
  33. return snapshot_id, legacy
  34. return None, False
  35. def _store_direct_snapshot(task_id: str, snapshot_id: str) -> None:
  36. def update(snapshots: dict) -> None:
  37. snapshots[f"direct:{task_id}"] = snapshot_id
  38. snapshots.pop(task_id, None)
  39. _update_snapshots(update)
  40. def _delete_direct_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
  41. def update(snapshots: dict) -> None:
  42. stale = [k for k in (f"direct:{task_id}", task_id)
  43. if snapshots.get(k) is not None and snapshot_id in (None, snapshots[k])]
  44. for key in stale:
  45. snapshots.pop(key)
  46. _update_snapshots(update)
  47. def _resolve_modal_image(image_spec: Any) -> Any:
  48. """Convert registry references or snapshot ids into Modal image objects. Registry images
  49. get pip repaired (ensurepip) before Modal's bootstrap; ubuntu/debian also get python3."""
  50. ensure_lazy_dep("modal")
  51. import modal as _modal
  52. if not isinstance(image_spec, str):
  53. return image_spec
  54. if image_spec.startswith("im-"):
  55. return _modal.Image.from_id(image_spec)
  56. setup_commands = [
  57. "RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
  58. "python -m ensurepip --upgrade --default-pip 2>/dev/null || true"]
  59. if any(base in image_spec.lower() for base in ("ubuntu", "debian")):
  60. setup_commands.insert(0,
  61. "RUN apt-get update -qq && apt-get install -y -qq python3 python3-venv > /dev/null 2>&1 || true")
  62. return _modal.Image.from_registry(image_spec, setup_dockerfile_commands=setup_commands)
  63. async def _stream_stdin(proc, payload: str, chunk_size: int) -> None:
  64. """Write byte-exact UTF-8 payload chunks to ``proc.stdin``, then EOF."""
  65. data = payload.encode("utf-8", "surrogateescape")
  66. for offset in range(0, len(data), chunk_size):
  67. proc.stdin.write(data[offset:offset + chunk_size])
  68. await proc.stdin.drain.aio()
  69. proc.stdin.write_eof()
  70. await proc.stdin.drain.aio()
  71. def _as_text(value) -> str:
  72. return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value
  73. class _AsyncWorker:
  74. """Background thread with its own event loop for async-safe Modal calls."""
  75. def __init__(self):
  76. self._loop: Optional[asyncio.AbstractEventLoop] = None
  77. self._thread: Optional[threading.Thread] = None
  78. self._started = threading.Event()
  79. def start(self):
  80. def _run_loop():
  81. self._loop = asyncio.new_event_loop()
  82. asyncio.set_event_loop(self._loop)
  83. self._started.set()
  84. self._loop.run_forever()
  85. self._thread = threading.Thread(target=_run_loop, daemon=True)
  86. self._thread.start()
  87. self._started.wait(timeout=30)
  88. def run_coroutine(self, coro, timeout=600):
  89. from agent.async_utils import safe_schedule_threadsafe
  90. # safe_schedule_threadsafe closes the coroutine and returns None for a missing/closed loop.
  91. future = safe_schedule_threadsafe(coro, self._loop)
  92. if future is None:
  93. raise RuntimeError("AsyncWorker loop is not running")
  94. return future.result(timeout=timeout)
  95. def stop(self):
  96. if self._loop and self._loop.is_running():
  97. self._loop.call_soon_threadsafe(self._loop.stop)
  98. if self._thread:
  99. self._thread.join(timeout=10)
  100. class ModalEnvironment(BaseEnvironment):
  101. """Modal cloud execution via native Modal sandboxes: spawn-per-call via _ThreadedProcessHandle
  102. wrapping async SDK calls, cancel_fn wired to sandbox.terminate for interrupt support."""
  103. _stdin_mode = "payload"
  104. _snapshot_timeout = 60 # Modal cold starts can be slow
  105. # Modal SDK stdin buffer limit: the command-router path allows 16 MB but the legacy server
  106. # path caps at 2 MB, so chunks stay under 2 MB and each is flushed individually via drain().
  107. _STDIN_CHUNK_SIZE = 1 * 1024 * 1024
  108. def __init__(self, image: str, cwd: str = "/root", timeout: int = 60,
  109. modal_sandbox_kwargs: Optional[dict[str, Any]] = None,
  110. persistent_filesystem: bool = True, task_id: str = "default"):
  111. super().__init__(cwd=cwd, timeout=timeout)
  112. self._persistent, self._task_id = persistent_filesystem, task_id
  113. self._sandbox = self._app = None
  114. self._worker = _AsyncWorker()
  115. self._sync_manager: FileSyncManager | None = None # initialized after sandbox creation
  116. restored_snapshot_id, restored_from_legacy_key = (
  117. _get_snapshot_restore_candidate(self._task_id) if self._persistent else (None, False))
  118. if restored_snapshot_id:
  119. logger.info("Modal: restoring from snapshot %s", restored_snapshot_id[:20])
  120. ensure_lazy_dep("modal")
  121. import modal as _modal
  122. self._worker.start()
  123. def _create(image_spec: Any) -> None:
  124. async def _create_sandbox():
  125. app = await _modal.App.lookup.aio("hermes-agent", create_if_missing=True)
  126. create_kwargs = dict(modal_sandbox_kwargs or {})
  127. sandbox = await _modal.Sandbox.create.aio(
  128. "sleep", "infinity", image=image_spec, app=app,
  129. timeout=int(create_kwargs.pop("timeout", 3600)), **create_kwargs)
  130. return app, sandbox
  131. self._app, self._sandbox = self._worker.run_coroutine(_create_sandbox(), timeout=300)
  132. try:
  133. try:
  134. _create(_resolve_modal_image(restored_snapshot_id or image))
  135. except Exception as exc:
  136. if not restored_snapshot_id:
  137. raise
  138. logger.warning("Modal: failed to restore snapshot %s, retrying with base image: %s",
  139. restored_snapshot_id[:20], exc)
  140. _delete_direct_snapshot(self._task_id, restored_snapshot_id)
  141. _create(_resolve_modal_image(image))
  142. else:
  143. if restored_snapshot_id and restored_from_legacy_key:
  144. _store_direct_snapshot(self._task_id, restored_snapshot_id)
  145. except Exception:
  146. self._worker.stop()
  147. raise
  148. logger.info("Modal: sandbox created (task=%s)", self._task_id)
  149. self._sync_manager = FileSyncManager(
  150. get_files_fn=lambda: iter_sync_files("/root/.hermes"),
  151. upload_fn=self._modal_upload, delete_fn=self._modal_delete,
  152. bulk_upload_fn=self._modal_bulk_upload, bulk_download_fn=self._modal_bulk_download)
  153. self._sync_manager.sync(force=True)
  154. self.init_session()
  155. def _exec(self, cmd: str, *, timeout: int, stdin: str | None = None, fail_label: str | None = None,
  156. capture: bool = False):
  157. """Run ``bash -c cmd`` in the sandbox. ``stdin`` is streamed in chunks; ``capture`` returns
  158. stdout; ``fail_label`` turns a non-zero exit into RuntimeError (with stderr unless capturing)."""
  159. async def _run():
  160. proc = await self._sandbox.exec.aio("bash", "-c", cmd)
  161. if stdin is not None:
  162. await _stream_stdin(proc, stdin, self._STDIN_CHUNK_SIZE)
  163. data = await proc.stdout.read.aio() if capture else None
  164. exit_code = await proc.wait.aio()
  165. if fail_label and exit_code != 0:
  166. detail = "" if capture else f": {await proc.stderr.read.aio()}"
  167. raise RuntimeError(f"Modal {fail_label} failed (exit {exit_code}){detail}")
  168. return data
  169. return self._worker.run_coroutine(_run(), timeout=timeout)
  170. def _modal_upload(self, host_path: str, remote_path: str) -> None:
  171. """Upload a single file via base64 piped through stdin."""
  172. cmd = f"mkdir -p {shlex.quote(str(Path(remote_path).parent))} && base64 -d > {shlex.quote(remote_path)}"
  173. self._exec(cmd, stdin=base64.b64encode(Path(host_path).read_bytes()).decode("ascii"), timeout=30)
  174. def _modal_bulk_upload(self, files: list[tuple[str, str]]) -> None:
  175. """Upload many files as one in-memory gzipped tar streamed through stdin
  176. into ``base64 -d | tar xzf -``, avoiding the SDK's 64 KB exec-arg limit."""
  177. if not files:
  178. return
  179. buf = io.BytesIO()
  180. with tarfile.open(fileobj=buf, mode="w:gz") as tar:
  181. for host_path, remote_path in files:
  182. tar.add(host_path, arcname=remote_path.lstrip("/"))
  183. payload = base64.b64encode(buf.getvalue()).decode("ascii")
  184. cmd = f"{quoted_mkdir_command(unique_parent_dirs(files))} && base64 -d | tar xzf - -C /"
  185. self._exec(cmd, stdin=payload, timeout=120, fail_label="bulk upload")
  186. def _modal_bulk_download(self, dest: Path) -> None:
  187. """Download remote .hermes/ as a tar archive (sandboxes run as root, so /root/.hermes)."""
  188. # --exclude: live sockets cannot be archived ("socket ignored") and must not fail the download.
  189. data = self._exec("tar cf - --exclude='*.sock' -C / root/.hermes", timeout=120, fail_label="bulk download", capture=True)
  190. dest.write_bytes(data.encode() if isinstance(data, str) else data)
  191. def _modal_delete(self, remote_paths: list[str]) -> None:
  192. self._exec(quoted_rm_command(remote_paths), timeout=15)
  193. def _before_execute(self) -> None:
  194. self._sync_manager.sync() # rate-limited internally
  195. def _run_bash(self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None):
  196. sandbox, worker = self._sandbox, self._worker
  197. def cancel():
  198. worker.run_coroutine(sandbox.terminate.aio(), timeout=15)
  199. def exec_fn() -> tuple[str, int]:
  200. async def _do():
  201. process = await sandbox.exec.aio(*bash_argv(cmd_string, login), timeout=timeout)
  202. if stdin_data is not None:
  203. await _stream_stdin(process, stdin_data, self._STDIN_CHUNK_SIZE)
  204. stdout = _as_text(await process.stdout.read.aio())
  205. stderr = _as_text(await process.stderr.read.aio())
  206. exit_code = await process.wait.aio()
  207. return "\n".join(part for part in (stdout, stderr) if part), exit_code
  208. return worker.run_coroutine(_do(), timeout=timeout + 30)
  209. return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
  210. def cleanup(self):
  211. """Snapshot the filesystem (if persistent) then stop the sandbox."""
  212. if self._sandbox is None:
  213. return
  214. if self._sync_manager:
  215. logger.info("Modal: syncing files from sandbox...")
  216. self._sync_manager.sync_back()
  217. if self._persistent:
  218. async def _snapshot():
  219. # ttl=None: the SDK default (30 days) would silently expire an idle
  220. # persistent sandbox's snapshot; Hermes owns its lifetime.
  221. return (await self._sandbox.snapshot_filesystem.aio(ttl=None)).object_id
  222. try:
  223. snapshot_id = self._worker.run_coroutine(_snapshot(), timeout=60)
  224. except Exception:
  225. snapshot_id = None # snapshot errors are non-fatal; the sandbox is still terminated
  226. if snapshot_id:
  227. try:
  228. _store_direct_snapshot(self._task_id, snapshot_id)
  229. logger.info("Modal: saved filesystem snapshot %s for task %s", snapshot_id[:20], self._task_id)
  230. except Exception as e:
  231. logger.warning("Modal: filesystem snapshot failed: %s", e)
  232. try:
  233. self._worker.run_coroutine(self._sandbox.terminate.aio(), timeout=15)
  234. except Exception:
  235. pass
  236. finally:
  237. self._worker.stop()
  238. self._sandbox = self._app = None