daytona.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207
  1. """Daytona cloud execution environment.
  2. Runs commands in Daytona cloud sandboxes via the Python SDK. Persistent mode stops
  3. the sandbox on cleanup and resumes it next time, preserving the filesystem.
  4. """
  5. import contextlib
  6. import logging
  7. import math
  8. import os
  9. import shlex
  10. import threading
  11. from pathlib import Path, PurePosixPath
  12. from tools.environments.base import BaseEnvironment
  13. from tools.environments.base_output import _ThreadedProcessHandle
  14. from tools.environments.file_sync import (
  15. FileSyncManager, iter_sync_files, quoted_mkdir_command, quoted_rm_command, unique_parent_dirs)
  16. from tools.environments.remote_common import ensure_lazy_dep
  17. logger = logging.getLogger(__name__)
  18. class DaytonaEnvironment(BaseEnvironment):
  19. """Daytona cloud sandbox execution backend.
  20. Spawn-per-call via _ThreadedProcessHandle wrapping blocking SDK calls; cancel_fn
  21. is wired to sandbox.stop() for interrupts. Shell timeout wrapper kept (SDK timeout unreliable).
  22. """
  23. _stdin_mode = "payload"
  24. def __init__(self, image: str, cwd: str = "/home/daytona", timeout: int = 60, cpu: int = 1,
  25. memory: int = 5120, disk: int = 10240, persistent_filesystem: bool = True,
  26. task_id: str = "default"):
  27. super().__init__(cwd=cwd, timeout=timeout)
  28. ensure_lazy_dep("daytona")
  29. from daytona import Daytona, CreateSandboxFromImageParams, DaytonaError, Resources, SandboxState
  30. self._persistent, self._task_id, self._SandboxState = persistent_filesystem, task_id, SandboxState
  31. self._daytona = Daytona()
  32. self._sandbox = None
  33. self._lock = threading.Lock()
  34. memory_gib, disk_gib = max(1, math.ceil(memory / 1024)), max(1, math.ceil(disk / 1024))
  35. if disk_gib > 10:
  36. logger.warning("Daytona: requested disk (%dGB) exceeds platform limit (10GB). "
  37. "Capping to 10GB.", disk_gib)
  38. disk_gib = 10
  39. resources = Resources(cpu=cpu, memory=memory_gib, disk=disk_gib)
  40. labels, sandbox_name = {"hermes_task_id": task_id}, f"hermes-{task_id}"
  41. if self._persistent:
  42. try:
  43. self._sandbox = self._daytona.get(sandbox_name)
  44. self._sandbox.start()
  45. logger.info("Daytona: resumed sandbox %s for task %s", self._sandbox.id, task_id)
  46. except Exception as e:
  47. if not isinstance(e, DaytonaError): # DaytonaError == not found: silently fall through
  48. logger.warning("Daytona: failed to resume sandbox for task %s: %s", task_id, e)
  49. self._sandbox = None
  50. if self._sandbox is None:
  51. try:
  52. # SDK list() is a cursor-paginated iterator (offset pagination is gone).
  53. self._sandbox = next(iter(self._daytona.list(labels=labels, limit=1)), None)
  54. if self._sandbox is not None:
  55. self._sandbox.start()
  56. logger.info("Daytona: resumed legacy sandbox %s for task %s", self._sandbox.id, task_id)
  57. except Exception as e:
  58. logger.debug("Daytona: no legacy sandbox found for task %s: %s", task_id, e)
  59. self._sandbox = None
  60. if self._sandbox is None:
  61. self._sandbox = self._daytona.create(CreateSandboxFromImageParams(
  62. image=image, name=sandbox_name, labels=labels, auto_stop_interval=0, resources=resources))
  63. logger.info("Daytona: created sandbox %s for task %s", self._sandbox.id, task_id)
  64. self._remote_home = "/root"
  65. with contextlib.suppress(Exception):
  66. home = self._sandbox.process.exec("echo $HOME").result.strip()
  67. if home:
  68. self._remote_home = home
  69. if cwd in {"~", "/home/daytona"}:
  70. self.cwd = home
  71. logger.info("Daytona: resolved home to %s, cwd to %s", self._remote_home, self.cwd)
  72. self._sync_manager = FileSyncManager(
  73. get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
  74. upload_fn=self._daytona_upload, delete_fn=self._daytona_delete,
  75. bulk_upload_fn=self._daytona_bulk_upload, bulk_download_fn=self._daytona_bulk_download)
  76. self._sync_manager.sync(force=True)
  77. self.init_session()
  78. def _daytona_upload(self, host_path: str, remote_path: str) -> None:
  79. """Upload a single file via Daytona SDK."""
  80. # remote_path is a POSIX path on the sandbox; never run it through the
  81. # host's Path (Windows would mangle the separators into backslashes).
  82. parent = str(PurePosixPath(remote_path).parent)
  83. self._sandbox.process.exec(quoted_mkdir_command([parent]))
  84. self._sandbox.fs.upload_file(host_path, remote_path)
  85. def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:
  86. """Upload many files in one multipart POST via ``sandbox.fs.upload_files()``."""
  87. from daytona.common.filesystem import FileUpload
  88. if not files:
  89. return
  90. parents = unique_parent_dirs(files)
  91. if parents:
  92. self._sandbox.process.exec(quoted_mkdir_command(parents))
  93. self._sandbox.fs.upload_files(
  94. [FileUpload(source=host_path, destination=remote_path) for host_path, remote_path in files])
  95. def _daytona_bulk_download(self, dest: Path) -> None:
  96. """Download remote .hermes/ as a tar archive."""
  97. rel_base = f"{self._remote_home}/.hermes".lstrip("/")
  98. # PID-suffixed remote temp path avoids collisions if sync_back runs concurrently.
  99. remote_tar = f"/tmp/.hermes_sync.{os.getpid()}.tar" # no-tmp: ok — remote sandbox path
  100. # --exclude: live sockets cannot be archived ("socket ignored") and must not fail the download.
  101. self._sandbox.process.exec(
  102. f"tar cf {shlex.quote(remote_tar)} --exclude='*.sock' -C / {shlex.quote(rel_base)}")
  103. self._sandbox.fs.download_file(remote_tar, str(dest))
  104. with contextlib.suppress(Exception): # best-effort cleanup
  105. self._sandbox.process.exec(f"rm -f {shlex.quote(remote_tar)}")
  106. def _daytona_delete(self, remote_paths: list[str]) -> None:
  107. self._sandbox.process.exec(quoted_rm_command(remote_paths))
  108. def _ensure_sandbox_ready(self) -> None:
  109. """Restart sandbox if it was stopped (e.g., by a previous interrupt)."""
  110. self._sandbox.refresh_data()
  111. if self._sandbox.state in {self._SandboxState.STOPPED, self._SandboxState.ARCHIVED}:
  112. self._sandbox.start()
  113. logger.info("Daytona: restarted sandbox %s", self._sandbox.id)
  114. def _before_execute(self) -> None:
  115. with self._lock:
  116. self._ensure_sandbox_ready()
  117. self._sync_manager.sync()
  118. def _run_bash(self, cmd_string: str, *, login: bool = False, timeout: int = 120,
  119. stdin_data: str | None = None):
  120. sandbox, lock = self._sandbox, self._lock
  121. # Guarded by ``lock`` so cancel() and dispatch agree on whether the shell
  122. # has taken ownership of (opened + unlinked) the staged stdin file.
  123. state = {"cancelled": False, "staged": None, "dispatched": False}
  124. def scrub_staged(): # caller holds ``lock``
  125. if state["staged"] and not state["dispatched"]:
  126. # Uploaded but never dispatched: nothing else will unlink it.
  127. # Once dispatched the user shell rm's it before running cmd.
  128. with contextlib.suppress(Exception):
  129. sandbox.fs.delete_file(state["staged"])
  130. state["staged"] = None
  131. def cancel():
  132. with lock:
  133. state["cancelled"] = True
  134. scrub_staged()
  135. with contextlib.suppress(Exception):
  136. sandbox.stop()
  137. def exec_fn() -> tuple[str, int]:
  138. command = cmd_string
  139. if stdin_data: # empty stdin == no stdin, as on base (heredoc skipped it)
  140. with lock:
  141. if state["cancelled"]:
  142. return ("", 130)
  143. remote_stdin = self._staged_stdin_path()
  144. sandbox.fs.upload_file(stdin_data.encode("utf-8", "surrogateescape"), remote_stdin)
  145. with lock:
  146. state["staged"] = remote_stdin
  147. sandbox.fs.set_file_permissions(remote_stdin, mode="600")
  148. command = self._redirect_stdin_from_file(cmd_string, remote_stdin)
  149. shell_cmd = f"bash {'-l ' if login else ''}-c {shlex.quote(command)}"
  150. with lock:
  151. if state["cancelled"]:
  152. # cancel() may have run mid-upload, before ``staged`` was set.
  153. scrub_staged()
  154. return ("", 130)
  155. state["dispatched"] = True
  156. response = sandbox.process.exec(shell_cmd, timeout=timeout)
  157. return (response.result or "", response.exit_code)
  158. return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
  159. def cleanup(self):
  160. with self._lock:
  161. if self._sandbox is None:
  162. return
  163. # sync_back runs inside the lock and after the None guard so an
  164. # already-cleaned-up env can't trigger a 3-attempt retry storm on a nil sandbox.
  165. if self._sync_manager:
  166. logger.info("Daytona: syncing files from sandbox...")
  167. try:
  168. self._sync_manager.sync_back()
  169. except Exception as e:
  170. logger.warning("Daytona: sync_back failed: %s", e)
  171. try:
  172. if self._persistent:
  173. self._sandbox.stop()
  174. logger.info("Daytona: stopped sandbox %s (filesystem preserved)", self._sandbox.id)
  175. else:
  176. self._daytona.delete(self._sandbox)
  177. logger.info("Daytona: deleted sandbox %s", self._sandbox.id)
  178. except Exception as e:
  179. logger.warning("Daytona: cleanup failed: %s", e)
  180. self._sandbox = None