vercel_sandbox.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. """Vercel Sandbox execution environment.
  2. Runs commands in Vercel cloud sandboxes through the shared ``BaseEnvironment``
  3. shell contract. With persistence enabled, task-scoped snapshot ids are stored
  4. under ``HERMES_HOME`` and new sandboxes are restored from them on task reuse.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import logging
  9. import math
  10. import os
  11. import shlex
  12. import threading
  13. import time
  14. from datetime import timedelta
  15. from functools import cache
  16. from pathlib import Path
  17. from typing import TYPE_CHECKING, Any
  18. import httpx
  19. from hermes_cli.config_defaults import DEFAULT_VERCEL_IMAGE
  20. from hermes_constants import get_hermes_home
  21. from tools.environments.base import BaseEnvironment, _load_json_store, _update_json_store
  22. from tools.environments.base_output import _ThreadedProcessHandle
  23. from tools.environments.file_sync import FileSyncManager, iter_sync_files, quoted_rm_command
  24. logger = logging.getLogger(__name__)
  25. if TYPE_CHECKING:
  26. from vercel.sandbox import Sandbox, SandboxStatus, WriteFile
  27. DEFAULT_VERCEL_CWD = "/vercel/sandbox"
  28. _DEFAULT_CONTAINER_DISK_MB = 51200
  29. _CREATE_RETRY_ATTEMPTS = 3
  30. _TRANSIENT_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504})
  31. _RUNNING_WAIT_TIMEOUT = timedelta(seconds=30)
  32. def _ensure_vercel_sdk() -> None:
  33. """Lazy-install vercel SDK on demand. Idempotent."""
  34. # The SDK (>=0.7) ships default-on telemetry; Hermes policy is opt-in only, so disable it
  35. # before the SDK is imported. setdefault: an explicit user value is never overridden.
  36. os.environ.setdefault("VERCEL_TELEMETRY_DISABLED", "1")
  37. try:
  38. from pm import ensure_import as _lazy_ensure
  39. except ImportError:
  40. return # pm unavailable — vercel's own import sites decide
  41. try:
  42. _lazy_ensure("vercel")
  43. except Exception as e:
  44. raise ImportError(str(e))
  45. _WRITE_RETRY_ATTEMPTS = 3
  46. _RETRY_BACKOFF_STEP = timedelta(milliseconds=100)
  47. _MIN_SANDBOX_TIMEOUT = timedelta(minutes=5)
  48. _MIN_RUNNING_WAIT = timedelta(seconds=1)
  49. _RUNNING_WAIT_POLL_INTERVAL = timedelta(milliseconds=250)
  50. _STOP_TIMEOUT = timedelta(seconds=15)
  51. _STOP_POLL_INTERVAL = timedelta(milliseconds=500)
  52. _SNAPSHOT_STORE_NAME = "vercel_sandbox_snapshots.json"
  53. def _exception_chain(exc: BaseException) -> list[BaseException]:
  54. chain: list[BaseException] = []
  55. current: BaseException | None = exc
  56. seen: set[int] = set()
  57. while current is not None and id(current) not in seen:
  58. chain.append(current)
  59. seen.add(id(current))
  60. current = current.__cause__ or current.__context__
  61. return chain
  62. def _extract_status_code(exc: BaseException) -> int | None:
  63. response = getattr(exc, "response", None)
  64. for value in (getattr(exc, "status_code", None), getattr(response, "status_code", None)):
  65. if isinstance(value, int):
  66. return value
  67. return None
  68. @cache
  69. def _transport_error_types() -> tuple[type[BaseException], ...]:
  70. """Network exceptions of every HTTP stack the SDK may raise from.
  71. vercel >= 0.12 moved the sandbox client from httpx to httpx2, a separate
  72. distribution whose exceptions do not subclass httpx's.
  73. """
  74. types: list[type[BaseException]] = [httpx.NetworkError, httpx.ProtocolError, httpx.ReadError]
  75. try:
  76. import httpx2
  77. except ImportError:
  78. return tuple(types)
  79. return (*types, httpx2.NetworkError, httpx2.ProtocolError, httpx2.ReadError)
  80. def _is_transient_vercel_error(exc: BaseException) -> bool:
  81. """True when any exception in the cause/context chain looks retryable."""
  82. seen: set[int] = set()
  83. error: BaseException | None = exc
  84. while error is not None and id(error) not in seen:
  85. seen.add(id(error))
  86. codes = (getattr(error, "status_code", None), getattr(getattr(error, "response", None), "status_code", None))
  87. status = next((c for c in codes if isinstance(c, int)), None)
  88. name = type(error).__name__.lower()
  89. if (status in _TRANSIENT_STATUS_CODES
  90. or isinstance(error, _transport_error_types())
  91. or "ratelimit" in name or "servererror" in name):
  92. return True
  93. error = error.__cause__ or error.__context__
  94. return False
  95. def _retry_vercel_call(label: str, callback, *, attempts: int):
  96. for attempt in range(1, attempts + 1):
  97. try:
  98. return callback()
  99. except Exception as exc:
  100. if attempt >= attempts or not _is_transient_vercel_error(exc):
  101. raise
  102. logger.warning("Vercel: %s failed (%s); retrying %d/%d", label, exc, attempt, attempts)
  103. time.sleep(0.1 * attempt)
  104. def _result_parts(result: Any) -> tuple[str, int]:
  105. """(output, returncode) from an SDK command result; tolerates raw strings and older result shapes."""
  106. try:
  107. value = result.output()
  108. except (AttributeError, TypeError):
  109. value = result
  110. if isinstance(value, bytes):
  111. value = value.decode("utf-8", errors="replace")
  112. exit_code = getattr(result, "exit_code" if hasattr(result, "exit_code") else "returncode", None)
  113. return ("" if value is None else str(value)), (exit_code if isinstance(exit_code, int) else 1)
  114. def _snapshot_store() -> Path:
  115. return get_hermes_home() / "vercel_sandbox_snapshots.json"
  116. def _load_snapshots() -> dict:
  117. return _load_json_store(_snapshot_store())
  118. def _store_snapshot(task_id: str, snapshot_id: str) -> None:
  119. if task_id and snapshot_id:
  120. _update_json_store(_snapshot_store(), lambda snapshots: snapshots.update({task_id: snapshot_id}))
  121. def _delete_snapshot(task_id: str, snapshot_id: str) -> None:
  122. """Drop the stored id for ``task_id`` only if it still equals ``snapshot_id``."""
  123. def update(snapshots: dict) -> None:
  124. if task_id and snapshots.get(task_id) == snapshot_id:
  125. snapshots.pop(task_id)
  126. _update_json_store(_snapshot_store(), update)
  127. def _extract_snapshot_id(snapshot: Any) -> str | None:
  128. """Accept SDK objects or raw dicts; attribute lookup first, then dict keys."""
  129. getters = [lambda k: getattr(snapshot, k, None)] + ([snapshot.get] if isinstance(snapshot, dict) else [])
  130. candidates = (get(key) for get in getters for key in ("snapshot_id", "snapshotId", "id"))
  131. return next((v for v in candidates if isinstance(v, str) and v), None)
  132. @cache
  133. def _sandbox_status_type() -> type[SandboxStatus]:
  134. _ensure_vercel_sdk()
  135. from vercel.sandbox import SandboxStatus
  136. return SandboxStatus
  137. def _is_terminal(status: Any) -> bool:
  138. S = _sandbox_status_type()
  139. return status in {S.ABORTED, S.FAILED, S.STOPPED}
  140. class VercelSandboxEnvironment(BaseEnvironment):
  141. """Vercel cloud sandbox backend."""
  142. _stdin_mode = "payload"
  143. def __init__(self, runtime: str | None = None, cwd: str = DEFAULT_VERCEL_CWD, timeout: int = 60,
  144. cpu: float = 1, memory: int = 5120, disk: int = _DEFAULT_CONTAINER_DISK_MB,
  145. persistent_filesystem: bool = True, task_id: str = "default",
  146. image: str | None = None):
  147. super().__init__(cwd=cwd, timeout=timeout)
  148. if disk not in {0, _DEFAULT_CONTAINER_DISK_MB}:
  149. raise ValueError(
  150. "Vercel Sandbox does not support configurable container_disk. "
  151. "Use the default shared setting.")
  152. self._persistent, self._task_id, self._requested_cwd = persistent_filesystem, task_id, cwd
  153. self._lock = threading.Lock()
  154. self._sandbox: Sandbox | None = None
  155. self._workspace_root = self._remote_home = DEFAULT_VERCEL_CWD
  156. self._sync_manager: FileSyncManager | None = None
  157. _ensure_vercel_sdk()
  158. from vercel.sandbox import Resources
  159. vcpus, memory_mb = (math.floor(cpu) if cpu > 0 else None), (memory if memory > 0 else None)
  160. # Vercel rejects runtime+image together; runtimes are deprecated (Aug 2026), so a runtime is
  161. # only sent when the user still pins one, and then the image is dropped.
  162. if runtime:
  163. logger.warning("Vercel: terminal.vercel_runtime is deprecated by Vercel; set terminal.vercel_image "
  164. "(default %s) instead", DEFAULT_VERCEL_IMAGE)
  165. self._create_kwargs = {
  166. "timeout": max(timedelta(seconds=max(self.timeout, 0)), timedelta(minutes=5)),
  167. "runtime": runtime or None,
  168. "image": None if runtime else (image or DEFAULT_VERCEL_IMAGE),
  169. "resources": Resources(vcpus=vcpus, memory=memory_mb) if (vcpus, memory_mb) != (None, None) else None}
  170. self._attach_fresh_sandbox(cwd)
  171. self._sync_manager.sync(force=True)
  172. self.init_session()
  173. def _require_sandbox(self) -> Sandbox:
  174. if self._sandbox is None:
  175. raise RuntimeError("Vercel sandbox is not attached")
  176. return self._sandbox
  177. def _remote_hermes_dir(self) -> str:
  178. return f"{self._remote_home.rstrip('/')}/.hermes"
  179. def _create_sandbox(self) -> Sandbox:
  180. _ensure_vercel_sdk()
  181. from vercel.sandbox import Sandbox
  182. snapshot_id = _load_snapshots().get(self._task_id) if self._persistent and self._task_id else None
  183. if isinstance(snapshot_id, str) and snapshot_id:
  184. try:
  185. source = {"type": "snapshot", "snapshot_id": snapshot_id}
  186. # A snapshot carries its own filesystem: Vercel refuses runtime with a snapshot
  187. # source, and the image would describe a base the snapshot already replaced.
  188. restore_kwargs = {k: v for k, v in self._create_kwargs.items() if k not in ("runtime", "image")}
  189. return _retry_vercel_call(
  190. "sandbox restore", lambda: Sandbox.create(**restore_kwargs, source=source),
  191. attempts=_CREATE_RETRY_ATTEMPTS)
  192. except Exception as exc:
  193. logger.warning("Vercel: failed to restore snapshot %s for task %s; falling back to a fresh sandbox: %s",
  194. snapshot_id, self._task_id, exc)
  195. _delete_snapshot(self._task_id, snapshot_id)
  196. return _retry_vercel_call("sandbox create", lambda: Sandbox.create(**self._create_kwargs),
  197. attempts=_CREATE_RETRY_ATTEMPTS)
  198. def _attach_fresh_sandbox(self, requested_cwd: str) -> None:
  199. """Create a sandbox, wait until it runs, then wire cwd/home and the file sync manager."""
  200. self._sandbox = self._create_sandbox()
  201. self._wait_for_running()
  202. cwd = self._require_sandbox().sandbox.cwd
  203. self._workspace_root = cwd if cwd.startswith("/") else DEFAULT_VERCEL_CWD
  204. self._remote_home = self._detect_remote_home()
  205. container_base = self._remote_hermes_dir()
  206. self._sync_manager = FileSyncManager(
  207. get_files_fn=lambda: iter_sync_files(container_base),
  208. upload_fn=lambda host_path, remote_path: self._vercel_bulk_upload([(host_path, remote_path)]),
  209. delete_fn=self._vercel_delete,
  210. bulk_upload_fn=self._vercel_bulk_upload, bulk_download_fn=self._vercel_bulk_download)
  211. self.cwd = {"~": self._remote_home, "": self._workspace_root,
  212. DEFAULT_VERCEL_CWD: self._workspace_root}.get(requested_cwd, requested_cwd)
  213. def _detect_remote_home(self) -> str:
  214. try:
  215. result = self._require_sandbox().run_command("sh", ["-lc", 'printf %s "$HOME"'], cwd=self._workspace_root)
  216. except Exception as exc:
  217. logger.debug("Vercel: home detection failed for task %s: %s", self._task_id, exc)
  218. return self._workspace_root
  219. home = _result_parts(result)[0].strip()
  220. return home if home.startswith("/") else self._workspace_root
  221. def _wait_for_running(self, timeout: timedelta = _RUNNING_WAIT_TIMEOUT) -> None:
  222. sandbox, running = self._require_sandbox(), _sandbox_status_type().RUNNING
  223. status = sandbox.status
  224. if status is None or status == running:
  225. return
  226. if _is_terminal(status):
  227. raise RuntimeError(f"Sandbox entered terminal state: {status}")
  228. try:
  229. sandbox.wait_for_status(running, timeout=max(timeout, timedelta(seconds=1)),
  230. poll_interval=timedelta(milliseconds=250))
  231. except TimeoutError as exc:
  232. status = sandbox.status
  233. if _is_terminal(status):
  234. raise RuntimeError(f"Sandbox entered terminal state: {status}") from exc
  235. raise RuntimeError(f"Sandbox did not reach running state (last status: {status})") from exc
  236. @staticmethod
  237. def _close_sandbox_client(sandbox: Sandbox | None) -> None:
  238. with contextlib.suppress(Exception): # None sandbox -> AttributeError -> no-op
  239. sandbox.client.close()
  240. @staticmethod
  241. def _stop_sandbox(sandbox: Sandbox | None) -> None:
  242. with contextlib.suppress(Exception): # None sandbox -> AttributeError -> no-op
  243. try:
  244. sandbox.stop(blocking=True, timeout=timedelta(seconds=15), poll_interval=timedelta(milliseconds=500))
  245. except TypeError: # older SDKs: stop() takes no arguments
  246. sandbox.stop()
  247. def _snapshot_sandbox(self, sandbox: Sandbox) -> None:
  248. if not self._persistent or not self._task_id:
  249. return
  250. try:
  251. snapshot_id = _extract_snapshot_id(sandbox.snapshot())
  252. except Exception as exc:
  253. logger.warning("Vercel: filesystem snapshot failed for task %s: %s", self._task_id, exc)
  254. return
  255. if not snapshot_id:
  256. logger.warning("Vercel: filesystem snapshot for task %s did not return a snapshot id", self._task_id)
  257. return
  258. _store_snapshot(self._task_id, snapshot_id)
  259. logger.info("Vercel: saved filesystem snapshot %s for task %s", snapshot_id, self._task_id)
  260. def _ensure_sandbox_ready(self) -> None:
  261. """Reuse a healthy sandbox; recreate when refresh fails or it hit a terminal state."""
  262. sandbox = self._sandbox
  263. requested_cwd = self.cwd or self._requested_cwd or DEFAULT_VERCEL_CWD
  264. if sandbox is not None:
  265. try:
  266. sandbox.refresh()
  267. except Exception as exc:
  268. logger.warning("Vercel: sandbox refresh failed for task %s: %s; recreating", self._task_id, exc)
  269. else:
  270. status = sandbox.status
  271. if not _is_terminal(status):
  272. self._wait_for_running()
  273. return
  274. logger.warning("Vercel: sandbox entered state %s for task %s; recreating", status, self._task_id)
  275. self._close_sandbox_client(sandbox)
  276. self._mark_recreated()
  277. self._attach_fresh_sandbox(requested_cwd)
  278. def _run_checked(self, script: str, label: str) -> None:
  279. output, returncode = _result_parts(
  280. self._require_sandbox().run_command("bash", ["-lc", script], cwd=self._workspace_root))
  281. if returncode != 0:
  282. raise RuntimeError(f"Vercel {label} failed: {output.strip()}")
  283. def _vercel_bulk_upload(self, files: list[tuple[str, str]]) -> None:
  284. if not files:
  285. return
  286. payload: list[WriteFile] = [
  287. {"path": remote_path, "content": Path(host_path).read_bytes()} for host_path, remote_path in files]
  288. sandbox = self._require_sandbox()
  289. _retry_vercel_call("write_files", lambda: sandbox.write_files(payload), attempts=3)
  290. def _vercel_delete(self, remote_paths: list[str]) -> None:
  291. if remote_paths:
  292. self._run_checked(quoted_rm_command(remote_paths), "delete")
  293. def _vercel_bulk_download(self, dest_tar_path: Path) -> None:
  294. archive_member = self._remote_hermes_dir().lstrip("/")
  295. remote_tar = f"/tmp/.hermes_sync.{os.getpid()}.tar" # no-tmp: ok — remote sandbox path
  296. sandbox = self._require_sandbox()
  297. try:
  298. # --exclude: live sockets cannot be archived ("socket ignored") and must not fail the download.
  299. self._run_checked(f"tar cf {shlex.quote(remote_tar)} --exclude='*.sock' -C / {shlex.quote(archive_member)}",
  300. "bulk download")
  301. sandbox.download_file(remote_tar, dest_tar_path)
  302. finally:
  303. with contextlib.suppress(Exception):
  304. sandbox.run_command("bash", ["-lc", f"rm -f {shlex.quote(remote_tar)}"], cwd=self._workspace_root)
  305. def _before_execute(self) -> None:
  306. with self._lock:
  307. self._ensure_sandbox_ready()
  308. if self._sync_manager is not None:
  309. self._sync_manager.sync()
  310. def _run_bash(self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None):
  311. """``timeout`` is enforced by the base ``_wait_for_process`` via ``cancel_fn`` (the SDK has no
  312. per-exec timeout). Payload stdin is staged through the SDK so it never becomes a shell argv."""
  313. del timeout
  314. sandbox, workspace_root, lock = self._require_sandbox(), self._workspace_root, self._lock
  315. # Guarded by ``lock`` so cancel() and dispatch agree on whether the shell
  316. # has taken ownership of (opened + unlinked) the staged stdin file.
  317. state = {"cancelled": False, "staged": None, "dispatched": False}
  318. def scrub_staged() -> None: # caller holds ``lock``
  319. if state["staged"] and not state["dispatched"]:
  320. # Staged but never dispatched: scrub the payload (may hold a
  321. # sudo password). Once dispatched the user shell unlinks it
  322. # itself, so kill() sends no extra command.
  323. with contextlib.suppress(Exception):
  324. sandbox.write_files([{"path": state["staged"], "content": b"", "mode": 0o600}])
  325. state["staged"] = None
  326. def cancel() -> None:
  327. with lock:
  328. state["cancelled"] = True
  329. scrub_staged()
  330. self._stop_sandbox(sandbox)
  331. def exec_fn() -> tuple[str, int]:
  332. command = cmd_string
  333. if stdin_data: # empty stdin == no stdin, as on base (heredoc skipped it)
  334. with lock:
  335. if state["cancelled"]:
  336. return ("", 130)
  337. remote_stdin = self._staged_stdin_path()
  338. _retry_vercel_call(
  339. "stdin upload",
  340. lambda: sandbox.write_files([{
  341. "path": remote_stdin,
  342. "content": stdin_data.encode("utf-8", "surrogateescape"),
  343. "mode": 0o600,
  344. }]),
  345. attempts=_WRITE_RETRY_ATTEMPTS,
  346. )
  347. command = self._redirect_stdin_from_file(cmd_string, remote_stdin)
  348. with lock:
  349. state["staged"] = remote_stdin
  350. with lock:
  351. if state["cancelled"]:
  352. # cancel() may have run mid-upload, before ``staged`` was set.
  353. scrub_staged()
  354. return ("", 130)
  355. state["dispatched"] = True
  356. return _result_parts(sandbox.run_command(
  357. "bash", ["-lc" if login else "-c", command], cwd=workspace_root))
  358. return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
  359. def cleanup(self):
  360. with self._lock:
  361. sandbox, sync_manager = self._sandbox, self._sync_manager
  362. if sandbox is not None and sync_manager is not None:
  363. try:
  364. sync_manager.sync_back()
  365. except Exception as exc:
  366. logger.warning("Vercel: sync_back failed for task %s: %s", self._task_id, exc)
  367. self._sandbox = None
  368. self._sync_manager = None
  369. if sandbox is None:
  370. return
  371. self._snapshot_sandbox(sandbox)
  372. # Always stop the sandbox during cleanup to avoid resource leaks (matches Modal/Daytona).
  373. self._stop_sandbox(sandbox)
  374. self._close_sandbox_client(sandbox)