managed_modal.py 10.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. """Managed Modal environment backed by tool-gateway.
  2. Deliberately overrides :meth:`BaseEnvironment.execute`: the tool-gateway does command
  3. preparation, CWD tracking and env-snapshot management server-side, so the base
  4. ``_wrap_command`` / ``_wait_for_process`` / snapshot machinery does not apply.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import os
  10. import requests
  11. import shlex
  12. import time
  13. import uuid
  14. from typing import Any, Dict, Optional
  15. from tools.environments.base import BaseEnvironment
  16. from tools.interrupt import is_interrupted
  17. from tools.managed_tool_gateway import resolve_managed_tool_gateway
  18. logger = logging.getLogger(__name__)
  19. _TERMINAL_EXEC_STATUSES = frozenset({"completed", "failed", "cancelled", "timeout"})
  20. _CLIENT_TIMEOUT_GRACE_SECONDS = 10.0
  21. def _coerce_number(value: Any, default: float) -> float:
  22. try:
  23. return default if value is None else float(value)
  24. except (TypeError, ValueError):
  25. return default
  26. def _request_timeout_env(name: str, default: float) -> float:
  27. value = _coerce_number(os.getenv(name), default)
  28. return value if value > 0 else default
  29. def _result(output: str, returncode: int = 1) -> dict:
  30. return {"output": output, "returncode": returncode}
  31. class ManagedModalEnvironment(BaseEnvironment):
  32. """Gateway-owned Modal sandbox with Hermes-compatible execute/cleanup."""
  33. _stdin_mode = "payload"
  34. _CONNECT_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CONNECT_TIMEOUT_SECONDS", 1.0)
  35. _POLL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_POLL_READ_TIMEOUT_SECONDS", 5.0)
  36. _CANCEL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CANCEL_READ_TIMEOUT_SECONDS", 5.0)
  37. def __init__(self, image: str, cwd: str = "/root", timeout: int = 60,
  38. modal_sandbox_kwargs: Optional[dict[str, Any]] = None,
  39. persistent_filesystem: bool = True, task_id: str = "default"):
  40. super().__init__(cwd=cwd, timeout=timeout)
  41. # Managed Modal does not sync or mount host credential files.
  42. try:
  43. from tools.credential_files import get_credential_file_mounts
  44. except Exception:
  45. get_credential_file_mounts = None
  46. if get_credential_file_mounts is not None and get_credential_file_mounts():
  47. raise ValueError(
  48. "Managed Modal does not support host credential-file passthrough. "
  49. "Use TERMINAL_MODAL_MODE=direct when skills or config require "
  50. "credential files inside the sandbox."
  51. )
  52. gateway = resolve_managed_tool_gateway("modal")
  53. if gateway is None:
  54. raise ValueError("Managed Modal requires a configured tool gateway and Nous user token")
  55. self._gateway_origin = gateway.gateway_origin.rstrip("/")
  56. self._nous_user_token = gateway.nous_user_token
  57. self._task_id, self._persistent, self._image = task_id, persistent_filesystem, image
  58. self._sandbox_kwargs = dict(modal_sandbox_kwargs or {})
  59. self._create_idempotency_key = str(uuid.uuid4())
  60. self._sandbox_id = self._create_sandbox()
  61. def execute(self, command: str, cwd: str = "", *, timeout: int | None = None, stdin_data: str | None = None,
  62. rewrite_compound_background: bool = True, bounded_capture: bool = False) -> dict:
  63. # Signature parity with BaseEnvironment.execute only: the gateway runs commands
  64. # explicitly (no shell background rewriting) and returns the remote result in one
  65. # payload, so streaming-time bounding does not apply (the terminal tool's final
  66. # truncation still caps it).
  67. del rewrite_compound_background, bounded_capture
  68. exec_command, sudo_stdin = self._prepare_command(command)
  69. if sudo_stdin is not None:
  70. # Feed sudo via a shell pipe: the transport has no direct stdin piping.
  71. exec_command = f"printf '%s\\n' {shlex.quote(sudo_stdin.rstrip())} | {exec_command}"
  72. timeout = timeout or self.timeout
  73. exec_id = str(uuid.uuid4())
  74. payload: dict[str, Any] = {"execId": exec_id, "command": exec_command, "cwd": cwd or self.cwd,
  75. "timeoutMs": int(timeout * 1000)}
  76. if stdin_data is not None:
  77. payload["stdinData"] = stdin_data
  78. try:
  79. response = self._request("POST", f"/v1/sandboxes/{self._sandbox_id}/execs", json=payload, timeout=10)
  80. body = response.json() if response.status_code < 400 else None
  81. except Exception as exc:
  82. return _result(f"Managed Modal exec failed: {exc}")
  83. if body is None:
  84. return _result(self._format_error("Managed Modal exec failed", response))
  85. if (final := self._result_from_body(body)) is not None:
  86. return final
  87. if body.get("execId") != exec_id:
  88. return _result("Managed Modal exec start did not return the expected exec id")
  89. deadline = time.monotonic() + timeout + _CLIENT_TIMEOUT_GRACE_SECONDS
  90. _now = time.monotonic()
  91. _activity_state = {"last_touch": _now, "start": _now}
  92. while True:
  93. if is_interrupted():
  94. self._cancel_exec(exec_id)
  95. return _result("[Command interrupted - Modal sandbox exec cancelled]", 130)
  96. try:
  97. if (result := self._poll_exec(exec_id)) is not None:
  98. return result
  99. except Exception as exc:
  100. return _result(f"Managed Modal exec failed: {exc}")
  101. if time.monotonic() >= deadline:
  102. self._cancel_exec(exec_id)
  103. return _result(f"Managed Modal exec timed out after {timeout}s", 124)
  104. # Periodic activity touch so the gateway knows we're alive (lazy import:
  105. # tests stub tools.environments.base with only BaseEnvironment)
  106. try:
  107. from tools.environments.base import touch_activity_if_due
  108. touch_activity_if_due(_activity_state, "modal command running")
  109. except Exception:
  110. pass
  111. time.sleep(0.25)
  112. @staticmethod
  113. def _result_from_body(body: dict) -> dict | None:
  114. """Final result dict if the exec body reports a terminal status, else ``None``."""
  115. if body.get("status") in _TERMINAL_EXEC_STATUSES:
  116. return _result(body.get("output", ""), body.get("returncode", 1))
  117. def _poll_exec(self, exec_id: str) -> dict | None:
  118. try:
  119. response = self._request("GET", f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}",
  120. timeout=(self._CONNECT_TIMEOUT_SECONDS, self._POLL_READ_TIMEOUT_SECONDS))
  121. except Exception as exc:
  122. return _result(f"Managed Modal exec poll failed: {exc}")
  123. if response.status_code == 404:
  124. return _result("Managed Modal exec not found")
  125. if response.status_code >= 400:
  126. return _result(self._format_error("Managed Modal exec poll failed", response))
  127. return self._result_from_body(response.json())
  128. def _cancel_exec(self, exec_id: str) -> None:
  129. try:
  130. self._request("POST", f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}/cancel",
  131. timeout=(self._CONNECT_TIMEOUT_SECONDS, self._CANCEL_READ_TIMEOUT_SECONDS))
  132. except Exception as exc:
  133. logger.warning("Managed Modal exec cancel failed: %s", exc)
  134. def cleanup(self):
  135. if not getattr(self, "_sandbox_id", None):
  136. return
  137. try:
  138. self._request("POST", f"/v1/sandboxes/{self._sandbox_id}/terminate",
  139. json={"snapshotBeforeTerminate": self._persistent}, timeout=60)
  140. except Exception as exc:
  141. logger.warning("Managed Modal cleanup failed: %s", exc)
  142. finally:
  143. self._sandbox_id = None
  144. def _create_sandbox(self) -> str:
  145. kw = self._sandbox_kwargs
  146. cpu = _coerce_number(kw.get("cpu"), 1)
  147. memory = _coerce_number(kw.get("memoryMiB", kw.get("memory")), 5120)
  148. disk = _coerce_number(kw.get("ephemeral_disk", kw.get("diskMiB")), None)
  149. create_payload = {
  150. "image": self._image, "cwd": self.cwd, "cpu": cpu, "memoryMiB": memory, "timeoutMs": 3_600_000,
  151. "idleTimeoutMs": max(300_000, int(self.timeout * 1000)),
  152. "persistentFilesystem": self._persistent, "logicalKey": self._task_id,
  153. }
  154. if disk is not None:
  155. create_payload["diskMiB"] = disk
  156. response = self._request("POST", "/v1/sandboxes", json=create_payload, timeout=60,
  157. extra_headers={"x-idempotency-key": self._create_idempotency_key})
  158. if response.status_code >= 400:
  159. raise RuntimeError(self._format_error("Managed Modal create failed", response))
  160. sandbox_id = response.json().get("id")
  161. if not isinstance(sandbox_id, str) or not sandbox_id:
  162. raise RuntimeError("Managed Modal create did not return a sandbox id")
  163. return sandbox_id
  164. def _request(self, method: str, path: str, *, json: dict[str, Any] | None = None, timeout: int = 30,
  165. extra_headers: dict[str, str] | None = None) -> requests.Response:
  166. headers = {"Authorization": f"Bearer {self._nous_user_token}", "Content-Type": "application/json",
  167. **(extra_headers or {})}
  168. return requests.request(method, f"{self._gateway_origin}{path}", headers=headers, json=json, timeout=timeout)
  169. @staticmethod
  170. def _format_error(prefix: str, response: requests.Response) -> str:
  171. try:
  172. payload = response.json()
  173. if isinstance(payload, dict):
  174. message = payload.get("error") or payload.get("message") or payload.get("code")
  175. if isinstance(message, str) and message:
  176. return f"{prefix}: {message}"
  177. return f"{prefix}: {json.dumps(payload, ensure_ascii=False)}"
  178. except Exception:
  179. pass
  180. text = response.text.strip()
  181. return f"{prefix}: {text}" if text else f"{prefix}: HTTP {response.status_code}"