terminal_tool_backends.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403
  1. """Execution-environment backends for the terminal tool: per-backend builders, config-to-kwargs
  2. shapers, and requirement checkers, routed by dispatch/spec tables. Split out of ``tools/terminal_tool.py``;
  3. the terminal tool imports the builders/checkers it uses from here."""
  4. import functools
  5. import importlib.util
  6. import inspect
  7. import logging
  8. import subprocess
  9. from typing import Any, Dict, Optional
  10. from tools.environments.docker import DockerEnvironment as _DockerEnvironment
  11. from tools.environments.local import LocalEnvironment as _LocalEnvironment
  12. from tools.environments.managed_modal import ManagedModalEnvironment as _ManagedModalEnvironment
  13. from tools.environments.modal import ModalEnvironment as _ModalEnvironment
  14. from tools.environments.ssh import SSHEnvironment as _SSHEnvironment
  15. from tools.managed_tool_gateway import is_managed_tool_gateway_ready
  16. from tools.terminal_tool_config import _get_plugin_env_provider
  17. from tools.tool_backend_helpers import (has_direct_modal_credentials, managed_nous_tools_enabled,
  18. nous_tool_gateway_unavailable_message, resolve_modal_backend_state)
  19. # Log-record parity with the origin module.
  20. logger = logging.getLogger("tools.terminal_tool")
  21. # Human reason for the most recent failed requirements check (None after a passing one). The CLI
  22. # startup notice and `hermes doctor` read it through terminal_backend_unavailable_reason() so the user
  23. # hears WHY the terminal tool is missing instead of discovering it on first use.
  24. _last_unavailable_reason: Optional[str] = None
  25. def _reject(reason: str) -> bool:
  26. """Log *reason*, remember it for the user-facing notice, and return False for the checker."""
  27. global _last_unavailable_reason
  28. _last_unavailable_reason = reason
  29. logger.error(reason)
  30. return False
  31. def _record_unavailable_reason(reason: Optional[str]) -> None:
  32. """Store (or clear) the last failure reason without logging; used by check_terminal_requirements."""
  33. global _last_unavailable_reason
  34. _last_unavailable_reason = reason
  35. def terminal_backend_unavailable_reason() -> Optional[str]:
  36. """Why the last terminal requirements check failed, in plain words; None when it passed."""
  37. return _last_unavailable_reason
  38. _VERCEL_SANDBOX_DEFAULT_CWD = "/vercel/sandbox"
  39. _SUPPORTED_VERCEL_RUNTIMES = ("node24", "node22", "python3.13")
  40. _BUILTIN_BACKENDS = "local, docker, modal, daytona, vercel_sandbox, ssh"
  41. # Config -> kwargs shapers, driven by (out_key, config_key, default) tables. The container table's
  42. # (key, default) literal is intentionally greppable; tools/terminal_tool.py keeps its own for the AST test.
  43. _SSH_KEYS = (("host", "ssh_host", ""), ("user", "ssh_user", ""), ("port", "ssh_port", 22),
  44. ("key", "ssh_key", ""), ("persistent", "ssh_persistent", True))
  45. _RESOURCE_KEYS = (("cpu", "container_cpu", 1), ("memory", "container_memory", 5120),
  46. ("disk", "container_disk", 51200), ("persistent_filesystem", "container_persistent", True))
  47. _CONTAINER_KEYS = (
  48. ("container_cpu", 1), ("container_memory", 5120), ("container_disk", 51200),
  49. ("container_persistent", True), ("modal_mode", "auto"), ("vercel_runtime", ""), ("vercel_image", ""),
  50. ("docker_volumes", []), ("docker_mount_cwd_to_workspace", False), ("docker_forward_env", []),
  51. ("docker_env", {}), ("docker_run_as_host_user", False), ("docker_extra_args", []),
  52. ("docker_shm_size", "1g"), ("docker_network", True), ("docker_persist_across_processes", True),
  53. ("docker_shared_container_key", ""), ("docker_orphan_reaper", True), ("docker_snap_compat", False),
  54. ("docker_image_pinned", False),
  55. )
  56. _DOCKER_KWARGS = (
  57. ("volumes", "docker_volumes", []), ("auto_mount_cwd", "docker_mount_cwd_to_workspace", False),
  58. ("forward_env", "docker_forward_env", []), ("env", "docker_env", {}),
  59. ("run_as_host_user", "docker_run_as_host_user", False), ("network", "docker_network", True),
  60. ("extra_args", "docker_extra_args", []), ("persist_across_processes", "docker_persist_across_processes", True),
  61. ("shared_container_key", "docker_shared_container_key", ""), ("shm_size", "docker_shm_size", "1g"),
  62. ("snap_compat", "docker_snap_compat", False), ("image_pinned", "docker_image_pinned", False),
  63. )
  64. def _ssh_config_from_config(config: dict[str, Any]) -> dict:
  65. """``ssh_config`` for :func:`_create_environment` (shared with the lazy ``ensure_task_env``)."""
  66. return {out: config.get(key, default) for out, key, default in _SSH_KEYS}
  67. def _local_config_from_config(config: dict[str, Any]) -> dict:
  68. """``local_config`` for :func:`_create_environment` (TERMINAL_LOCAL_PERSISTENT, default on)."""
  69. return {"persistent": config.get("local_persistent", True)}
  70. def _container_config_from_config(config: dict[str, Any]) -> dict:
  71. """``container_config`` for :func:`_create_environment` (shared with the lazy ``ensure_task_env``)."""
  72. return {k: config.get(k, d) for k, d in _CONTAINER_KEYS}
  73. def _resources(cc: dict[str, Any]) -> dict:
  74. """Common sandbox resource kwargs (cpu/memory in MB/disk in MB/persistence)."""
  75. return {out: cc.get(key, default) for out, key, default in _RESOURCE_KEYS}
  76. def _is_supported_vercel_runtime(runtime: str) -> bool:
  77. return not runtime or runtime in _SUPPORTED_VERCEL_RUNTIMES
  78. def _get_modal_backend_state(modal_mode: object | None) -> dict[str, Any]:
  79. """Resolve direct vs managed Modal backend selection."""
  80. return resolve_modal_backend_state(modal_mode, has_direct=has_direct_modal_credentials(),
  81. managed_ready=is_managed_tool_gateway_ready("modal"))
  82. def _modal_unavailable_reason(modal_state: dict[str, Any]) -> tuple[str, str]:
  83. """(log message, ValueError message) for a modal_state with no selected backend.
  84. Single decision shared by the requirements checker and the env builder."""
  85. gateway = nous_tool_gateway_unavailable_message("managed Modal execution")
  86. if modal_state["managed_mode_blocked"] or modal_state["mode"] == "managed":
  87. tail = (("Nous Tool Gateway access is not currently available and no direct Modal credentials/config "
  88. f"were found. {gateway} Choose TERMINAL_MODAL_MODE=direct/auto to use direct Modal credentials.")
  89. if modal_state["managed_mode_blocked"] else f"the managed tool gateway is unavailable. {gateway}")
  90. return (f"Modal backend selected with TERMINAL_MODAL_MODE=managed, but {tail}",
  91. f"Modal backend is configured for managed mode, but {tail}")
  92. managed = managed_nous_tools_enabled()
  93. if modal_state["mode"] == "direct":
  94. return ("Modal backend selected with TERMINAL_MODAL_MODE=direct, but no direct Modal credentials/config "
  95. f"were found. Configure Modal or choose TERMINAL_MODAL_MODE={'managed/auto' if managed else 'auto'}.",
  96. "Modal backend is configured for direct mode, but no direct Modal credentials/config were found.")
  97. found = "or managed tool gateway was found" if managed else "was found"
  98. fix = ", set up the managed gateway, or" if managed else " or"
  99. return (f"Modal backend selected but no direct Modal credentials/config {found}. "
  100. f"Configure Modal{fix} choose a different TERMINAL_ENV.",
  101. f"Modal backend selected but no direct Modal credentials/config {found}.")
  102. # --- Environment builders. Signature: (*, image, cwd, timeout, cc, task_id, ssh_config, local_config, host_cwd)
  103. # (env_type is only forwarded to the plugin-registry fallback, not to the built-in builders.)
  104. def _build_local_env(*, cwd, timeout, local_config=None, **_):
  105. return _LocalEnvironment(cwd=cwd, timeout=timeout, persistent=(local_config or {}).get("persistent", True))
  106. def _build_docker_env(*, image, cwd, timeout, cc, task_id, host_cwd, **_):
  107. from tools.terminal_tool import (_docker_session_isolation_enabled, _has_isolation_overrides,
  108. _maybe_reap_docker_orphans)
  109. # One-shot reaper for labeled containers orphaned by prior Hermes processes that died before
  110. # atexit (SIGKILL / OOM / closed terminal); ``terminal.docker_orphan_reaper: false`` disables it.
  111. _maybe_reap_docker_orphans(cc)
  112. # A session-keyed container must not outlive its session, so cross-process reuse/persist is
  113. # disabled for it (cleanup_vm()/idle reaper stop+rm it). The shared "default" container and
  114. # RL/benchmark override sandboxes keep their existing lifecycle.
  115. session_scoped = (_docker_session_isolation_enabled() and task_id != "default"
  116. and not _has_isolation_overrides(task_id))
  117. kwargs = {out: cc.get(key, default) for out, key, default in _DOCKER_KWARGS}
  118. if session_scoped:
  119. kwargs["persist_across_processes"] = False
  120. docker_env_obj = _DockerEnvironment(image=image, cwd=cwd, timeout=timeout, task_id=task_id, host_cwd=host_cwd,
  121. **_resources(cc), **kwargs)
  122. # Marker read by is_persistent_env(): a session-scoped container survives BETWEEN turns (skip
  123. # per-turn teardown) but is removed at session close / idle timeout. Test doubles may reject attrs.
  124. if session_scoped:
  125. try:
  126. docker_env_obj._session_scoped = True
  127. except AttributeError:
  128. pass
  129. return docker_env_obj
  130. def _build_modal_env(*, image, cwd, timeout, cc, task_id, **_):
  131. res = _resources(cc)
  132. sandbox_kwargs = {k: res[k] for k in ("cpu", "memory") if res[k] > 0}
  133. if res["disk"] > 0:
  134. try:
  135. import modal
  136. if "ephemeral_disk" in inspect.signature(modal.Sandbox.create).parameters:
  137. sandbox_kwargs["ephemeral_disk"] = res["disk"]
  138. except Exception:
  139. pass
  140. modal_state = _get_modal_backend_state(cc.get("modal_mode"))
  141. selected = modal_state["selected_backend"]
  142. if selected not in ("managed", "direct"):
  143. raise ValueError(_modal_unavailable_reason(modal_state)[1])
  144. cls = _ManagedModalEnvironment if selected == "managed" else _ModalEnvironment
  145. return cls(image=image, cwd=cwd, timeout=timeout, modal_sandbox_kwargs=sandbox_kwargs,
  146. persistent_filesystem=res["persistent_filesystem"], task_id=task_id)
  147. # env_type -> (class getter, takes image, extra kwargs from (cc, resource kwargs)). SDK-backed modules
  148. # (daytona/vercel) are imported lazily so they are only required when that backend is selected.
  149. _SANDBOX_ROWS = {
  150. "daytona": (lambda: importlib.import_module("tools.environments.daytona").DaytonaEnvironment, True,
  151. lambda cc, kw: {"cpu": int(kw["cpu"])}),
  152. "vercel_sandbox": (lambda: importlib.import_module("tools.environments.vercel_sandbox").VercelSandboxEnvironment,
  153. False, lambda cc, kw: {"runtime": cc.get("vercel_runtime") or None,
  154. "image": cc.get("vercel_image") or None}),
  155. }
  156. def _build_sandbox_env(env_type, *, image, cwd, timeout, cc, task_id, **_):
  157. cls, with_image, extra = _SANDBOX_ROWS[env_type]
  158. kwargs = dict(cwd=cwd, timeout=timeout, task_id=task_id, **_resources(cc),
  159. **({"image": image} if with_image else {}))
  160. kwargs.update(extra(cc, kwargs))
  161. return cls()(**kwargs)
  162. _build_daytona_env = functools.partial(_build_sandbox_env, "daytona")
  163. _build_vercel_env = functools.partial(_build_sandbox_env, "vercel_sandbox")
  164. def _build_ssh_env(*, cwd, timeout, ssh_config, probe_only=False, **_):
  165. if not ssh_config or not ssh_config.get("host") or not ssh_config.get("user"):
  166. raise ValueError("SSH environment requires ssh_host and ssh_user to be configured")
  167. return _SSHEnvironment(host=ssh_config["host"], user=ssh_config["user"], port=ssh_config.get("port", 22),
  168. key_path=ssh_config.get("key", ""), cwd=cwd, timeout=timeout, probe_only=probe_only,
  169. persistent=ssh_config.get("persistent", True))
  170. def _stamp_sandbox_home(env_obj, provider, cwd) -> None:
  171. """Record the provider's declared home on this environment: a declared absolute path whatever ``cwd`` the
  172. environment started in (``terminal.cwd`` may pick another directory), or, for ``~``, the path it resolved
  173. to inside THIS sandbox when it started there. Per environment, never per backend, so a second sandbox
  174. with a different home gets its own."""
  175. try:
  176. declared = provider.default_cwd
  177. except Exception:
  178. # A plugin's property must not take environment creation down; it just gets no home.
  179. logger.warning("%s.default_cwd raised; not recording a sandbox home", type(provider).__name__,
  180. exc_info=True)
  181. declared = None
  182. if declared == "~":
  183. home = getattr(env_obj, "cwd", None) if cwd == "~" else None
  184. else:
  185. home = declared
  186. if isinstance(home, str) and home.startswith("/") and home != "/":
  187. try:
  188. env_obj._hermes_sandbox_home = home
  189. except AttributeError:
  190. pass
  191. def _build_plugin_env(*, env_type, image, cwd, timeout, cc, task_id, **_):
  192. provider = _get_plugin_env_provider(env_type)
  193. if provider is not None:
  194. from hermes_cli.plugin_host import is_hosted_object
  195. if is_hosted_object(provider):
  196. # The environment would come back as an opaque placeholder, and its execute() would run outside
  197. # this process's interrupt, activity and terminal-scope state: refuse before the host starts a
  198. # sandbox nothing here could use or clean up.
  199. raise ValueError(
  200. f"terminal.backend is {env_type}, a plugin that runs in the plugin host (plugins.isolation: host), "
  201. f"and the host cannot hand its environment to Hermes, so commands cannot run in it. Set "
  202. f"plugins.isolation: in_process to use this backend.")
  203. env_obj = provider.create_environment(cwd=cwd, timeout=timeout, task_id=task_id, image=image,
  204. container_config=cc)
  205. # Stamp the backend name so path-resolution and progress surfaces can identify plugin
  206. # backends without class-name sniffing. Test doubles may reject attributes.
  207. try:
  208. env_obj._hermes_backend_name = provider.name.strip().lower()
  209. except AttributeError:
  210. pass
  211. _stamp_sandbox_home(env_obj, provider, cwd)
  212. return env_obj
  213. try:
  214. from agent.terminal_env_registry import plugin_backend_names
  215. plugin_names = plugin_backend_names()
  216. except Exception:
  217. plugin_names = []
  218. known = ", ".join(f"'{n}'" for n in _BUILTIN_BACKENDS.split(", ") + list(plugin_names))
  219. from hermes_cli.left_core_migration import terminal_backend_install_hint
  220. raise ValueError(f"Unknown environment type: {env_type}. Use {known}{terminal_backend_install_hint(env_type)}")
  221. # Built-in backend -> builder. Anything else is looked up in the plugin registry.
  222. _ENV_BUILDERS = {"local": _build_local_env, "docker": _build_docker_env, "modal": _build_modal_env,
  223. "daytona": _build_daytona_env, "vercel_sandbox": _build_vercel_env,
  224. "ssh": _build_ssh_env}
  225. def _create_environment(env_type: str, image: str, cwd: str, timeout: int,
  226. ssh_config: dict | None = None, container_config: dict | None = None,
  227. local_config: dict | None = None, task_id: str = "default",
  228. host_cwd: Optional[str] = None, probe_only: bool = False):
  229. """Create an execution environment (instance with ``execute()``) for *env_type*. ``image`` is ignored
  230. for local/ssh/vercel; ``container_config`` carries the container_*/docker_* resource keys; ``host_cwd`` is
  231. the host dir bound into Docker when cwd mounting is enabled. ``probe_only`` asks ssh for a throwaway
  232. connection with no remote setup/sync (the prompt-time probe). Unknown types fall through to plugin backends."""
  233. builder = _ENV_BUILDERS.get(env_type)
  234. kwargs = dict(image=image, cwd=cwd, timeout=timeout, cc=container_config or {}, task_id=task_id,
  235. ssh_config=ssh_config, local_config=local_config, host_cwd=host_cwd, probe_only=probe_only)
  236. if builder is not None:
  237. env = builder(**kwargs)
  238. else:
  239. env = _build_plugin_env(env_type=env_type, **kwargs)
  240. # Backend tag for consumers that only hold the instance (cwd sanitizers on
  241. # live cached envs); __slots__ plugin providers simply keep going untagged.
  242. try:
  243. env.env_type = env_type
  244. except Exception:
  245. pass
  246. return env
  247. # --- Requirement checkers: one generic path driven by _BACKEND_SPECS; optional fields, checked in order:
  248. # pre(config) -> True (satisfied) / False (rejected, already logged) / None (continue);
  249. # binary=(finder, version_arg, missing_log_or_None) runs ``<binary> <arg>``, ok iff rc == 0;
  250. # module=(find_spec name, log message when absent); post(config) -> bool.
  251. def _check_vercel(config: dict[str, Any]) -> bool:
  252. """Runtime -> disk -> SDK -> auth (OIDC token, else the full TOKEN/PROJECT_ID/TEAM_ID tuple)."""
  253. runtime = (config.get("vercel_runtime") or "").strip()
  254. disk = config.get("container_disk", 51200)
  255. if not _is_supported_vercel_runtime(runtime):
  256. return _reject(f"Vercel Sandbox runtime {runtime!r} is not supported. Set TERMINAL_VERCEL_RUNTIME to one of: "
  257. f"{', '.join(_SUPPORTED_VERCEL_RUNTIMES)}.")
  258. if disk not in {0, 51200}:
  259. return _reject(f"Vercel Sandbox does not support custom TERMINAL_CONTAINER_DISK={disk}. "
  260. "Use the default shared setting (51200 MB).")
  261. if importlib.util.find_spec("vercel") is None:
  262. return _reject("vercel is required for the Vercel Sandbox terminal backend. Run hermes setup terminal and select Vercel Sandbox.")
  263. from agent.secret_scope import get_secret
  264. if get_secret("VERCEL_OIDC_TOKEN"):
  265. return True
  266. present = [bool(get_secret(k)) for k in ("VERCEL_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID")]
  267. if all(present):
  268. return True
  269. head = ("selected with token auth, but VERCEL_TOKEN, VERCEL_PROJECT_ID, and VERCEL_TEAM_ID must all be "
  270. "set together." if any(present) else
  271. "selected but no supported auth configuration was found. Set VERCEL_TOKEN, VERCEL_PROJECT_ID, "
  272. "and VERCEL_TEAM_ID for normal use.")
  273. return _reject(f"Vercel Sandbox backend {head} VERCEL_OIDC_TOKEN is supported for one-off local development only.")
  274. def _modal_pre(config: dict[str, Any]) -> Optional[bool]:
  275. modal_state = _get_modal_backend_state(config.get("modal_mode"))
  276. if modal_state["selected_backend"] == "managed":
  277. return True
  278. if modal_state["selected_backend"] != "direct":
  279. return _reject(_modal_unavailable_reason(modal_state)[0])
  280. return None
  281. def _ssh_pre(config: dict[str, Any]) -> bool:
  282. if config.get("ssh_host") and config.get("ssh_user"):
  283. return True
  284. return _reject("the SSH host and user are not configured (TERMINAL_SSH_HOST / TERMINAL_SSH_USER); "
  285. "run `hermes setup terminal` to enter them or pick the 'local' backend")
  286. def _daytona_post(config: dict[str, Any]) -> bool:
  287. from daytona import Daytona
  288. from agent.secret_scope import get_secret
  289. return get_secret("DAYTONA_API_KEY") is not None
  290. _BACKEND_SPECS: dict[str, dict[str, Any]] = {
  291. "local": {},
  292. "docker": {"binary": (lambda: importlib.import_module("tools.environments.docker").find_docker(), "version",
  293. "Docker is not installed — no docker executable in PATH or the usual install locations")},
  294. "ssh": {"pre": _ssh_pre},
  295. "modal": {"pre": _modal_pre,
  296. "module": ("modal", "modal is required for direct modal terminal backend. Run hermes setup terminal and select Modal.")},
  297. "vercel_sandbox": {"pre": _check_vercel},
  298. "daytona": {"post": _daytona_post},
  299. }
  300. _PROBE_FAILED_REASONS = {
  301. "docker": "Docker is installed but the Docker daemon is not running",
  302. }
  303. def _check_requirements(env_type: str, config: dict[str, Any]) -> bool:
  304. _record_unavailable_reason(None)
  305. spec = _BACKEND_SPECS[env_type]
  306. verdict = spec["pre"](config) if "pre" in spec else None
  307. if verdict is not None:
  308. return verdict
  309. if "binary" in spec:
  310. finder, arg, missing_msg = spec["binary"]
  311. executable = finder()
  312. if not executable:
  313. return _reject(missing_msg or f"the {env_type!r} backend's executable was not found")
  314. probe = subprocess.run([executable, arg], capture_output=True, timeout=5, stdin=subprocess.DEVNULL, check=False)
  315. if probe.returncode != 0:
  316. return _reject(f"{_PROBE_FAILED_REASONS[env_type]} (`{executable} {arg}` exited with code {probe.returncode})")
  317. return True
  318. if "module" in spec and importlib.util.find_spec(spec["module"][0]) is None:
  319. return _reject(spec["module"][1])
  320. if "post" in spec and not spec["post"](config):
  321. return _reject(f"the {env_type!r} backend is missing its API key or SDK")
  322. return True
  323. def _check_plugin_requirements(config: dict[str, Any]) -> bool:
  324. _record_unavailable_reason(None)
  325. env_type = config["env_type"]
  326. provider = _get_plugin_env_provider(env_type)
  327. if provider is not None:
  328. ok = bool(provider.check_requirements(config))
  329. return ok or _reject(f"the {env_type!r} plugin backend reported its requirements are not met")
  330. from hermes_cli.left_core_migration import terminal_backend_install_hint
  331. return _reject(f"Unknown TERMINAL_ENV '{env_type}'. Use one of: {_BUILTIN_BACKENDS}, or a plugin-registered "
  332. f"backend{terminal_backend_install_hint(env_type)}")
  333. # Built-in backend -> requirements checker; unknown backends go to the plugin registry.
  334. _REQUIREMENT_CHECKERS = {name: functools.partial(_check_requirements, name) for name in _BACKEND_SPECS}
  335. _check_vercel_sandbox_requirements = _REQUIREMENT_CHECKERS["vercel_sandbox"] # used by code_execution_tool