terminal_tool_lifecycle.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333
  1. """Sandbox lifecycle for the terminal tool: idle reaping, teardown, manual/atexit
  2. cleanup, and the lazy ensure_task_env bring-up. The env cache dicts and locks
  3. stay in tools.terminal_tool (tests patch them there) and are read through it
  4. at call time.
  5. Split out of ``tools/terminal_tool.py``; every public/patched name is re-imported there,
  6. so ``tools.terminal_tool.<name>`` keeps resolving (and monkeypatching) as before.
  7. """
  8. import logging
  9. import inspect
  10. import threading
  11. import time
  12. from contextlib import contextmanager
  13. from typing import Any, Dict, Optional
  14. from tools.terminal_tool_backends import (
  15. _container_config_from_config,
  16. _local_config_from_config,
  17. _ssh_config_from_config,
  18. )
  19. from tools.terminal_tool_config import _quiet
  20. # Log-record parity with the origin module.
  21. logger = logging.getLogger("tools.terminal_tool")
  22. def _create_configured_env(
  23. config: dict[str, Any], env_type: str, *, image: str, cwd: str, timeout: int,
  24. task_id: str, host_cwd: Optional[str],
  25. ):
  26. """``_create_environment`` with the ssh/local/container kwargs shaped from *config*
  27. (shared by the terminal tool, the file tools and the lazy :func:`ensure_task_env` bring-up)."""
  28. from tools.terminal_tool_backends import _create_environment
  29. from tools.terminal_tool_config import _is_container_backend
  30. return _create_environment(
  31. env_type=env_type, image=image, cwd=cwd, timeout=timeout,
  32. ssh_config=_ssh_config_from_config(config) if env_type == "ssh" else None,
  33. container_config=(
  34. _container_config_from_config(config) if _is_container_backend(env_type) else None
  35. ),
  36. local_config=_local_config_from_config(config) if env_type == "local" else None,
  37. task_id=task_id, host_cwd=host_cwd,
  38. )
  39. def _cleanup_env(env: Any, *, force_remove: Optional[bool] = None) -> None:
  40. """Tear down one environment via cleanup()/stop()/terminate(), whichever it has.
  41. ``force_remove`` is forwarded to ``cleanup()`` only when given and the backend's
  42. signature accepts it (``DockerEnvironment``, issue #20561; other backends don't).
  43. Shared by ``cleanup_vm``, the idle reaper and the prompt-time backend probe so
  44. the signature check lives in one place.
  45. """
  46. if hasattr(env, 'cleanup'):
  47. if force_remove is not None and "force_remove" in inspect.signature(env.cleanup).parameters:
  48. env.cleanup(force_remove=force_remove)
  49. else:
  50. env.cleanup()
  51. elif hasattr(env, 'stop'):
  52. env.stop()
  53. elif hasattr(env, 'terminate'):
  54. env.terminate()
  55. def _teardown_env(env: Any, task_id: str, *, force_remove: Optional[bool] = None, done_msg: str = "Cleaned up inactive environment for task: %s", stop_idle: bool = False) -> None:
  56. """``_cleanup_env`` (or, with *stop_idle*, the backend's ``stop_idle``) plus outcome logging.
  57. A 404/"not found" error means the sandbox is already gone — logged at info."""
  58. try:
  59. if stop_idle:
  60. env.stop_idle()
  61. else:
  62. _cleanup_env(env, force_remove=force_remove)
  63. logger.info(done_msg, task_id)
  64. except Exception as e:
  65. error_str = str(e)
  66. if "404" in error_str or "not found" in error_str.lower():
  67. logger.info("Environment for task %s already cleaned up", task_id)
  68. else:
  69. logger.warning("Error cleaning up environment for task %s: %s", task_id, e)
  70. def _clear_file_ops_cache(task_id: str) -> None:
  71. """Invalidate the file_ops cache entry so ShellFileOperations can't reference a dead sandbox."""
  72. try:
  73. from tools.file_tools import clear_file_ops_cache
  74. clear_file_ops_cache(task_id)
  75. except ImportError:
  76. pass
  77. def _unregister_env(task_id: str):
  78. """Pop *task_id* from the env cache, activity map and creation locks; return
  79. the env (or None). Callers run the (slow) teardown OUTSIDE the lock —
  80. Modal/Docker teardown can block 10-15s and would stall every concurrent
  81. terminal/file tool call."""
  82. from tools.terminal_tool import (
  83. _active_environments, _creation_locks, _creation_locks_lock, _env_lock,
  84. _last_activity,
  85. )
  86. with _env_lock:
  87. env = _active_environments.pop(task_id, None)
  88. _last_activity.pop(task_id, None)
  89. with _creation_locks_lock:
  90. _creation_locks.pop(task_id, None)
  91. return env
  92. @contextmanager
  93. def _call_in_flight(*keys: Optional[str]):
  94. """Hold the env cached under *keys* against the idle reaper for the duration of a call.
  95. Only background processes on workspace keys refresh ``_last_activity`` mid-run, so a
  96. foreground command longer than ``lifetime_seconds`` had its sandbox reaped out from under
  97. it. Refcounted (parallel calls share a key); exit restamps, so the idle countdown starts at
  98. call end.
  99. """
  100. from tools.terminal_tool import _calls_in_flight, _env_lock, _last_activity
  101. held = [k for k in dict.fromkeys(keys) if k]
  102. with _env_lock:
  103. for k in held:
  104. _calls_in_flight[k] = _calls_in_flight.get(k, 0) + 1
  105. try:
  106. yield
  107. finally:
  108. with _env_lock:
  109. now = time.time()
  110. for k in held:
  111. left = _calls_in_flight.pop(k) - 1
  112. if left:
  113. _calls_in_flight[k] = left
  114. if k in _last_activity:
  115. _last_activity[k] = now
  116. def _cleanup_inactive_envs(lifetime_seconds: int = 300):
  117. """Clean up environments that have been inactive for longer than lifetime_seconds."""
  118. from tools.terminal_tool import (
  119. _active_environments, _calls_in_flight, _creation_locks, _creation_locks_lock, _env_lock,
  120. _last_activity,
  121. )
  122. current_time = time.time()
  123. # Sandboxes with active background processes stay alive (refresh activity).
  124. try:
  125. from tools.process_registry import process_registry
  126. for task_id in list(_last_activity.keys()):
  127. # A workspace container is keyed by its digest, not the spawning task id: also keep it
  128. # while any process it hosts is still running (stopping it would kill that process).
  129. if process_registry.has_active_processes(task_id) or (
  130. _is_workspace_key(task_id)
  131. and process_registry._any_running(lambda s, key=task_id: s.task_id == key)):
  132. _last_activity[task_id] = current_time
  133. except ImportError:
  134. pass
  135. # Phase 1: unregister stale entries atomically under the lock; phase 2:
  136. # stop them outside it (see _unregister_env for why).
  137. with _env_lock:
  138. # A key with a foreground call in flight is busy, however old its stamp.
  139. stale = [t for t, last in list(_last_activity.items())
  140. if current_time - last > lifetime_seconds and t not in _calls_in_flight]
  141. envs_to_stop = [(t, _active_environments.pop(t, None)) for t in stale]
  142. for t in stale:
  143. _last_activity.pop(t, None)
  144. with _creation_locks_lock:
  145. for t in stale:
  146. _creation_locks.pop(t, None)
  147. for task_id, env in envs_to_stop:
  148. if env is not None:
  149. _clear_file_ops_cache(task_id)
  150. if _is_workspace_key(task_id) and callable(getattr(env, "stop_idle", None)):
  151. # Persistent per-workspace container: stop it (state kept, restarted on next use)
  152. # instead of leaving it running forever like the one profile container.
  153. _teardown_env(env, task_id, stop_idle=True)
  154. else:
  155. _teardown_env(env, task_id)
  156. def _is_workspace_key(key: str) -> bool:
  157. """A persistent Docker key bound to one session workspace (``<base>:ws:<digest>``)."""
  158. return ":ws:" in str(key or "")
  159. def get_active_env(task_id: str):
  160. """Return the active BaseEnvironment for *task_id*, or None."""
  161. from tools.terminal_tool import _active_environments, _env_lock, _resolve_container_task_id
  162. lookup = _resolve_container_task_id(task_id)
  163. with _env_lock:
  164. return _active_environments.get(lookup) or _active_environments.get(task_id)
  165. def ensure_task_env(task_id: Optional[str] = None):
  166. """Lazily create and cache the sandbox env for *task_id* if none is active.
  167. Lets non-terminal callers (``tools.image_source`` reading container-only
  168. paths) bring the sandbox up on demand with the same machinery as the
  169. terminal tool. No-op on local. Returns the env, or ``None`` when local or
  170. when creation fails (best-effort; the caller's fail-closed path stays intact).
  171. :func:`terminal_tool` creates the environment on the first terminal command, but nothing else did — so
  172. under a non-local backend (ssh, docker, …) a session whose first action is ``vision_analyze`` on a
  173. container-only path hit "no active sandbox session" because the SSH/Docker handshake never ran (issue
  174. #62825). vision reads such paths inside the sandbox (see ``tools.image_source``), so it calls this to
  175. bring the env up on demand, reusing the same creation machinery as the terminal tool.
  176. """
  177. from tools.terminal_tool import (
  178. _active_environments, _creation_locks, _creation_locks_lock, _env_lock,
  179. _get_env_config, _last_activity, _resolve_container_task_id,
  180. _resolve_task_host_cwd, _select_image, _start_cleanup_thread, resolve_task_overrides,
  181. )
  182. config = _get_env_config()
  183. env_type = config["env_type"]
  184. if env_type == "local":
  185. return None
  186. effective_task_id = _resolve_container_task_id(task_id)
  187. existing = get_active_env(effective_task_id)
  188. if existing is not None:
  189. with _env_lock:
  190. _last_activity[effective_task_id] = time.time()
  191. return existing
  192. image = _select_image(env_type, resolve_task_overrides(task_id), config)
  193. _start_cleanup_thread()
  194. with _creation_locks_lock:
  195. task_lock = _creation_locks.setdefault(effective_task_id, threading.Lock())
  196. with task_lock:
  197. existing = get_active_env(effective_task_id)
  198. if existing is not None:
  199. return existing
  200. try:
  201. new_env = _create_configured_env(
  202. config, env_type, image=image, cwd=config["cwd"],
  203. timeout=config["timeout"], task_id=effective_task_id,
  204. host_cwd=_resolve_task_host_cwd(config, task_id),
  205. )
  206. except Exception as exc:
  207. logger.warning(
  208. "Lazy %s environment init failed for task %s: %s",
  209. env_type, effective_task_id[:8], exc,
  210. )
  211. return None
  212. with _env_lock:
  213. _active_environments[effective_task_id] = new_env
  214. _last_activity[effective_task_id] = time.time()
  215. logger.info(
  216. "%s environment lazily initialized for task %s",
  217. env_type, effective_task_id[:8],
  218. )
  219. return new_env
  220. def is_persistent_env(task_id: str) -> bool:
  221. """True if *task_id*'s active env persists across turns.
  222. The agent loop skips per-turn teardown for these (persistent docker,
  223. daytona, modal, …); non-persistent backends are torn down at end of turn
  224. to prevent leakage, and the idle reaper handles the rest. Session-scoped
  225. docker containers count as persistent HERE: their lifetime is the session
  226. (removed by ``AIAgent.close()`` → ``cleanup_vm`` and the idle reaper).
  227. """
  228. env = get_active_env(task_id)
  229. if env is None:
  230. return False
  231. return bool(getattr(env, "_session_scoped", False) or getattr(env, "_persistent", False))
  232. def cleanup_all_environments():
  233. """Clean up ALL active environments (process exit). Use with caution."""
  234. from tools.environments.base import kill_live_foreground_processes
  235. from tools.terminal_tool import _active_environments
  236. # A command still running when the host exits would outlive it in its own process group.
  237. kill_live_foreground_processes()
  238. cleaned = 0
  239. for task_id in list(_active_environments.keys()):
  240. try:
  241. cleanup_vm(task_id)
  242. cleaned += 1
  243. except Exception as e:
  244. logger.error("Error cleaning %s: %s", task_id, e, exc_info=True)
  245. if cleaned > 0:
  246. logger.info("Cleaned %d environments", cleaned)
  247. return cleaned
  248. def cleanup_vm(task_id: str, *, force_remove: bool = False):
  249. """Manually clean up a specific environment by task_id.
  250. *force_remove* is forwarded to backends that accept it (currently only
  251. ``DockerEnvironment``). Default False matches session-lifecycle semantics:
  252. callers (``AIAgent.close()`` on TUI/gateway session teardown, the per-turn
  253. cleanup of non-persistent envs) must honor the user's persist-mode
  254. preference — stopping the container here would break the "ONE long-lived
  255. container shared across sessions" contract. Pass ``force_remove=True``
  256. only for user-initiated teardown. The idle reaper calls ``env.cleanup()``
  257. directly, so persist-mode idle envs are likewise no-op'd; only the orphan
  258. reaper at next startup reclaims them.
  259. """
  260. env = _unregister_env(task_id)
  261. _clear_file_ops_cache(task_id)
  262. if env is None:
  263. return
  264. _teardown_env(
  265. env, task_id, force_remove=force_remove,
  266. done_msg="Manually cleaned up environment for task: %s",
  267. )
  268. def _evict_environment_for_task(task_id: Optional[str]) -> None:
  269. """Drop any cached env for *task_id* (and its collapsed key) after an
  270. infrastructure failure, so later calls don't reuse a dead connection."""
  271. from tools.terminal_tool import (
  272. _active_environments, _env_lock, _last_activity, _resolve_container_task_id,
  273. )
  274. keys = {_resolve_container_task_id(task_id)}
  275. if task_id:
  276. keys.add(task_id)
  277. evicted = []
  278. with _env_lock:
  279. for key in keys:
  280. env = _active_environments.pop(key, None)
  281. _last_activity.pop(key, None)
  282. if env is not None:
  283. evicted.append(env)
  284. for env in evicted:
  285. with _quiet("cleanup of degraded environment failed"):
  286. env.cleanup()