streams.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274
  1. """A long-lived process INSIDE a terminal backend with both stdio pipes open.
  2. ``BaseEnvironment.execute`` is request/response: it captures output and returns. The desktop that
  3. follows the terminal backend needs three things that are not: the RFB byte stream between the
  4. Desktop pane and the sandbox's Xvnc, cua-driver's MCP-over-stdio server, and the bot's Chromium,
  5. which must outlive the call that started it. All three are "spawn argv in the sandbox, keep
  6. stdin/stdout as pipes". Each spawn-per-call backend has a local argv prefix that gives exactly that
  7. (``docker exec -i``, ``ssh``); this module is the one place that knows them.
  8. SDK backends (modal, daytona, vercel) have no argv prefix: ``exec_prefix`` returns None there and
  9. the desktop stays on the gateway host (or refuses, per ``bot_desktop.placement``). Plugin backends
  10. that declare ``supports_streams`` supply the prefix from their environment's own ``exec_prefix()``.
  11. """
  12. from __future__ import annotations
  13. import logging
  14. import shlex
  15. import socket
  16. import subprocess
  17. import threading
  18. from typing import Optional, Sequence
  19. from tools.environments.base import BaseEnvironment
  20. logger = logging.getLogger(__name__)
  21. def exec_prefix(env: BaseEnvironment, *, user: Optional[str] = None, interactive: bool = True) -> Optional[list[str]]:
  22. """Local argv that runs its remainder inside ``env``, or None for backends without one.
  23. ``user`` selects the sandbox-side account (docker only; ssh runs as the configured login).
  24. ``interactive`` keeps stdin open (``docker exec -i``); ssh always does.
  25. """
  26. from tools.environments.docker import DockerEnvironment
  27. from tools.environments.ssh import SSHEnvironment
  28. if isinstance(env, DockerEnvironment):
  29. container = getattr(env, "_container_id", None)
  30. if not container:
  31. return None
  32. argv = [env._docker_exe, "exec"]
  33. if interactive:
  34. argv.append("-i")
  35. if user:
  36. argv += ["-u", user]
  37. return argv + [container]
  38. if isinstance(env, SSHEnvironment):
  39. return env._build_ssh_command()
  40. return _plugin_exec_prefix(env, user=user, interactive=interactive)
  41. def _plugin_exec_prefix(env: BaseEnvironment, *, user: Optional[str], interactive: bool) -> Optional[list[str]]:
  42. """A plugin environment's own ``exec_prefix()`` (``TerminalEnvironmentProvider.supports_streams``)."""
  43. hook = getattr(env, "exec_prefix", None)
  44. if not callable(hook):
  45. return None
  46. try:
  47. prefix = hook(user=user, interactive=interactive)
  48. except Exception:
  49. logger.warning("%s.exec_prefix() failed; treating the backend as unable to stream",
  50. type(env).__name__, exc_info=True)
  51. return None
  52. if not prefix:
  53. return None
  54. if not isinstance(prefix, (list, tuple)):
  55. # A plain string would be split into one-character argv words.
  56. logger.warning("%s.exec_prefix() returned %s, not a list of argv words; treating the backend as unable "
  57. "to stream", type(env).__name__, type(prefix).__name__)
  58. return None
  59. return [str(part) for part in prefix]
  60. def sandbox_temp_dir(env: BaseEnvironment) -> str:
  61. """``env.get_temp_dir()``, or ``/tmp`` for a plugin environment that doesn't subclass ``BaseEnvironment``."""
  62. get_temp_dir = getattr(env, "get_temp_dir", None)
  63. return str(get_temp_dir()) if callable(get_temp_dir) else "/tmp" # no-tmp: ok — sandbox-side temp dir
  64. def supports_streams(env: BaseEnvironment) -> bool:
  65. return exec_prefix(env, interactive=True) is not None
  66. def remote_argv(prefix: Sequence[str], argv: Sequence[str], *, env: Optional[dict] = None,
  67. shell_joined: bool = False) -> list[str]:
  68. """``prefix`` + a ``bash -c`` that exports ``env`` and execs ``argv``.
  69. The script is built once, then delivered the way the client needs it: argv-preserving clients (docker,
  70. apptainer) get ``["bash", "-c", script]``; ``shell_joined`` clients (OpenSSH joins the words after the host
  71. with spaces and the remote login shell splits them again) get the whole ``bash -c '<script>'`` as ONE
  72. quoted word, or the remote side runs ``bash -c export`` and hands the rest to the login shell
  73. (``SSHEnvironment.run_bash`` quotes for the same reason). Prefer :func:`remote_command`, which knows."""
  74. exports = " ".join(f"export {k}={shlex.quote(v)};" for k, v in (env or {}).items())
  75. script = f"{exports} exec {' '.join(shlex.quote(a) for a in argv)}"
  76. if shell_joined:
  77. return [*prefix, shlex.join(["bash", "-c", script])]
  78. return [*prefix, "bash", "-c", script]
  79. def shell_joined(env: BaseEnvironment) -> bool:
  80. from tools.environments.ssh import SSHEnvironment
  81. return isinstance(env, SSHEnvironment)
  82. def remote_command(env: BaseEnvironment, argv: Sequence[str], *, child_env: Optional[dict] = None,
  83. user: Optional[str] = None, interactive: bool = True) -> Optional[list[str]]:
  84. """Full local argv running ``argv`` inside ``env`` with ``child_env`` exported, quoted for that backend's
  85. client; None for a backend without an exec prefix."""
  86. prefix = exec_prefix(env, user=user, interactive=interactive)
  87. if prefix is None:
  88. return None
  89. return remote_argv(prefix, argv, env=child_env, shell_joined=shell_joined(env))
  90. def open_stream(env: BaseEnvironment, argv: Sequence[str], *, child_env: Optional[dict] = None,
  91. user: Optional[str] = None, stderr=subprocess.DEVNULL) -> subprocess.Popen:
  92. """Spawn ``argv`` inside ``env`` with stdin and stdout as pipes (bytes). Raises ``RuntimeError`` for a
  93. backend that cannot host a stream."""
  94. command = remote_command(env, argv, child_env=child_env, user=user, interactive=True)
  95. if command is None:
  96. raise RuntimeError(f"{type(env).__name__} cannot host a long-lived stdio stream")
  97. return subprocess.Popen( # windows-footgun: ok — the prefix is a Linux sandbox's client (docker/ssh/apptainer)
  98. command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, close_fds=True)
  99. def run_in(env: BaseEnvironment, argv: Sequence[str], *, child_env: Optional[dict] = None, user: Optional[str] = None,
  100. timeout: float = 30.0, stdin: Optional[bytes] = None) -> subprocess.CompletedProcess:
  101. """One short command inside ``env`` with captured bytes output (probes: ``command -v``, ``cat env``)."""
  102. command = remote_command(env, argv, child_env=child_env, user=user, interactive=stdin is not None)
  103. if command is None:
  104. raise RuntimeError(f"{type(env).__name__} cannot exec argv")
  105. return subprocess.run( # windows-footgun: ok — Linux sandbox client
  106. command, input=stdin, capture_output=True, timeout=timeout, check=False)
  107. # ── TCP port forwarding through the exec stream ──────────────────────────────────────────────────────
  108. # A sandbox's loopback ports (Chromium's CDP endpoint, a dev server) are unreachable from the gateway host:
  109. # Docker bridge / ssh remote / apptainer netns. Rather than teach every client a backend-specific tunnel
  110. # (`docker port` needs a published port at create time; ssh -L is ssh-only), a local listener proxies each
  111. # accepted connection over ONE exec stream running a tiny relay inside the sandbox — the same mechanism
  112. # the Bot Screen's RFB relay uses, so it works on every backend that can exec.
  113. _TCP_RELAY = (
  114. "import os,socket,sys,threading\n"
  115. "s=socket.create_connection((sys.argv[1],int(sys.argv[2])))\n"
  116. "def up():\n"
  117. " while True:\n"
  118. " d=os.read(0,65536)\n"
  119. " if not d: break\n"
  120. " s.sendall(d)\n"
  121. " s.shutdown(socket.SHUT_WR)\n"
  122. "threading.Thread(target=up,daemon=True).start()\n"
  123. "while True:\n"
  124. " d=s.recv(65536)\n"
  125. " if not d: break\n"
  126. " os.write(1,d)\n"
  127. )
  128. _forwards: dict[tuple[int, str, int], PortForward] = {}
  129. _forwards_lock = threading.Lock()
  130. class PortForward:
  131. """Local ``127.0.0.1:<local_port>`` whose connections land on ``remote_host:remote_port`` inside ``env``."""
  132. def __init__(self, env: BaseEnvironment, remote_host: str, remote_port: int, *, user: Optional[str]):
  133. self.env, self.remote_host, self.remote_port, self.user = env, remote_host, remote_port, user
  134. self._server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  135. self._server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  136. self._server.bind(("127.0.0.1", 0))
  137. self._server.listen(16)
  138. self.local_port: int = self._server.getsockname()[1]
  139. self._closed = False
  140. # Connections that never reached the target (relay not started, it exited non-zero before
  141. # relaying a byte back, or the browser path it asked for is gone): the one cheap signal that
  142. # whatever listened there may be gone.
  143. self.failures = 0
  144. threading.Thread(target=self._accept_loop, name=f"port-forward-{self.local_port}", daemon=True).start()
  145. def _accept_loop(self) -> None:
  146. while not self._closed:
  147. try:
  148. conn, _ = self._server.accept()
  149. except OSError:
  150. return
  151. threading.Thread(target=self._serve, args=(conn,), daemon=True).start()
  152. def _serve(self, conn: socket.socket) -> None:
  153. try:
  154. proc = open_stream(self.env, ["python3", "-c", _TCP_RELAY, self.remote_host, str(self.remote_port)],
  155. user=self.user)
  156. except Exception:
  157. conn.close()
  158. self.failures += 1
  159. return
  160. request: list[bytes] = []
  161. def to_remote() -> None:
  162. try:
  163. while True:
  164. data = conn.recv(65536)
  165. if not data:
  166. break
  167. if not request:
  168. request.append(data)
  169. proc.stdin.write(data)
  170. proc.stdin.flush()
  171. except (OSError, ValueError):
  172. pass
  173. finally:
  174. try:
  175. proc.stdin.close()
  176. except OSError:
  177. pass
  178. threading.Thread(target=to_remote, daemon=True).start()
  179. reply = b""
  180. try:
  181. while True:
  182. data = proc.stdout.read1(65536) if hasattr(proc.stdout, "read1") else proc.stdout.read(65536)
  183. if not data:
  184. break
  185. reply = reply or data
  186. conn.sendall(data)
  187. except OSError:
  188. pass
  189. finally:
  190. try:
  191. conn.close()
  192. finally:
  193. if (not reply and _exited_nonzero(proc)) or _stale_browser_path(request, reply):
  194. self.failures += 1
  195. proc.kill()
  196. def close(self) -> None:
  197. self._closed = True
  198. try:
  199. self._server.close()
  200. except OSError:
  201. pass
  202. def _stale_browser_path(request: list[bytes], reply: bytes) -> bool:
  203. """A Chromium relaunched on the same port answers the old ``/devtools/browser/<id>`` with 404: the
  204. endpoint the caller cached is gone even though the port relayed a reply."""
  205. return (bool(request) and request[0].startswith(b"GET /devtools/browser/")
  206. and reply.startswith((b"HTTP/1.1 404 ", b"HTTP/1.0 404 ")))
  207. def _exited_nonzero(proc: subprocess.Popen) -> bool:
  208. try:
  209. return proc.wait(timeout=2) != 0
  210. except subprocess.TimeoutExpired:
  211. return False
  212. def forward_failures(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1") -> int:
  213. """How many connections through ``forward_port(env, remote_port)`` never reached the target (0 while no
  214. forward is open). A caller caching "the target is up" re-checks only once this changes."""
  215. with _forwards_lock:
  216. fwd = _forwards.get((id(env), remote_host, int(remote_port)))
  217. return fwd.failures if fwd is not None else 0
  218. def forward_port(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1",
  219. user: Optional[str] = None) -> int:
  220. """Local port on 127.0.0.1 that reaches ``remote_host:remote_port`` inside ``env``; one listener per
  221. (env, host, port) is kept for the process lifetime (daemon threads, nothing to clean up)."""
  222. key = (id(env), remote_host, int(remote_port))
  223. with _forwards_lock:
  224. fwd = _forwards.get(key)
  225. if fwd is None or fwd._closed:
  226. fwd = _forwards[key] = PortForward(env, remote_host, int(remote_port), user=user)
  227. return fwd.local_port