base.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. """Base class for all Hermes execution environment backends.
  2. Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
  3. A session snapshot (env vars, functions, aliases) is captured once at init and
  4. re-sourced before each command. CWD persists via in-band stdout markers (remote)
  5. or a temp file (local). Cohesive pieces live in sibling modules (``base_output``,
  6. ``base_session_env``, ``base_wait``, ``path_utils``).
  7. """
  8. import json
  9. import logging
  10. import os
  11. import shlex
  12. import subprocess
  13. import threading
  14. import time
  15. import uuid
  16. from abc import ABC, abstractmethod
  17. from pathlib import Path
  18. from typing import Callable, Iterable
  19. from hermes_constants import get_hermes_home
  20. from tools.interrupt import consume_yield, is_interrupted, is_thread_interrupted
  21. from tools.environments.base_output import (
  22. ProcessHandle, _finalize_wait_result, _new_output_collector, _start_drain_thread,
  23. )
  24. from tools.environments.base_session_env import (
  25. _SHELL_ENV_NAME_RE, _SNAP_TMP_SUFFIX, _cwd_marker, _pipe_marker, _snapshot_bootstrap_script, _split_cwd_marker,
  26. _wrap_command_script,
  27. )
  28. from tools.environments.base_wait import _WaitTrace
  29. from utils import atomic_json_write, env_var_enabled, file_signature
  30. logger = logging.getLogger(__name__)
  31. # Opt-in debug tracing for the interrupt/activity/poll machinery
  32. # (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
  33. _DEBUG_INTERRUPT = env_var_enabled("HERMES_DEBUG_INTERRUPT")
  34. # Extra seconds the ``run_bounded_sync`` backstop waits past the inner ``_wait_for_process``
  35. # deadline: the inner loop returns partial output + 124; the outer bound only fires when that
  36. # loop never returns. Keep small so a healthy timeout still comes from the inner path.
  37. # The inner poll loop is what returns partial output + returncode 124; this outer bound only exists for when
  38. # that loop itself never returns (family A of #94285: a blocked wait that silently disables asyncio timers).
  39. _EXECUTE_WAIT_BOUND_GRACE_S = 2.0
  40. if _DEBUG_INTERRUPT:
  41. # quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
  42. # swallow every trace; force this logger back to INFO in the opt-in case.
  43. logger.setLevel(logging.INFO)
  44. # Thread-local activity callback: the agent sets it before a tool call so
  45. # long-running _wait_for_process loops can report liveness to the gateway.
  46. _activity_callback_local = threading.local()
  47. # Foreground commands in flight in THIS process, across every environment. Each runs in its
  48. # own session/process group, so a host that exits mid-command (TUI client gone, SIGTERM) would
  49. # orphan the whole tree; the process-exit funnel ``cleanup_all_environments`` kills them.
  50. _live_foreground: "dict[int, tuple[BaseEnvironment, ProcessHandle]]" = {}
  51. # Reentrant, and the hard-exit path only ever takes it with a timeout: a signal handler can run
  52. # on a thread that already holds it.
  53. _live_foreground_cond = threading.Condition(threading.RLock())
  54. _exit_fenced = False # one-way, set by the hard-exit kill: no foreground command spawns after it
  55. _spawns_in_flight = 0 # past the fence check, child maybe alive, not yet in _live_foreground
  56. _HARD_KILL_BUDGET_S = 0.5
  57. def _enter_foreground_spawn() -> bool:
  58. global _spawns_in_flight
  59. with _live_foreground_cond:
  60. if _exit_fenced:
  61. return False
  62. _spawns_in_flight += 1
  63. return True
  64. def _leave_foreground_spawn(env: "BaseEnvironment", spawned) -> bool:
  65. """Publish ``spawned`` (None: the spawn failed); True when the exit fence went up meanwhile."""
  66. global _spawns_in_flight
  67. with _live_foreground_cond:
  68. _spawns_in_flight -= 1
  69. if spawned is not None:
  70. _live_foreground[id(spawned)] = (env, spawned)
  71. _live_foreground_cond.notify_all()
  72. return _exit_fenced
  73. def _quiet_kill(kill: Callable, proc) -> None:
  74. try:
  75. kill(proc)
  76. except Exception:
  77. logger.debug("exit-time kill of a foreground command failed", exc_info=True)
  78. def kill_live_foreground_processes(*, now: bool = False) -> int:
  79. """Kill every in-flight foreground command's process tree; returns how many were signalled.
  80. ``now=True`` is for a caller about to ``os._exit``: the graceful kill TERMs, waits and only then
  81. KILLs, so a SIGTERM-ignoring command outlives a hard exit that lands inside that window. It also
  82. raises the exit fence and waits for spawns already past it to register, so no command started
  83. around the snapshot survives, and it never blocks past ``_HARD_KILL_BUDGET_S``: SDK cancels
  84. (Modal, Daytona, Vercel) run on daemon threads under that one deadline."""
  85. global _exit_fenced
  86. if not now:
  87. with _live_foreground_cond:
  88. live = list(_live_foreground.values())
  89. for env, proc in live:
  90. _quiet_kill(env._kill_process, proc)
  91. return len(live)
  92. deadline = time.monotonic() + _HARD_KILL_BUDGET_S
  93. _exit_fenced = True
  94. if _live_foreground_cond.acquire(timeout=_HARD_KILL_BUDGET_S):
  95. try:
  96. _live_foreground_cond.wait_for(lambda: _spawns_in_flight == 0, max(0.0, deadline - time.monotonic()))
  97. live = list(_live_foreground.values())
  98. finally:
  99. _live_foreground_cond.release()
  100. else: # the holder is stuck under our signal: a lock-free copy beats hanging the exit
  101. live = list(_live_foreground.values())
  102. remote = []
  103. for env, proc in live:
  104. if isinstance(proc, subprocess.Popen): # killpg/kill: never blocks
  105. _quiet_kill(env._force_kill_process, proc)
  106. else:
  107. remote.append(threading.Thread(target=_quiet_kill, args=(env._force_kill_process, proc), daemon=True))
  108. remote[-1].start()
  109. for t in remote:
  110. t.join(max(0.0, deadline - time.monotonic()))
  111. return len(live)
  112. class FileFetchError(RuntimeError):
  113. """A file could not be extracted from the backend filesystem."""
  114. # Pulling a large artifact over a slow exec channel can outlast the per-command terminal timeout.
  115. _FETCH_TIMEOUT_SECONDS = 300
  116. class EnvironmentConnectionError(RuntimeError):
  117. """Infrastructure/connection-class failure of a terminal backend (SSH host down, Docker
  118. daemon not running, remote sync on a dead link) — never a command that merely exited
  119. nonzero. Subclassing RuntimeError keeps every ``except RuntimeError`` catcher working.
  120. ``terminal_tool`` turns this into a structured ``status: "degraded"`` result; the failed
  121. backend is never cached, so a later call retries from scratch."""
  122. def __init__(self, reason: str, *, retry_hint: str = ""):
  123. super().__init__(reason)
  124. self.reason = reason
  125. self.retry_hint = retry_hint or (
  126. "This is an infrastructure failure, not a command failure. "
  127. "Verify the backend is reachable (network, service running, "
  128. "credentials), then retry the same command — recovery is "
  129. "automatic once the backend is back.")
  130. def set_activity_callback(cb: Callable[[str], None] | None) -> None:
  131. """Register a callback that _wait_for_process fires periodically."""
  132. _activity_callback_local.callback = cb
  133. def get_activity_callback() -> Callable[[str], None] | None:
  134. """Thread-local activity callback; capture it before handing work to another thread.
  135. Public accessor for callers outside this module that need to capture the calling thread's callback
  136. before handing work to another thread (the callback is thread-local, so a freshly spawned thread cannot
  137. read it back) — e.g. the manual cron-run heartbeat (#76502).
  138. """
  139. return getattr(_activity_callback_local, "callback", None)
  140. def touch_activity_if_due(state: dict, label: str) -> None:
  141. """Fire the activity callback at most once every ``state['interval']`` (default 10 s).
  142. *state* holds ``last_touch``/``start`` monotonic timestamps. Swallows all exceptions."""
  143. now = time.monotonic()
  144. if now - state["last_touch"] < state.get("interval", 10.0):
  145. return
  146. state["last_touch"] = now
  147. try:
  148. cb = get_activity_callback()
  149. if cb:
  150. cb(f"{label} ({int(now - state['start'])}s elapsed)")
  151. except Exception:
  152. logger.debug("activity callback failed during a long-running command", exc_info=True)
  153. def get_sandbox_dir() -> Path:
  154. """Host-side root for all sandbox storage (Docker workspaces, plugin backends'
  155. host-side state). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
  156. custom = os.getenv("TERMINAL_SANDBOX_DIR")
  157. p = Path(custom) if custom else get_hermes_home() / "sandboxes"
  158. p.mkdir(parents=True, exist_ok=True)
  159. return p
  160. def _load_json_store(path: Path) -> dict:
  161. """Treat a missing or damaged snapshot store as empty."""
  162. try:
  163. data = json.loads(path.read_text(encoding="utf-8-sig"))
  164. except (OSError, ValueError):
  165. return {}
  166. return data if isinstance(data, dict) else {}
  167. def _save_json_store(path: Path, data: dict) -> None:
  168. """Write *data* as pretty-printed JSON to *path* atomically (readers never see a torn file)."""
  169. path.parent.mkdir(parents=True, exist_ok=True)
  170. atomic_json_write(path, data, indent=2)
  171. _JSON_STORE_LOCK_HOLDER = threading.local()
  172. def _update_json_store(path: Path, update: Callable[[dict], None]) -> None:
  173. """Read-modify-write the JSON store at *path* under a cross-process lock.
  174. Concurrent sandbox cleanups (delegate children, gateway sessions, the idle reaper) each
  175. record a snapshot id; an unlocked load/mutate/save lets one cleanup drop the other's entry.
  176. An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
  177. update: a sandbox lifecycle step (modal's restore fallback included) must not abort on it."""
  178. from contextlib import ExitStack
  179. from hermes_cli.auth import _file_lock
  180. lock_path = path.with_name(f"{path.name}.lock")
  181. with ExitStack() as stack:
  182. try:
  183. stack.enter_context(_file_lock(
  184. lock_path, _JSON_STORE_LOCK_HOLDER, 10.0, f"Timed out waiting for the snapshot store lock ({path})",
  185. ))
  186. except (OSError, TimeoutError):
  187. logger.warning("Updating %s without its lock", path, exc_info=True)
  188. data = _load_json_store(path)
  189. before = dict(data)
  190. update(data)
  191. if data != before:
  192. _save_json_store(path, data)
  193. def _file_mtime_key(host_path: str) -> tuple[int, int, int, int] | None:
  194. """Return the ``file_signature`` change key, or ``None`` if unreadable.
  195. mtime + size alone miss a same-size credential refresh landing in the same mtime tick;
  196. inode and ctime catch the atomic replace / rewrite."""
  197. try:
  198. return file_signature(Path(host_path).stat())
  199. except OSError:
  200. return None
  201. def env_uses_host_filesystem(env: object) -> bool:
  202. """Whether the file tools may touch *env*'s files on the host (``BaseEnvironment.uses_host_filesystem``).
  203. Strict ``is True``: an environment without the declaration (``None``, a duck-typed plugin
  204. backend, a test double) is not the host."""
  205. return getattr(env, "uses_host_filesystem", False) is True
  206. # ---------------------------------------------------------------------------
  207. # BaseEnvironment
  208. # ---------------------------------------------------------------------------
  209. class BaseEnvironment(ABC):
  210. """Common interface and unified execution flow for all Hermes backends. Subclasses
  211. implement ``_run_bash()`` and ``cleanup()``; the base provides ``execute()`` with
  212. snapshot sourcing, CWD tracking, interrupt handling and timeout enforcement."""
  213. # How execute() hands stdin to _run_bash: "pipe" (process stdin, default),
  214. # "payload" (passed through as stdin_data; the backend transports it via its
  215. # SDK -- Modal/managed Modal stream it, Daytona/Vercel stage a file) or
  216. # "heredoc" (embedded in the command; no built-in backend, plugins only).
  217. _stdin_mode: str = "pipe" # "pipe" | "payload" | "heredoc"
  218. # True only when commands execute on the SAME host as the Hermes process
  219. # (LocalEnvironment); controller-host facts then describe the execution target.
  220. is_local: bool = False
  221. # True when the file tools may read, write, patch and search this backend's files straight on
  222. # the host instead of through execute() (only LocalEnvironment). A host backend that confines its
  223. # commands (a sandbox hiding paths, mounting them read-only or giving them their own $TMPDIR) sets
  224. # it False, so the file tools see exactly what its commands see. Read via env_uses_host_filesystem().
  225. uses_host_filesystem: bool = False
  226. # Snapshot creation timeout (override for slow cold-starts).
  227. _snapshot_timeout: int = 30
  228. # Re-dump exported env into the snapshot after each command (see __init__).
  229. _persist_env: bool = True
  230. # Opt in only when a timed-out probe can kill its command without tearing
  231. # down the whole backend. SDK adapters cancel by stopping the sandbox. Plugin
  232. # backends set this too; the name avoids the sudoers keyword the install
  233. # scanner flags in plugin code.
  234. _passwordless_sudo_probe_supported: bool = False
  235. # Local and Docker override this because they resolve allowlisted values
  236. # through the active profile scope; other backends keep plain snapshots.
  237. _profile_scoped_passthrough: bool = False
  238. # Per-session PIPESTATUS marker set in __init__; empty means the wrapper emits none.
  239. _pipe_marker: str = ""
  240. def get_temp_dir(self) -> str:
  241. """Return the backend temp directory used for session artifacts.
  242. Most sandboxed backends use ``/tmp`` inside the target environment.
  243. LocalEnvironment overrides this on hosts where ``/tmp`` may be missing
  244. and ``TMPDIR`` is the portable writable location.
  245. """
  246. return "/tmp" # no-tmp: ok — sandbox-side (remote container) temp dir, not the host
  247. def __init__(self, cwd: str, timeout: int, env: dict | None = None, persist_env: bool = True):
  248. self.cwd = cwd
  249. self.timeout = timeout
  250. self.env = env or {}
  251. # False: each command sources only the login-time snapshot, so exported vars do not
  252. # carry over (terminal.persistent_shell / TERMINAL_LOCAL_PERSISTENT). CWD still does.
  253. self._persist_env = persist_env
  254. self._session_id = uuid.uuid4().hex[:12]
  255. temp_dir = self.get_temp_dir().rstrip("/") or "/"
  256. self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
  257. self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
  258. self._cwd_marker = _cwd_marker(self._session_id)
  259. self._pipe_marker = _pipe_marker(self._session_id)
  260. self._snapshot_ready = False
  261. self._snapshot_passthrough_names: set[str] = set()
  262. # True when login bash is unusable (e.g. broken Git-for-Windows startup)
  263. # so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
  264. self._prefer_nonlogin = False
  265. # --- Abstract methods ---
  266. def _run_bash(
  267. self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None,
  268. ) -> ProcessHandle:
  269. """Spawn a bash process to run *cmd_string*; every backend overrides this."""
  270. raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
  271. @abstractmethod
  272. def cleanup(self):
  273. """Release backend resources (container, instance, connection)."""
  274. ...
  275. # --- File extraction (backend -> host) ---
  276. def fetch_file(self, remote_path: str, local_dest: Path, *, max_bytes: int) -> None:
  277. """Copy a regular file out of the backend filesystem to ``local_dest`` on the host.
  278. One transport for every backend: base64 over the exec channel, bounded INSIDE the sandbox
  279. (``head -c max+1`` so /dev/zero can't stream unbounded data into host memory — the same
  280. shape ``tools.image_source`` uses to read sandbox images). The payload is fenced between
  281. unique markers so login-shell noise in the merged stdout/stderr can't corrupt the decode.
  282. Raises :class:`FileFetchError` on a missing/unreadable/oversized file.
  283. """
  284. import base64
  285. import binascii
  286. marker = f"__HERMES_FETCH_{uuid.uuid4().hex[:12]}__"
  287. quoted = shlex.quote(remote_path)
  288. # ``[ -f ]`` follows symlinks, so a link to a denied host file is judged by the CALLER on
  289. # ``readlink -f`` output before any bytes move.
  290. result = self.execute(
  291. f"[ -f {quoted} ] && echo {marker} && head -c {max_bytes + 1} < {quoted} | base64 && echo {marker}",
  292. timeout=_FETCH_TIMEOUT_SECONDS, rewrite_compound_background=False)
  293. output = result.get("output") or ""
  294. first, last = output.find(marker), output.rfind(marker)
  295. if int(result.get("returncode") or 0) != 0 or first == -1 or last <= first:
  296. raise FileFetchError(f"could not read {remote_path!r} in the sandbox (missing, not a regular file, or unreadable)")
  297. try:
  298. data = base64.b64decode("".join(output[first + len(marker):last].split()), validate=True)
  299. except (ValueError, binascii.Error) as exc:
  300. raise FileFetchError(f"transfer of {remote_path!r} was corrupted in transit: {exc}") from exc
  301. if len(data) > max_bytes:
  302. raise FileFetchError(f"{remote_path!r} exceeds the {max_bytes // (1024 * 1024)} MB delivery limit")
  303. Path(local_dest).write_bytes(data)
  304. def fetch_realpath(self, remote_path: str) -> str | None:
  305. """``readlink -f`` inside the backend, or None when it cannot be resolved."""
  306. result = self.execute(f"readlink -f {shlex.quote(remote_path)} 2>/dev/null", rewrite_compound_background=False)
  307. if int(result.get("returncode") or 0) != 0:
  308. return None
  309. return next((ln.strip() for ln in reversed((result.get("output") or "").splitlines()) if ln.strip().startswith("/")), None)
  310. # --- Session snapshot (init_session) ---
  311. def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
  312. """Return backend-specific names that must not persist in snapshots."""
  313. return ()
  314. def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
  315. """Profile-scoped names that must not persist in the snapshot. Monotonic for the
  316. environment lifetime: an allowlist can be cleared after a value was captured, and
  317. retaining the exclusion keeps that old value from leaking to a later profile."""
  318. if not self._profile_scoped_passthrough:
  319. return ()
  320. try:
  321. from agent.secret_scope import is_multiplex_active
  322. if is_multiplex_active():
  323. from tools.env_passthrough import get_all_passthrough
  324. names = (*get_all_passthrough(), *self._additional_profile_scoped_passthrough_names())
  325. self._snapshot_passthrough_names.update(
  326. name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name))
  327. except Exception:
  328. logger.debug("Could not refresh profile-scoped snapshot exclusions", exc_info=True)
  329. return tuple(sorted(self._snapshot_passthrough_names))
  330. def _snapshot_script_kwargs(self, cwd: str) -> dict:
  331. """Quoting inputs shared by the bootstrap and per-command wrapper scripts.
  332. ``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) let the Windows
  333. subclass rewrite ``C:\\...`` to ``/c/...`` so ``cd`` resolves and MSYS doesn't choke."""
  334. return dict(
  335. quoted_cwd=self._quote_cwd_for_cd(cwd),
  336. quoted_snap=self._quote_shell_path(self._snapshot_path),
  337. snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX),
  338. cwd_marker=self._cwd_marker)
  339. def init_session(self):
  340. """Capture the login shell environment into the snapshot file (once, after construction).
  341. On success ``_snapshot_ready`` is set so commands source the snapshot instead of running
  342. under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login
  343. probe shows login bash itself is dead, in which case prefer ``bash -c``."""
  344. bootstrap = _snapshot_bootstrap_script(
  345. excluded_names=self._snapshot_excluded_passthrough_names(), **self._snapshot_script_kwargs(self.cwd))
  346. try:
  347. proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
  348. result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
  349. if int(result.get("returncode") or 0) != 0:
  350. raise RuntimeError(f"snapshot bootstrap failed with exit code {result.get('returncode')}")
  351. self._snapshot_ready = True
  352. self._update_cwd(result)
  353. logger.info("Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd)
  354. except Exception as exc:
  355. self._snapshot_ready = False
  356. self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
  357. if self._prefer_nonlogin:
  358. logger.warning(
  359. "init_session failed (session=%s): %s — "
  360. "login bash unusable; falling back to non-login bash -c",
  361. self._session_id, exc)
  362. else:
  363. logger.warning(
  364. "init_session failed (session=%s): %s — falling back to bash -l per command",
  365. self._session_id, detail)
  366. def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
  367. """Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
  368. probe_timeout = min(15, self._snapshot_timeout)
  369. try:
  370. probe = self._run_bash("true", login=False, timeout=probe_timeout)
  371. probe_result = self._wait_for_process(probe, timeout=probe_timeout)
  372. prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
  373. if not prefer_nonlogin:
  374. detail = (probe_result.get("stdout") or detail).strip() or detail
  375. return prefer_nonlogin, detail
  376. except Exception as probe_exc:
  377. return False, f"{detail}; non-login probe: {probe_exc}"
  378. # --- Command wrapping ---
  379. @staticmethod
  380. def _quote_cwd_for_cd(cwd: str) -> str:
  381. """Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
  382. goes through ``$HOME`` so suffixes with spaces stay one word)."""
  383. if cwd == "~":
  384. return cwd
  385. if cwd == "~/":
  386. return "$HOME"
  387. if cwd.startswith("~/"):
  388. return f"$HOME/{shlex.quote(cwd[2:])}"
  389. return shlex.quote(cwd)
  390. def _quote_shell_path(self, path: str) -> str:
  391. """Quote *path* for a bash script. LocalEnvironment overrides this to
  392. rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
  393. return shlex.quote(path)
  394. def _wrap_command(self, command: str, cwd: str) -> str:
  395. """Full bash script: source snapshot, cd, run, re-dump env, emit CWD markers."""
  396. return _wrap_command_script(
  397. command,
  398. passthrough_names=self._snapshot_excluded_passthrough_names(),
  399. snapshot_ready=self._snapshot_ready,
  400. persist_env=self._persist_env,
  401. pipe_marker=self._pipe_marker,
  402. **self._snapshot_script_kwargs(cwd))
  403. @staticmethod
  404. def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
  405. """Redirect stdin_data to the complete command with a shell heredoc (SDK backends).
  406. A heredoc body always ends in a newline stdin_data may lack, and write_file verifies a
  407. byte-exact hash, so a process substitution re-emits the body minus that last character.
  408. Redirections apply left to right, so the substitution inherits the heredoc as its stdin;
  409. the heredoc stays outside ``<( )`` because bash 3.2 mis-parses bodies inside it. ``|| :``
  410. keeps an inherited ``set -e`` from killing the reader on read's EOF status."""
  411. delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
  412. return (f"{{\n{command}\n}} << '{delimiter}' < <(IFS= read -r -d '' s || :; printf '%s' \"${{s%?}}\")\n"
  413. f"{stdin_data}\n{delimiter}")
  414. def _staged_stdin_path(self) -> str:
  415. """Unique sandbox path for staging a payload-mode stdin file."""
  416. temp_dir = self.get_temp_dir().rstrip("/") or "/"
  417. return f"{temp_dir}/.hermes-stdin-{uuid.uuid4().hex}"
  418. @staticmethod
  419. def _redirect_stdin_from_file(command: str, path: str) -> str:
  420. """Prefix ``command`` so the shell reads stdin from the staged ``path`` and
  421. unlinks it before ``command`` runs (the shell then owns the payload)."""
  422. quoted = shlex.quote(path)
  423. return f"exec 0< {quoted} || exit $?\nrm -f -- {quoted} || exit $?\n{command}"
  424. # --- Process lifecycle ---
  425. def _wait_for_process(
  426. self, proc: ProcessHandle, timeout: int = 120, *,
  427. bounded_capture: bool = False, watch_interrupt_tid: int | None = None,
  428. output=None, yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
  429. """Poll-based wait with interrupt checking and stdout draining (shared, not overridden).
  430. ``yield_handler(proc, output_so_far)``: when the tool thread is asked to yield
  431. (``tools.interrupt.request_yield`` — a user message arrived mid-command), the drain
  432. thread is stopped, the still-running process is handed to the handler and its dict
  433. is returned as the result; the process is NOT killed.
  434. ``bounded_capture=True`` (foreground terminal-tool path only) retains at most
  435. ``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the
  436. process; the default keeps full fidelity for internal consumers. Fires the activity
  437. callback every 10s so the gateway's inactivity timeout doesn't kill long commands.
  438. ``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()``
  439. may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the
  440. original tid, so both bits are honored. ``KeyboardInterrupt``/``SystemExit`` mid-poll
  441. kills the process first — the local backend spawns into its own process group, so an
  442. unkilled child would be orphaned.
  443. The default (False) preserves full-fidelity capture for internal consumers — file-operation ``cat``
  444. reads feeding the patch engine, code-execution RPC reads, log reads — where truncation would corrupt
  445. data. See #64435.
  446. """
  447. if output is None:
  448. output = _new_output_collector(proc, bounded_capture)
  449. drain_stop = threading.Event() if yield_handler is not None else None
  450. drain_thread = _start_drain_thread(proc, output, drain_stop)
  451. _now = time.monotonic()
  452. deadline = _now + timeout
  453. _activity_state = {"last_touch": _now, "start": _now}
  454. trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
  455. trace.enter()
  456. def _kill_and_join():
  457. self._kill_process(proc)
  458. drain_thread.join(timeout=2)
  459. try:
  460. # Adaptive poll: start at 5ms so fast commands return in ~6ms, back
  461. # off exponentially toward 200ms so long builds don't pay poll CPU.
  462. _poll_sleep = 0.005
  463. while proc.poll() is None:
  464. trace.iterations += 1
  465. if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
  466. trace.interrupted()
  467. _kill_and_join()
  468. return {**self._finalize_wait_result(output, output.render(suffix="\n[Command interrupted]"), 130),
  469. "hermes_interrupted": True}
  470. if yield_handler is not None and consume_yield(watch_interrupt_tid):
  471. drain_stop.set()
  472. drain_thread.join(timeout=1)
  473. try:
  474. handed = yield_handler(proc, output.render())
  475. except Exception:
  476. logger.warning("yield-to-background handoff failed; continuing to wait", exc_info=True)
  477. handed = None
  478. if handed is not None:
  479. output.close_spill()
  480. return handed
  481. drain_stop.clear()
  482. drain_thread = _start_drain_thread(proc, output, drain_stop)
  483. if time.monotonic() > deadline:
  484. trace.timed_out()
  485. _kill_and_join()
  486. rendered = output.render(suffix=f"\n[Command timed out after {timeout}s]")
  487. if output.total_chars == 0:
  488. rendered = rendered.lstrip()
  489. # The flag tells Hermes' own deadline apart from a command's own ``exit 124``.
  490. return {**self._finalize_wait_result(output, rendered, 124), "hermes_timed_out": True}
  491. touch_activity_if_due(_activity_state, "terminal command running")
  492. trace.heartbeat()
  493. time.sleep(_poll_sleep)
  494. if _poll_sleep < 0.2:
  495. _poll_sleep = min(_poll_sleep * 1.5, 0.2)
  496. except (KeyboardInterrupt, SystemExit):
  497. trace.exception_exit()
  498. try:
  499. _kill_and_join()
  500. except Exception:
  501. pass # cleanup is best-effort
  502. raise
  503. # The drain thread exits promptly after bash does (~300ms idle check);
  504. # a long join here would itself indicate a bug in the drain loop.
  505. drain_thread.join(timeout=2)
  506. try:
  507. proc.stdout.close()
  508. except Exception:
  509. pass
  510. trace.natural_exit(proc.returncode)
  511. # Join the stdin writer before reading its error list: a child that exits without
  512. # reading stdin can otherwise race ahead of a recorded encode failure. The timeout
  513. # is a pure safety net (write raises BrokenPipeError once the pipe closes).
  514. stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
  515. if stdin_thread is not None:
  516. stdin_thread.join(timeout=5)
  517. rendered = output.render()
  518. result = self._finalize_wait_result(output, rendered, proc.returncode)
  519. if stdin_errors := getattr(proc, "_hermes_stdin_errors", None):
  520. result["stdin_error"] = err = str(stdin_errors[0])
  521. result["output"] = rendered + f"\n[stdin write failed: {err}]"
  522. return result
  523. _finalize_wait_result = staticmethod(_finalize_wait_result)
  524. def _kill_process(self, proc: ProcessHandle):
  525. """Terminate a process. Subclasses may override for process-group kill."""
  526. try:
  527. proc.kill()
  528. except (ProcessLookupError, PermissionError, OSError):
  529. pass
  530. def _force_kill_process(self, proc: ProcessHandle):
  531. """Kill without waiting, for a host that hard-exits next. Subclasses kill the whole tree."""
  532. self._kill_process(proc)
  533. # --- CWD extraction ---
  534. def _update_cwd(self, result: dict):
  535. """Extract CWD from command output. Override for local file-based read."""
  536. self._extract_cwd_from_output(result)
  537. def _extract_cwd_from_output(self, result: dict):
  538. """Parse the ``__HERMES_CWD_{session}__`` marker from ``result["output"]``, update
  539. ``self.cwd`` and strip the marker line. ``result["cwd_observed"]``/``["cwd"]`` are set
  540. only when THIS command emitted a marker: a killed/timed-out command emits none and
  541. ``self.cwd`` keeps the previous value. The environment is shared across sessions, so
  542. concurrent callers must read ``result["cwd"]`` rather than ``self.cwd``."""
  543. pipe_split = _split_cwd_marker(result.get("output", ""), self._pipe_marker) if self._pipe_marker else None
  544. if pipe_split is not None:
  545. statuses, cleaned = pipe_split
  546. result["output"] = cleaned
  547. try:
  548. result["pipeline_status"] = [int(s) for s in (statuses or "").split()]
  549. except ValueError:
  550. pass
  551. split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
  552. if split is None:
  553. return
  554. cwd_path, cleaned = split
  555. if cwd_path:
  556. self.cwd = cwd_path
  557. result["cwd_observed"] = True
  558. result["cwd"] = cwd_path
  559. result["output"] = cleaned
  560. # --- Hooks ---
  561. def _before_execute(self) -> None:
  562. """Hook before each command. Remote backends (SSH, Modal, Daytona)
  563. trigger their FileSyncManager here; bind-mount backends and Local don't."""
  564. def _mark_recreated(self) -> None:
  565. """Flag that the live container/sandbox was replaced while serving the
  566. current command. ``execute`` folds the flag into the result as
  567. ``environment_recreated`` so the tool layer can warn the model that
  568. background processes died and non-persisted files may be gone —
  569. without this the recovery is silent and the model keeps assuming the
  570. old workspace state (lobehub/lobehub#19329 class)."""
  571. self._recreated_notice_pending = True
  572. # --- Unified execute() ---
  573. def execute(
  574. self,
  575. command: str,
  576. cwd: str = "",
  577. *,
  578. timeout: int | None = None,
  579. stdin_data: str | None = None,
  580. rewrite_compound_background: bool = True,
  581. bounded_capture: bool = False,
  582. yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
  583. """Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True``
  584. caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal
  585. tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the
  586. patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is
  587. bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past
  588. ``timeout`` and silently disable every asyncio timer in the process.
  589. ``bounded_capture=True`` caps stdout/stderr retention at ``tool_output.max_bytes`` WHILE the stream
  590. is drained (head/tail window) instead of holding the full output in memory (#64435).
  591. See #94285.
  592. """
  593. self._before_execute()
  594. exec_command, sudo_stdin = self._prepare_command(command)
  595. # Guard against the `A && B &` subshell-wait trap by default; callers
  596. # that already produce shell-safe wrappers (spawn_via_env) pass False.
  597. if rewrite_compound_background:
  598. from tools.terminal_tool_sudo import _rewrite_compound_background
  599. exec_command = _rewrite_compound_background(exec_command)
  600. effective_timeout = timeout or self.timeout
  601. effective_cwd = cwd or self.cwd
  602. # Merge sudo stdin with caller stdin.
  603. effective_stdin = sudo_stdin + (stdin_data or "") if sudo_stdin is not None else stdin_data
  604. if effective_stdin and self._stdin_mode == "heredoc":
  605. exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
  606. effective_stdin = None
  607. wrapped = self._wrap_command(exec_command, effective_cwd)
  608. # Login shell if the snapshot failed (so the user's profile still
  609. # loads), unless login itself is broken — then non-login is the only path.
  610. login = not self._snapshot_ready and not self._prefer_nonlogin
  611. parent_tid = threading.current_thread().ident
  612. # The activity callback is thread-local and the wait runs on the
  613. # deadline worker, so copy it across or long commands look idle.
  614. parent_activity_cb = get_activity_callback()
  615. proc_holder: list = []
  616. output_holder: list = []
  617. def _spawn_and_wait() -> dict:
  618. if parent_activity_cb is not None:
  619. set_activity_callback(parent_activity_cb)
  620. if not _enter_foreground_spawn():
  621. return {"output": "[host is exiting: command not started]", "returncode": 130}
  622. spawned = None
  623. try:
  624. spawned = self._run_bash(wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin)
  625. finally:
  626. fenced = _leave_foreground_spawn(self, spawned)
  627. proc_holder.append(spawned)
  628. if fenced: # the hard-exit kill may have stopped waiting for us before we registered
  629. self._force_kill_process(spawned)
  630. output = _new_output_collector(spawned, bounded_capture)
  631. output_holder.append(output)
  632. try:
  633. return self._wait_for_process(
  634. spawned, timeout=effective_timeout, bounded_capture=bounded_capture,
  635. watch_interrupt_tid=parent_tid, output=output,
  636. **({"yield_handler": yield_handler} if yield_handler is not None else {}))
  637. finally:
  638. with _live_foreground_cond:
  639. _live_foreground.pop(id(spawned), None)
  640. def _on_timeout() -> None:
  641. if proc_holder:
  642. self._kill_spawned_tree(proc_holder[0])
  643. # Hard wall-clock backstop: ``_wait_for_process`` polls to ``effective_timeout`` on the
  644. # tool thread; if that is the event-loop thread, or the wait never returns (Windows
  645. # pipe/poll hang), every asyncio timer is silently disabled. ``run_bounded_sync`` drives
  646. # expiry from a daemon worker + ``Event.wait`` so a blocked loop cannot disable it; the
  647. # grace lets the inner loop return the partial-output 124 path.
  648. # See #94285.
  649. from agent.deadline import run_bounded_sync
  650. try:
  651. bound_s = float(effective_timeout)
  652. except (TypeError, ValueError):
  653. bound_s = 120.0 # a non-numeric timeout must not disable the backstop
  654. bound_s += _EXECUTE_WAIT_BOUND_GRACE_S
  655. try:
  656. bounded = run_bounded_sync(
  657. _spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout)
  658. except (KeyboardInterrupt, SystemExit):
  659. _on_timeout()
  660. raise
  661. if bounded.timed_out:
  662. suffix = f"\n[Command timed out after {effective_timeout}s]"
  663. if output_holder:
  664. collector = output_holder[0]
  665. result = self._finalize_wait_result(collector, collector.render(suffix=suffix).lstrip("\n"), 124)
  666. else:
  667. result = {"output": suffix.lstrip(), "returncode": 124}
  668. result["hermes_timed_out"] = True
  669. else:
  670. result = bounded.value
  671. self._update_cwd(result)
  672. if getattr(self, "_recreated_notice_pending", False):
  673. self._recreated_notice_pending = False
  674. result["environment_recreated"] = True
  675. return result
  676. def _kill_spawned_tree(self, spawned) -> None:
  677. """Best-effort kill of a wedged spawned process and its tree (backstop path)."""
  678. try:
  679. self._kill_process(spawned)
  680. except Exception:
  681. logger.debug("terminal wait-bound kill_process failed", exc_info=True)
  682. pid = getattr(spawned, "pid", None)
  683. if not pid:
  684. return
  685. try:
  686. from agent.deadline import kill_process_tree
  687. kill_process_tree(int(pid))
  688. except Exception:
  689. logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
  690. # --- Shared helpers ---
  691. def __del__(self):
  692. try:
  693. self.cleanup()
  694. except Exception:
  695. pass
  696. def _prepare_command(self, command: str) -> tuple[str, str | None]:
  697. """Rewrite sudo for a piped password, or leave it alone when this backend has NOPASSWD.
  698. Also applies the macOS ``open`` frontmost raise-ladder (a pure string
  699. rewrite, no-op on non-Darwin) so files opened via tool calls are
  700. brought to the front instead of landing behind the Hermes window.
  701. """
  702. from tools.terminal_tool_macos_open import _transform_macos_open_command
  703. from tools.terminal_tool_sudo import _transform_sudo_command
  704. return _transform_sudo_command(
  705. _transform_macos_open_command(command),
  706. sudo_nopasswd_check=self._sudo_nopasswd_works,
  707. )
  708. _SUDO_PROBE_TIMEOUT_S = 3
  709. def _sudo_nopasswd_works(self) -> bool:
  710. """``sudo -n true`` inside THIS backend (host sudo state must not leak into a sandbox).
  711. Fails closed: any error or a timed-out probe means "assume a password is needed"."""
  712. if not self._passwordless_sudo_probe_supported:
  713. return False
  714. try:
  715. proc = self._run_bash("sudo -n true", timeout=self._SUDO_PROBE_TIMEOUT_S)
  716. return self._wait_for_process(proc, timeout=self._SUDO_PROBE_TIMEOUT_S).get("returncode") == 0
  717. except Exception:
  718. return False