| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832 |
- """Base class for all Hermes execution environment backends.
- Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
- A session snapshot (env vars, functions, aliases) is captured once at init and
- re-sourced before each command. CWD persists via in-band stdout markers (remote)
- or a temp file (local). Cohesive pieces live in sibling modules (``base_output``,
- ``base_session_env``, ``base_wait``, ``path_utils``).
- """
- import json
- import logging
- import os
- import shlex
- import subprocess
- import threading
- import time
- import uuid
- from abc import ABC, abstractmethod
- from pathlib import Path
- from typing import Callable, Iterable
- from hermes_constants import get_hermes_home
- from tools.interrupt import consume_yield, is_interrupted, is_thread_interrupted
- from tools.environments.base_output import (
- ProcessHandle, _finalize_wait_result, _new_output_collector, _start_drain_thread,
- )
- from tools.environments.base_session_env import (
- _SHELL_ENV_NAME_RE, _SNAP_TMP_SUFFIX, _cwd_marker, _pipe_marker, _snapshot_bootstrap_script, _split_cwd_marker,
- _wrap_command_script,
- )
- from tools.environments.base_wait import _WaitTrace
- from utils import atomic_json_write, env_var_enabled, file_signature
- logger = logging.getLogger(__name__)
- # Opt-in debug tracing for the interrupt/activity/poll machinery
- # (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
- _DEBUG_INTERRUPT = env_var_enabled("HERMES_DEBUG_INTERRUPT")
- # Extra seconds the ``run_bounded_sync`` backstop waits past the inner ``_wait_for_process``
- # deadline: the inner loop returns partial output + 124; the outer bound only fires when that
- # loop never returns. Keep small so a healthy timeout still comes from the inner path.
- # The inner poll loop is what returns partial output + returncode 124; this outer bound only exists for when
- # that loop itself never returns (family A of #94285: a blocked wait that silently disables asyncio timers).
- _EXECUTE_WAIT_BOUND_GRACE_S = 2.0
- if _DEBUG_INTERRUPT:
- # quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
- # swallow every trace; force this logger back to INFO in the opt-in case.
- logger.setLevel(logging.INFO)
- # Thread-local activity callback: the agent sets it before a tool call so
- # long-running _wait_for_process loops can report liveness to the gateway.
- _activity_callback_local = threading.local()
- # Foreground commands in flight in THIS process, across every environment. Each runs in its
- # own session/process group, so a host that exits mid-command (TUI client gone, SIGTERM) would
- # orphan the whole tree; the process-exit funnel ``cleanup_all_environments`` kills them.
- _live_foreground: "dict[int, tuple[BaseEnvironment, ProcessHandle]]" = {}
- # Reentrant, and the hard-exit path only ever takes it with a timeout: a signal handler can run
- # on a thread that already holds it.
- _live_foreground_cond = threading.Condition(threading.RLock())
- _exit_fenced = False # one-way, set by the hard-exit kill: no foreground command spawns after it
- _spawns_in_flight = 0 # past the fence check, child maybe alive, not yet in _live_foreground
- _HARD_KILL_BUDGET_S = 0.5
- def _enter_foreground_spawn() -> bool:
- global _spawns_in_flight
- with _live_foreground_cond:
- if _exit_fenced:
- return False
- _spawns_in_flight += 1
- return True
- def _leave_foreground_spawn(env: "BaseEnvironment", spawned) -> bool:
- """Publish ``spawned`` (None: the spawn failed); True when the exit fence went up meanwhile."""
- global _spawns_in_flight
- with _live_foreground_cond:
- _spawns_in_flight -= 1
- if spawned is not None:
- _live_foreground[id(spawned)] = (env, spawned)
- _live_foreground_cond.notify_all()
- return _exit_fenced
- def _quiet_kill(kill: Callable, proc) -> None:
- try:
- kill(proc)
- except Exception:
- logger.debug("exit-time kill of a foreground command failed", exc_info=True)
- def kill_live_foreground_processes(*, now: bool = False) -> int:
- """Kill every in-flight foreground command's process tree; returns how many were signalled.
- ``now=True`` is for a caller about to ``os._exit``: the graceful kill TERMs, waits and only then
- KILLs, so a SIGTERM-ignoring command outlives a hard exit that lands inside that window. It also
- raises the exit fence and waits for spawns already past it to register, so no command started
- around the snapshot survives, and it never blocks past ``_HARD_KILL_BUDGET_S``: SDK cancels
- (Modal, Daytona, Vercel) run on daemon threads under that one deadline."""
- global _exit_fenced
- if not now:
- with _live_foreground_cond:
- live = list(_live_foreground.values())
- for env, proc in live:
- _quiet_kill(env._kill_process, proc)
- return len(live)
- deadline = time.monotonic() + _HARD_KILL_BUDGET_S
- _exit_fenced = True
- if _live_foreground_cond.acquire(timeout=_HARD_KILL_BUDGET_S):
- try:
- _live_foreground_cond.wait_for(lambda: _spawns_in_flight == 0, max(0.0, deadline - time.monotonic()))
- live = list(_live_foreground.values())
- finally:
- _live_foreground_cond.release()
- else: # the holder is stuck under our signal: a lock-free copy beats hanging the exit
- live = list(_live_foreground.values())
- remote = []
- for env, proc in live:
- if isinstance(proc, subprocess.Popen): # killpg/kill: never blocks
- _quiet_kill(env._force_kill_process, proc)
- else:
- remote.append(threading.Thread(target=_quiet_kill, args=(env._force_kill_process, proc), daemon=True))
- remote[-1].start()
- for t in remote:
- t.join(max(0.0, deadline - time.monotonic()))
- return len(live)
- class FileFetchError(RuntimeError):
- """A file could not be extracted from the backend filesystem."""
- # Pulling a large artifact over a slow exec channel can outlast the per-command terminal timeout.
- _FETCH_TIMEOUT_SECONDS = 300
- class EnvironmentConnectionError(RuntimeError):
- """Infrastructure/connection-class failure of a terminal backend (SSH host down, Docker
- daemon not running, remote sync on a dead link) — never a command that merely exited
- nonzero. Subclassing RuntimeError keeps every ``except RuntimeError`` catcher working.
- ``terminal_tool`` turns this into a structured ``status: "degraded"`` result; the failed
- backend is never cached, so a later call retries from scratch."""
- def __init__(self, reason: str, *, retry_hint: str = ""):
- super().__init__(reason)
- self.reason = reason
- self.retry_hint = retry_hint or (
- "This is an infrastructure failure, not a command failure. "
- "Verify the backend is reachable (network, service running, "
- "credentials), then retry the same command — recovery is "
- "automatic once the backend is back.")
- def set_activity_callback(cb: Callable[[str], None] | None) -> None:
- """Register a callback that _wait_for_process fires periodically."""
- _activity_callback_local.callback = cb
- def get_activity_callback() -> Callable[[str], None] | None:
- """Thread-local activity callback; capture it before handing work to another thread.
- Public accessor for callers outside this module that need to capture the calling thread's callback
- before handing work to another thread (the callback is thread-local, so a freshly spawned thread cannot
- read it back) — e.g. the manual cron-run heartbeat (#76502).
- """
- return getattr(_activity_callback_local, "callback", None)
- def touch_activity_if_due(state: dict, label: str) -> None:
- """Fire the activity callback at most once every ``state['interval']`` (default 10 s).
- *state* holds ``last_touch``/``start`` monotonic timestamps. Swallows all exceptions."""
- now = time.monotonic()
- if now - state["last_touch"] < state.get("interval", 10.0):
- return
- state["last_touch"] = now
- try:
- cb = get_activity_callback()
- if cb:
- cb(f"{label} ({int(now - state['start'])}s elapsed)")
- except Exception:
- logger.debug("activity callback failed during a long-running command", exc_info=True)
- def get_sandbox_dir() -> Path:
- """Host-side root for all sandbox storage (Docker workspaces, plugin backends'
- host-side state). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
- custom = os.getenv("TERMINAL_SANDBOX_DIR")
- p = Path(custom) if custom else get_hermes_home() / "sandboxes"
- p.mkdir(parents=True, exist_ok=True)
- return p
- def _load_json_store(path: Path) -> dict:
- """Treat a missing or damaged snapshot store as empty."""
- try:
- data = json.loads(path.read_text(encoding="utf-8-sig"))
- except (OSError, ValueError):
- return {}
- return data if isinstance(data, dict) else {}
- def _save_json_store(path: Path, data: dict) -> None:
- """Write *data* as pretty-printed JSON to *path* atomically (readers never see a torn file)."""
- path.parent.mkdir(parents=True, exist_ok=True)
- atomic_json_write(path, data, indent=2)
- _JSON_STORE_LOCK_HOLDER = threading.local()
- def _update_json_store(path: Path, update: Callable[[dict], None]) -> None:
- """Read-modify-write the JSON store at *path* under a cross-process lock.
- Concurrent sandbox cleanups (delegate children, gateway sessions, the idle reaper) each
- record a snapshot id; an unlocked load/mutate/save lets one cleanup drop the other's entry.
- An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
- update: a sandbox lifecycle step (modal's restore fallback included) must not abort on it."""
- from contextlib import ExitStack
- from hermes_cli.auth import _file_lock
- lock_path = path.with_name(f"{path.name}.lock")
- with ExitStack() as stack:
- try:
- stack.enter_context(_file_lock(
- lock_path, _JSON_STORE_LOCK_HOLDER, 10.0, f"Timed out waiting for the snapshot store lock ({path})",
- ))
- except (OSError, TimeoutError):
- logger.warning("Updating %s without its lock", path, exc_info=True)
- data = _load_json_store(path)
- before = dict(data)
- update(data)
- if data != before:
- _save_json_store(path, data)
- def _file_mtime_key(host_path: str) -> tuple[int, int, int, int] | None:
- """Return the ``file_signature`` change key, or ``None`` if unreadable.
- mtime + size alone miss a same-size credential refresh landing in the same mtime tick;
- inode and ctime catch the atomic replace / rewrite."""
- try:
- return file_signature(Path(host_path).stat())
- except OSError:
- return None
- def env_uses_host_filesystem(env: object) -> bool:
- """Whether the file tools may touch *env*'s files on the host (``BaseEnvironment.uses_host_filesystem``).
- Strict ``is True``: an environment without the declaration (``None``, a duck-typed plugin
- backend, a test double) is not the host."""
- return getattr(env, "uses_host_filesystem", False) is True
- # ---------------------------------------------------------------------------
- # BaseEnvironment
- # ---------------------------------------------------------------------------
- class BaseEnvironment(ABC):
- """Common interface and unified execution flow for all Hermes backends. Subclasses
- implement ``_run_bash()`` and ``cleanup()``; the base provides ``execute()`` with
- snapshot sourcing, CWD tracking, interrupt handling and timeout enforcement."""
- # How execute() hands stdin to _run_bash: "pipe" (process stdin, default),
- # "payload" (passed through as stdin_data; the backend transports it via its
- # SDK -- Modal/managed Modal stream it, Daytona/Vercel stage a file) or
- # "heredoc" (embedded in the command; no built-in backend, plugins only).
- _stdin_mode: str = "pipe" # "pipe" | "payload" | "heredoc"
- # True only when commands execute on the SAME host as the Hermes process
- # (LocalEnvironment); controller-host facts then describe the execution target.
- is_local: bool = False
- # True when the file tools may read, write, patch and search this backend's files straight on
- # the host instead of through execute() (only LocalEnvironment). A host backend that confines its
- # commands (a sandbox hiding paths, mounting them read-only or giving them their own $TMPDIR) sets
- # it False, so the file tools see exactly what its commands see. Read via env_uses_host_filesystem().
- uses_host_filesystem: bool = False
- # Snapshot creation timeout (override for slow cold-starts).
- _snapshot_timeout: int = 30
- # Re-dump exported env into the snapshot after each command (see __init__).
- _persist_env: bool = True
- # Opt in only when a timed-out probe can kill its command without tearing
- # down the whole backend. SDK adapters cancel by stopping the sandbox. Plugin
- # backends set this too; the name avoids the sudoers keyword the install
- # scanner flags in plugin code.
- _passwordless_sudo_probe_supported: bool = False
- # Local and Docker override this because they resolve allowlisted values
- # through the active profile scope; other backends keep plain snapshots.
- _profile_scoped_passthrough: bool = False
- # Per-session PIPESTATUS marker set in __init__; empty means the wrapper emits none.
- _pipe_marker: str = ""
- def get_temp_dir(self) -> str:
- """Return the backend temp directory used for session artifacts.
- Most sandboxed backends use ``/tmp`` inside the target environment.
- LocalEnvironment overrides this on hosts where ``/tmp`` may be missing
- and ``TMPDIR`` is the portable writable location.
- """
- return "/tmp" # no-tmp: ok — sandbox-side (remote container) temp dir, not the host
- def __init__(self, cwd: str, timeout: int, env: dict | None = None, persist_env: bool = True):
- self.cwd = cwd
- self.timeout = timeout
- self.env = env or {}
- # False: each command sources only the login-time snapshot, so exported vars do not
- # carry over (terminal.persistent_shell / TERMINAL_LOCAL_PERSISTENT). CWD still does.
- self._persist_env = persist_env
- self._session_id = uuid.uuid4().hex[:12]
- temp_dir = self.get_temp_dir().rstrip("/") or "/"
- self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
- self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
- self._cwd_marker = _cwd_marker(self._session_id)
- self._pipe_marker = _pipe_marker(self._session_id)
- self._snapshot_ready = False
- self._snapshot_passthrough_names: set[str] = set()
- # True when login bash is unusable (e.g. broken Git-for-Windows startup)
- # so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
- self._prefer_nonlogin = False
- # --- Abstract methods ---
- def _run_bash(
- self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None,
- ) -> ProcessHandle:
- """Spawn a bash process to run *cmd_string*; every backend overrides this."""
- raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
- @abstractmethod
- def cleanup(self):
- """Release backend resources (container, instance, connection)."""
- ...
- # --- File extraction (backend -> host) ---
- def fetch_file(self, remote_path: str, local_dest: Path, *, max_bytes: int) -> None:
- """Copy a regular file out of the backend filesystem to ``local_dest`` on the host.
- One transport for every backend: base64 over the exec channel, bounded INSIDE the sandbox
- (``head -c max+1`` so /dev/zero can't stream unbounded data into host memory — the same
- shape ``tools.image_source`` uses to read sandbox images). The payload is fenced between
- unique markers so login-shell noise in the merged stdout/stderr can't corrupt the decode.
- Raises :class:`FileFetchError` on a missing/unreadable/oversized file.
- """
- import base64
- import binascii
- marker = f"__HERMES_FETCH_{uuid.uuid4().hex[:12]}__"
- quoted = shlex.quote(remote_path)
- # ``[ -f ]`` follows symlinks, so a link to a denied host file is judged by the CALLER on
- # ``readlink -f`` output before any bytes move.
- result = self.execute(
- f"[ -f {quoted} ] && echo {marker} && head -c {max_bytes + 1} < {quoted} | base64 && echo {marker}",
- timeout=_FETCH_TIMEOUT_SECONDS, rewrite_compound_background=False)
- output = result.get("output") or ""
- first, last = output.find(marker), output.rfind(marker)
- if int(result.get("returncode") or 0) != 0 or first == -1 or last <= first:
- raise FileFetchError(f"could not read {remote_path!r} in the sandbox (missing, not a regular file, or unreadable)")
- try:
- data = base64.b64decode("".join(output[first + len(marker):last].split()), validate=True)
- except (ValueError, binascii.Error) as exc:
- raise FileFetchError(f"transfer of {remote_path!r} was corrupted in transit: {exc}") from exc
- if len(data) > max_bytes:
- raise FileFetchError(f"{remote_path!r} exceeds the {max_bytes // (1024 * 1024)} MB delivery limit")
- Path(local_dest).write_bytes(data)
- def fetch_realpath(self, remote_path: str) -> str | None:
- """``readlink -f`` inside the backend, or None when it cannot be resolved."""
- result = self.execute(f"readlink -f {shlex.quote(remote_path)} 2>/dev/null", rewrite_compound_background=False)
- if int(result.get("returncode") or 0) != 0:
- return None
- return next((ln.strip() for ln in reversed((result.get("output") or "").splitlines()) if ln.strip().startswith("/")), None)
- # --- Session snapshot (init_session) ---
- def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
- """Return backend-specific names that must not persist in snapshots."""
- return ()
- def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
- """Profile-scoped names that must not persist in the snapshot. Monotonic for the
- environment lifetime: an allowlist can be cleared after a value was captured, and
- retaining the exclusion keeps that old value from leaking to a later profile."""
- if not self._profile_scoped_passthrough:
- return ()
- try:
- from agent.secret_scope import is_multiplex_active
- if is_multiplex_active():
- from tools.env_passthrough import get_all_passthrough
- names = (*get_all_passthrough(), *self._additional_profile_scoped_passthrough_names())
- self._snapshot_passthrough_names.update(
- name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name))
- except Exception:
- logger.debug("Could not refresh profile-scoped snapshot exclusions", exc_info=True)
- return tuple(sorted(self._snapshot_passthrough_names))
- def _snapshot_script_kwargs(self, cwd: str) -> dict:
- """Quoting inputs shared by the bootstrap and per-command wrapper scripts.
- ``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) let the Windows
- subclass rewrite ``C:\\...`` to ``/c/...`` so ``cd`` resolves and MSYS doesn't choke."""
- return dict(
- quoted_cwd=self._quote_cwd_for_cd(cwd),
- quoted_snap=self._quote_shell_path(self._snapshot_path),
- snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX),
- cwd_marker=self._cwd_marker)
- def init_session(self):
- """Capture the login shell environment into the snapshot file (once, after construction).
- On success ``_snapshot_ready`` is set so commands source the snapshot instead of running
- under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login
- probe shows login bash itself is dead, in which case prefer ``bash -c``."""
- bootstrap = _snapshot_bootstrap_script(
- excluded_names=self._snapshot_excluded_passthrough_names(), **self._snapshot_script_kwargs(self.cwd))
- try:
- proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
- result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
- if int(result.get("returncode") or 0) != 0:
- raise RuntimeError(f"snapshot bootstrap failed with exit code {result.get('returncode')}")
- self._snapshot_ready = True
- self._update_cwd(result)
- logger.info("Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd)
- except Exception as exc:
- self._snapshot_ready = False
- self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
- if self._prefer_nonlogin:
- logger.warning(
- "init_session failed (session=%s): %s — "
- "login bash unusable; falling back to non-login bash -c",
- self._session_id, exc)
- else:
- logger.warning(
- "init_session failed (session=%s): %s — falling back to bash -l per command",
- self._session_id, detail)
- def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
- """Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
- probe_timeout = min(15, self._snapshot_timeout)
- try:
- probe = self._run_bash("true", login=False, timeout=probe_timeout)
- probe_result = self._wait_for_process(probe, timeout=probe_timeout)
- prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
- if not prefer_nonlogin:
- detail = (probe_result.get("stdout") or detail).strip() or detail
- return prefer_nonlogin, detail
- except Exception as probe_exc:
- return False, f"{detail}; non-login probe: {probe_exc}"
- # --- Command wrapping ---
- @staticmethod
- def _quote_cwd_for_cd(cwd: str) -> str:
- """Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
- goes through ``$HOME`` so suffixes with spaces stay one word)."""
- if cwd == "~":
- return cwd
- if cwd == "~/":
- return "$HOME"
- if cwd.startswith("~/"):
- return f"$HOME/{shlex.quote(cwd[2:])}"
- return shlex.quote(cwd)
- def _quote_shell_path(self, path: str) -> str:
- """Quote *path* for a bash script. LocalEnvironment overrides this to
- rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
- return shlex.quote(path)
- def _wrap_command(self, command: str, cwd: str) -> str:
- """Full bash script: source snapshot, cd, run, re-dump env, emit CWD markers."""
- return _wrap_command_script(
- command,
- passthrough_names=self._snapshot_excluded_passthrough_names(),
- snapshot_ready=self._snapshot_ready,
- persist_env=self._persist_env,
- pipe_marker=self._pipe_marker,
- **self._snapshot_script_kwargs(cwd))
- @staticmethod
- def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
- """Redirect stdin_data to the complete command with a shell heredoc (SDK backends).
- A heredoc body always ends in a newline stdin_data may lack, and write_file verifies a
- byte-exact hash, so a process substitution re-emits the body minus that last character.
- Redirections apply left to right, so the substitution inherits the heredoc as its stdin;
- the heredoc stays outside ``<( )`` because bash 3.2 mis-parses bodies inside it. ``|| :``
- keeps an inherited ``set -e`` from killing the reader on read's EOF status."""
- delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
- return (f"{{\n{command}\n}} << '{delimiter}' < <(IFS= read -r -d '' s || :; printf '%s' \"${{s%?}}\")\n"
- f"{stdin_data}\n{delimiter}")
- def _staged_stdin_path(self) -> str:
- """Unique sandbox path for staging a payload-mode stdin file."""
- temp_dir = self.get_temp_dir().rstrip("/") or "/"
- return f"{temp_dir}/.hermes-stdin-{uuid.uuid4().hex}"
- @staticmethod
- def _redirect_stdin_from_file(command: str, path: str) -> str:
- """Prefix ``command`` so the shell reads stdin from the staged ``path`` and
- unlinks it before ``command`` runs (the shell then owns the payload)."""
- quoted = shlex.quote(path)
- return f"exec 0< {quoted} || exit $?\nrm -f -- {quoted} || exit $?\n{command}"
- # --- Process lifecycle ---
- def _wait_for_process(
- self, proc: ProcessHandle, timeout: int = 120, *,
- bounded_capture: bool = False, watch_interrupt_tid: int | None = None,
- output=None, yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
- """Poll-based wait with interrupt checking and stdout draining (shared, not overridden).
- ``yield_handler(proc, output_so_far)``: when the tool thread is asked to yield
- (``tools.interrupt.request_yield`` — a user message arrived mid-command), the drain
- thread is stopped, the still-running process is handed to the handler and its dict
- is returned as the result; the process is NOT killed.
- ``bounded_capture=True`` (foreground terminal-tool path only) retains at most
- ``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the
- process; the default keeps full fidelity for internal consumers. Fires the activity
- callback every 10s so the gateway's inactivity timeout doesn't kill long commands.
- ``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()``
- may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the
- original tid, so both bits are honored. ``KeyboardInterrupt``/``SystemExit`` mid-poll
- kills the process first — the local backend spawns into its own process group, so an
- unkilled child would be orphaned.
- The default (False) preserves full-fidelity capture for internal consumers — file-operation ``cat``
- reads feeding the patch engine, code-execution RPC reads, log reads — where truncation would corrupt
- data. See #64435.
- """
- if output is None:
- output = _new_output_collector(proc, bounded_capture)
- drain_stop = threading.Event() if yield_handler is not None else None
- drain_thread = _start_drain_thread(proc, output, drain_stop)
- _now = time.monotonic()
- deadline = _now + timeout
- _activity_state = {"last_touch": _now, "start": _now}
- trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
- trace.enter()
- def _kill_and_join():
- self._kill_process(proc)
- drain_thread.join(timeout=2)
- try:
- # Adaptive poll: start at 5ms so fast commands return in ~6ms, back
- # off exponentially toward 200ms so long builds don't pay poll CPU.
- _poll_sleep = 0.005
- while proc.poll() is None:
- trace.iterations += 1
- if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
- trace.interrupted()
- _kill_and_join()
- return {**self._finalize_wait_result(output, output.render(suffix="\n[Command interrupted]"), 130),
- "hermes_interrupted": True}
- if yield_handler is not None and consume_yield(watch_interrupt_tid):
- drain_stop.set()
- drain_thread.join(timeout=1)
- try:
- handed = yield_handler(proc, output.render())
- except Exception:
- logger.warning("yield-to-background handoff failed; continuing to wait", exc_info=True)
- handed = None
- if handed is not None:
- output.close_spill()
- return handed
- drain_stop.clear()
- drain_thread = _start_drain_thread(proc, output, drain_stop)
- if time.monotonic() > deadline:
- trace.timed_out()
- _kill_and_join()
- rendered = output.render(suffix=f"\n[Command timed out after {timeout}s]")
- if output.total_chars == 0:
- rendered = rendered.lstrip()
- # The flag tells Hermes' own deadline apart from a command's own ``exit 124``.
- return {**self._finalize_wait_result(output, rendered, 124), "hermes_timed_out": True}
- touch_activity_if_due(_activity_state, "terminal command running")
- trace.heartbeat()
- time.sleep(_poll_sleep)
- if _poll_sleep < 0.2:
- _poll_sleep = min(_poll_sleep * 1.5, 0.2)
- except (KeyboardInterrupt, SystemExit):
- trace.exception_exit()
- try:
- _kill_and_join()
- except Exception:
- pass # cleanup is best-effort
- raise
- # The drain thread exits promptly after bash does (~300ms idle check);
- # a long join here would itself indicate a bug in the drain loop.
- drain_thread.join(timeout=2)
- try:
- proc.stdout.close()
- except Exception:
- pass
- trace.natural_exit(proc.returncode)
- # Join the stdin writer before reading its error list: a child that exits without
- # reading stdin can otherwise race ahead of a recorded encode failure. The timeout
- # is a pure safety net (write raises BrokenPipeError once the pipe closes).
- stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
- if stdin_thread is not None:
- stdin_thread.join(timeout=5)
- rendered = output.render()
- result = self._finalize_wait_result(output, rendered, proc.returncode)
- if stdin_errors := getattr(proc, "_hermes_stdin_errors", None):
- result["stdin_error"] = err = str(stdin_errors[0])
- result["output"] = rendered + f"\n[stdin write failed: {err}]"
- return result
- _finalize_wait_result = staticmethod(_finalize_wait_result)
- def _kill_process(self, proc: ProcessHandle):
- """Terminate a process. Subclasses may override for process-group kill."""
- try:
- proc.kill()
- except (ProcessLookupError, PermissionError, OSError):
- pass
- def _force_kill_process(self, proc: ProcessHandle):
- """Kill without waiting, for a host that hard-exits next. Subclasses kill the whole tree."""
- self._kill_process(proc)
- # --- CWD extraction ---
- def _update_cwd(self, result: dict):
- """Extract CWD from command output. Override for local file-based read."""
- self._extract_cwd_from_output(result)
- def _extract_cwd_from_output(self, result: dict):
- """Parse the ``__HERMES_CWD_{session}__`` marker from ``result["output"]``, update
- ``self.cwd`` and strip the marker line. ``result["cwd_observed"]``/``["cwd"]`` are set
- only when THIS command emitted a marker: a killed/timed-out command emits none and
- ``self.cwd`` keeps the previous value. The environment is shared across sessions, so
- concurrent callers must read ``result["cwd"]`` rather than ``self.cwd``."""
- pipe_split = _split_cwd_marker(result.get("output", ""), self._pipe_marker) if self._pipe_marker else None
- if pipe_split is not None:
- statuses, cleaned = pipe_split
- result["output"] = cleaned
- try:
- result["pipeline_status"] = [int(s) for s in (statuses or "").split()]
- except ValueError:
- pass
- split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
- if split is None:
- return
- cwd_path, cleaned = split
- if cwd_path:
- self.cwd = cwd_path
- result["cwd_observed"] = True
- result["cwd"] = cwd_path
- result["output"] = cleaned
- # --- Hooks ---
- def _before_execute(self) -> None:
- """Hook before each command. Remote backends (SSH, Modal, Daytona)
- trigger their FileSyncManager here; bind-mount backends and Local don't."""
- def _mark_recreated(self) -> None:
- """Flag that the live container/sandbox was replaced while serving the
- current command. ``execute`` folds the flag into the result as
- ``environment_recreated`` so the tool layer can warn the model that
- background processes died and non-persisted files may be gone —
- without this the recovery is silent and the model keeps assuming the
- old workspace state (lobehub/lobehub#19329 class)."""
- self._recreated_notice_pending = True
- # --- Unified execute() ---
- def execute(
- self,
- command: str,
- cwd: str = "",
- *,
- timeout: int | None = None,
- stdin_data: str | None = None,
- rewrite_compound_background: bool = True,
- bounded_capture: bool = False,
- yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
- """Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True``
- caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal
- tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the
- patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is
- bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past
- ``timeout`` and silently disable every asyncio timer in the process.
- ``bounded_capture=True`` caps stdout/stderr retention at ``tool_output.max_bytes`` WHILE the stream
- is drained (head/tail window) instead of holding the full output in memory (#64435).
- See #94285.
- """
- self._before_execute()
- exec_command, sudo_stdin = self._prepare_command(command)
- # Guard against the `A && B &` subshell-wait trap by default; callers
- # that already produce shell-safe wrappers (spawn_via_env) pass False.
- if rewrite_compound_background:
- from tools.terminal_tool_sudo import _rewrite_compound_background
- exec_command = _rewrite_compound_background(exec_command)
- effective_timeout = timeout or self.timeout
- effective_cwd = cwd or self.cwd
- # Merge sudo stdin with caller stdin.
- effective_stdin = sudo_stdin + (stdin_data or "") if sudo_stdin is not None else stdin_data
- if effective_stdin and self._stdin_mode == "heredoc":
- exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
- effective_stdin = None
- wrapped = self._wrap_command(exec_command, effective_cwd)
- # Login shell if the snapshot failed (so the user's profile still
- # loads), unless login itself is broken — then non-login is the only path.
- login = not self._snapshot_ready and not self._prefer_nonlogin
- parent_tid = threading.current_thread().ident
- # The activity callback is thread-local and the wait runs on the
- # deadline worker, so copy it across or long commands look idle.
- parent_activity_cb = get_activity_callback()
- proc_holder: list = []
- output_holder: list = []
- def _spawn_and_wait() -> dict:
- if parent_activity_cb is not None:
- set_activity_callback(parent_activity_cb)
- if not _enter_foreground_spawn():
- return {"output": "[host is exiting: command not started]", "returncode": 130}
- spawned = None
- try:
- spawned = self._run_bash(wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin)
- finally:
- fenced = _leave_foreground_spawn(self, spawned)
- proc_holder.append(spawned)
- if fenced: # the hard-exit kill may have stopped waiting for us before we registered
- self._force_kill_process(spawned)
- output = _new_output_collector(spawned, bounded_capture)
- output_holder.append(output)
- try:
- return self._wait_for_process(
- spawned, timeout=effective_timeout, bounded_capture=bounded_capture,
- watch_interrupt_tid=parent_tid, output=output,
- **({"yield_handler": yield_handler} if yield_handler is not None else {}))
- finally:
- with _live_foreground_cond:
- _live_foreground.pop(id(spawned), None)
- def _on_timeout() -> None:
- if proc_holder:
- self._kill_spawned_tree(proc_holder[0])
- # Hard wall-clock backstop: ``_wait_for_process`` polls to ``effective_timeout`` on the
- # tool thread; if that is the event-loop thread, or the wait never returns (Windows
- # pipe/poll hang), every asyncio timer is silently disabled. ``run_bounded_sync`` drives
- # expiry from a daemon worker + ``Event.wait`` so a blocked loop cannot disable it; the
- # grace lets the inner loop return the partial-output 124 path.
- # See #94285.
- from agent.deadline import run_bounded_sync
- try:
- bound_s = float(effective_timeout)
- except (TypeError, ValueError):
- bound_s = 120.0 # a non-numeric timeout must not disable the backstop
- bound_s += _EXECUTE_WAIT_BOUND_GRACE_S
- try:
- bounded = run_bounded_sync(
- _spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout)
- except (KeyboardInterrupt, SystemExit):
- _on_timeout()
- raise
- if bounded.timed_out:
- suffix = f"\n[Command timed out after {effective_timeout}s]"
- if output_holder:
- collector = output_holder[0]
- result = self._finalize_wait_result(collector, collector.render(suffix=suffix).lstrip("\n"), 124)
- else:
- result = {"output": suffix.lstrip(), "returncode": 124}
- result["hermes_timed_out"] = True
- else:
- result = bounded.value
- self._update_cwd(result)
- if getattr(self, "_recreated_notice_pending", False):
- self._recreated_notice_pending = False
- result["environment_recreated"] = True
- return result
- def _kill_spawned_tree(self, spawned) -> None:
- """Best-effort kill of a wedged spawned process and its tree (backstop path)."""
- try:
- self._kill_process(spawned)
- except Exception:
- logger.debug("terminal wait-bound kill_process failed", exc_info=True)
- pid = getattr(spawned, "pid", None)
- if not pid:
- return
- try:
- from agent.deadline import kill_process_tree
- kill_process_tree(int(pid))
- except Exception:
- logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
- # --- Shared helpers ---
- def __del__(self):
- try:
- self.cleanup()
- except Exception:
- pass
- def _prepare_command(self, command: str) -> tuple[str, str | None]:
- """Rewrite sudo for a piped password, or leave it alone when this backend has NOPASSWD.
- Also applies the macOS ``open`` frontmost raise-ladder (a pure string
- rewrite, no-op on non-Darwin) so files opened via tool calls are
- brought to the front instead of landing behind the Hermes window.
- """
- from tools.terminal_tool_macos_open import _transform_macos_open_command
- from tools.terminal_tool_sudo import _transform_sudo_command
- return _transform_sudo_command(
- _transform_macos_open_command(command),
- sudo_nopasswd_check=self._sudo_nopasswd_works,
- )
- _SUDO_PROBE_TIMEOUT_S = 3
- def _sudo_nopasswd_works(self) -> bool:
- """``sudo -n true`` inside THIS backend (host sudo state must not leak into a sandbox).
- Fails closed: any error or a timed-out probe means "assume a password is needed"."""
- if not self._passwordless_sudo_probe_supported:
- return False
- try:
- proc = self._run_bash("sudo -n true", timeout=self._SUDO_PROBE_TIMEOUT_S)
- return self._wait_for_process(proc, timeout=self._SUDO_PROBE_TIMEOUT_S).get("returncode") == 0
- except Exception:
- return False
|