terminal_tool.py 84 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758
  1. #!/usr/bin/env python3
  2. """Terminal tool: run shell commands in the configured backend.
  3. Backends (``TERMINAL_ENV``): local (default), docker, modal (direct or
  4. managed gateway), daytona, vercel_sandbox, ssh, plus plugin-registered
  5. backends (the ``singularity`` catalog plugin, ...). Handles background
  6. processes, sandbox lifecycle (per-task cache, idle reaper, atexit teardown)
  7. and sudo password plumbing.
  8. Cloud-sandbox persistent filesystems preserve working state across sandbox
  9. recreation but do NOT guarantee the same live sandbox or long-running
  10. processes survive cleanup, idle reaping, or Hermes exit.
  11. Companion modules (re-exported here, so ``tools.terminal_tool.<name>`` stays the
  12. import/patch target): ``terminal_tool_config`` (TERMINAL_* reads, ``_quiet``),
  13. ``terminal_tool_backends`` (env builders + requirement checkers),
  14. ``terminal_tool_lifecycle`` (reaper/teardown/ensure_task_env),
  15. ``terminal_tool_sudo`` (sudo password + shell rewrites), ``terminal_tool_guards``
  16. (pre-exec blocks), ``terminal_tool_background`` (background spawn),
  17. ``terminal_tool_result`` (foreground result post-processing).
  18. """
  19. import json
  20. import logging
  21. import os
  22. import sys
  23. import time
  24. import threading
  25. import atexit
  26. from dataclasses import dataclass
  27. from typing import Optional, Dict, Any, List
  28. logger = logging.getLogger(__name__)
  29. def _redact_terminal_error_text(value: Any) -> str:
  30. """Force-redact text before serializing a terminal error envelope."""
  31. from agent.redact import redact_sensitive_text
  32. return redact_sensitive_text("" if value is None else str(value), force=True)
  33. from tools.registry import tool_error
  34. from tools.terminal_tool_lifecycle import (
  35. _call_in_flight, _cleanup_inactive_envs, _create_configured_env,
  36. _evict_environment_for_task, cleanup_all_environments, ensure_task_env,
  37. )
  38. from tools.terminal_tool_config import (
  39. _is_container_backend, _is_host_cwd, _is_mounted_host_cwd, _is_unusable_container_cwd,
  40. _is_windows_drive_path, _parse_env_var, _plugin_env_flag, _quiet, _safe_getcwd, _tenv, _tenv_bool,
  41. _in_sandbox_home, coerce_ssh_remote_cwd, home_profile_name, profile_container_key,
  42. translate_mounted_host_path,
  43. )
  44. from tools.terminal_tool_backends import (
  45. _REQUIREMENT_CHECKERS, _VERCEL_SANDBOX_DEFAULT_CWD, _check_plugin_requirements,
  46. _record_unavailable_reason, terminal_backend_unavailable_reason,
  47. )
  48. # display_hermes_home imported lazily at call site (stale-module safety during hermes update)
  49. from tools.tool_backend_helpers import coerce_modal_mode, managed_nous_tools_enabled
  50. def _safe_parse_import_env(name: str, default: Any, converter, type_label: str):
  51. """Parse a module-level numeric env var; a malformed value must never make
  52. the module unloadable at import time (CLI, ACP, tests, tool discovery)."""
  53. raw = os.getenv(name)
  54. if raw is None or raw == "":
  55. return default
  56. try:
  57. return converter(raw)
  58. except (TypeError, ValueError):
  59. logger.warning(
  60. "Invalid value for %s: %r (expected %s). Falling back to %r.",
  61. name, raw, type_label, default,
  62. )
  63. return default
  64. # Hard cap on foreground timeout; override via TERMINAL_MAX_FOREGROUND_TIMEOUT env var.
  65. FOREGROUND_MAX_TIMEOUT = _safe_parse_import_env("TERMINAL_MAX_FOREGROUND_TIMEOUT", 600, int, "integer")
  66. # Approval / sudo-prompt UI callbacks (CLI registers prompt_toolkit-aware
  67. # ones). Thread-local so overlapping ACP sessions, each on its own executor
  68. # thread, can't stomp on each other (GHSA-qg5c-hvr5-hjgr). Gateway mode
  69. # resolves approvals via the per-session queue in tools.approval instead.
  70. _callback_tls = threading.local()
  71. def _get_sudo_password_callback():
  72. return getattr(_callback_tls, "sudo_password", None)
  73. def _get_approval_callback():
  74. return getattr(_callback_tls, "approval", None)
  75. def set_sudo_password_callback(cb):
  76. """Register the CLI's sudo password prompt callback (per-thread slot)."""
  77. _callback_tls.sudo_password = cb
  78. def set_approval_callback(cb):
  79. """Register the dangerous-command approval prompt callback (per-thread slot)."""
  80. _callback_tls.approval = cb
  81. def _current_session_key() -> str:
  82. """Active gateway/WebUI session key, or "" outside sessions (ContextVar with
  83. the ``get_session_env`` os.environ fallback for CLI/cron/tests)."""
  84. from gateway.session_context import get_session_env
  85. return get_session_env("HERMES_SESSION_KEY", "")
  86. def _current_session_profile() -> str:
  87. """Active session's Hermes profile name, or "" (same lookup discipline as
  88. :func:`_current_session_key`)."""
  89. from gateway.session_context import get_session_env
  90. return get_session_env("HERMES_SESSION_PROFILE", "")
  91. def _container_profile() -> str:
  92. """Profile whose persistent container a session-bound command uses: the bound home's profile when it is not
  93. the session's own (a profile-routed subagent runs in its target's home while the session, and the delivery
  94. identity HERMES_SESSION_PROFILE carries, stay the parent's; that target may be the launch profile, reached
  95. from a routed profile's chat), else the session's. A session with no profile name (a gateway launched as a
  96. named profile stamps none on its own chats) is the launch profile's, so it keys that profile by name, like
  97. the profile's Desktop sessions and its session-less work (:func:`_sessionless_container_key`)."""
  98. from hermes_constants import get_hermes_home_override, profile_name_for_home
  99. from tools.environments.local import _is_routed_home
  100. override = get_hermes_home_override()
  101. session = _current_session_profile()
  102. bound = profile_name_for_home(override) if override else None
  103. if bound and (bound != session if session else _is_routed_home(override)):
  104. return bound
  105. return session or home_profile_name()
  106. from tools.approval import (
  107. check_all_command_guards as _check_all_guards_impl,
  108. )
  109. def _docker_volume_uses_host_path(volume_spec: str) -> bool:
  110. """Return True when a docker volume spec bind-mounts a host path."""
  111. if not isinstance(volume_spec, str):
  112. return False
  113. vol = volume_spec.strip()
  114. return bool(vol) and (
  115. vol.startswith(("/", "~", "./", "../")) or
  116. (len(vol) >= 3 and vol[1] == ":" and vol[2] in ("/", "\\"))
  117. )
  118. def _docker_has_host_access(config: dict[str, Any]) -> bool:
  119. """Return True when a Docker sandbox exposes host paths through bind mounts."""
  120. if config.get("env_type") != "docker":
  121. return False
  122. if config.get("host_cwd") and config.get("docker_mount_cwd_to_workspace"):
  123. return True
  124. return any(_docker_volume_uses_host_path(vol) for vol in config.get("docker_volumes", []))
  125. def _check_all_guards(command: str, env_type: str,
  126. has_host_access: bool = False) -> dict:
  127. """Delegate to the consolidated command guard with the CLI callback."""
  128. return _check_all_guards_impl(command, env_type,
  129. approval_callback=_get_approval_callback(),
  130. has_host_access=has_host_access)
  131. from tools.environments.base import EnvironmentConnectionError
  132. # Tool description for LLM
  133. TERMINAL_TOOL_DESCRIPTION = """Run shell commands on the OS/shell/backend named in your environment section. The working directory, files and exported variables persist between calls: activate a virtualenv or export once, not before every command.
  134. Use read_file, search_files, patch and write_file instead of cat/head/tail, grep/find/ls, sed/awk and echo/heredoc. Output is auto-truncated with the full text saved to a file; never pipe through head/tail to shorten it.
  135. Foreground (default) returns as soon as the command finishes. To wait a fixed time, run the wait in the foreground; never background a sleep or polling loop. Background is only for commands that must keep running after this call returns; health-check a server in a separate call and manage jobs with process_manage. When a command changes directory, trust the result's "cwd" instead of prefixing later commands with cd.
  136. """
  137. # Environment lifecycle state.
  138. _active_environments: dict[str, Any] = {}
  139. _last_activity: dict[str, float] = {}
  140. _env_lock = threading.Lock()
  141. _creation_locks: dict[str, threading.Lock] = {} # Per-task locks for sandbox creation
  142. _creation_locks_lock = threading.Lock() # Protects _creation_locks dict itself
  143. _calls_in_flight: dict[str, int] = {} # Foreground calls running per env key; the reaper skips these
  144. _cleanup_thread = None
  145. _cleanup_running = False
  146. # Once-per-process guard for the docker orphan reaper.
  147. _docker_orphan_reaper_ran = False
  148. _docker_orphan_reaper_lock = threading.Lock()
  149. def _maybe_reap_docker_orphans(container_config: dict[str, Any]) -> None:
  150. """Run the docker orphan reaper once per process, if enabled.
  151. Sweeps Exited containers labeled ``hermes-agent=1`` for the current
  152. profile — leftovers of Hermes processes that died without firing
  153. ``atexit`` (SIGKILL, OOM, closed terminal). Conservative: only containers
  154. older than ``2 × lifetime_seconds``, profile-scoped. Gates:
  155. ``terminal.docker_orphan_reaper: false`` (operator opt-out, e.g. several
  156. Hermes processes sharing a profile) and the once-per-interpreter flag so
  157. parallel subagent / RL-rollout calls don't re-sweep.
  158. """
  159. global _docker_orphan_reaper_ran
  160. if not container_config.get("docker_orphan_reaper", True):
  161. return
  162. if _docker_orphan_reaper_ran: # double-checked locking
  163. return
  164. with _docker_orphan_reaper_lock:
  165. if _docker_orphan_reaper_ran:
  166. return
  167. _docker_orphan_reaper_ran = True
  168. # 2 × lifetime gives sibling processes a grace window; floor at 60s so
  169. # TERMINAL_LIFETIME_SECONDS=0 can't instant-reap a sibling's own setup.
  170. # container_config only carries container_* keys, so read the env var.
  171. try:
  172. lifetime = int(_tenv("TERMINAL_LIFETIME_SECONDS", "300"))
  173. except (TypeError, ValueError):
  174. lifetime = 300
  175. max_age = max(60, lifetime) * 2
  176. try:
  177. from tools.environments.docker import reap_orphan_containers, _container_identity
  178. except ImportError:
  179. return
  180. # Never fail the env-creation path because of a janitor problem.
  181. with _quiet("Docker orphan reaper raised"):
  182. profile = _container_identity(container_config.get("docker_shared_container_key", ""))
  183. removed = reap_orphan_containers(max_age_seconds=max_age, profile_filter=profile)
  184. if removed:
  185. logger.info(
  186. "Docker orphan reaper removed %d stale container(s) for profile %s",
  187. removed, profile,
  188. )
  189. # Per-task environment overrides (never exposed to the model). RL/benchmark
  190. # envs and ACP register a custom image / cwd for a task_id BEFORE the agent
  191. # loop; sandbox creation consults this first, then the TERMINAL_* env vars.
  192. _task_env_overrides: dict[str, dict[str, Any]] = {}
  193. # Per-session cwd records: the durable source of truth for "which directory
  194. # is THIS session in". Keyed by the raw session/task key, NOT the collapsed
  195. # container id — the env is shared across sessions, so cwd state stored on
  196. # it is a global mutable timeshared between sessions (the wrong-worktree bug
  197. # class). Written after every completed command and on cwd-override
  198. # registration; readers resolve against it before any env-side cwd.
  199. _session_cwd: dict[str, str] = {}
  200. _session_cwd_lock = threading.Lock()
  201. # Subagent → parent container aliasing. delegate_task children have their own
  202. # task_id but must share the PARENT's container; under per-session isolation
  203. # the collapse-to-"default" shortcut no longer provides that, so the spawn
  204. # site registers an explicit alias.
  205. _container_aliases: dict[str, str] = {}
  206. _container_alias_lock = threading.Lock()
  207. def record_session_cwd(session_key: Optional[str], cwd: Optional[str]) -> None:
  208. """Record *cwd* as *session_key*'s working directory (after a completed
  209. command, or on workspace-override registration). None/empty keys collapse
  210. to ``"default"``; non-string / empty cwds are ignored. Keys are routed-profile
  211. qualified (:func:`_qualify_task_key`); the read/clear side qualifies identically."""
  212. if not isinstance(cwd, str) or not cwd.strip():
  213. return
  214. key = _qualify_task_key(str(session_key or "default"))
  215. with _session_cwd_lock:
  216. if _session_cwd.get(key) != cwd:
  217. _session_cwd[key] = cwd
  218. def get_session_cwd(session_key: Optional[str]) -> Optional[str]:
  219. """Recorded cwd for *session_key*, or None. No fallback chain on purpose:
  220. callers decide what an absent record means. None/empty keys read ``"default"``."""
  221. with _session_cwd_lock:
  222. return _session_cwd.get(_qualify_task_key(str(session_key or "default")))
  223. def clear_session_cwd(session_key: str) -> None:
  224. """Drop a session's cwd record (session teardown)."""
  225. with _session_cwd_lock:
  226. _session_cwd.pop(_qualify_task_key(session_key), None)
  227. def _sanitize_cwd_for_live_env(env: Any, new_cwd: str) -> Optional[str]:
  228. """Cwd to write into a LIVE cached env, or None to leave it untouched.
  229. On container backends a raw host path (a desktop/TUI session registering its
  230. workspace, e.g. ``C:\\Users\\me`` or ``/Users/me/workspace``) cannot be the
  231. in-sandbox workdir: every file-tools ``_exec`` wrapper does
  232. ``builtin cd -- <env.cwd> || exit 126``, so a host cwd poisons all later
  233. file operations with an unrelated ``cd:`` error. Prefix-shaped host paths
  234. are already rejected on the creation paths. This write classifies the
  235. directory mounted at ``/workspace`` as unusable before that prefix
  236. heuristic, then remaps the match (or a child of it) to its container mount
  237. instead of storing the host path. SSH maps the Hermes subprocess home onto
  238. the peer's home, as environment creation already does. Other backends apply
  239. the override verbatim (ACP project-root switching must keep working).
  240. """
  241. env_type = getattr(env, "env_type", None)
  242. new_cwd = coerce_ssh_remote_cwd(new_cwd, env_type)
  243. if not env_type or not _is_container_backend(env_type):
  244. return new_cwd
  245. host_mount = getattr(env, "host_cwd", None)
  246. mounted = host_mount if isinstance(host_mount, str) and host_mount else None
  247. # Mount equality before the prefix heuristic. /mnt and /srv are absolute,
  248. # so the heuristic alone would write the host path through as env.cwd and
  249. # every later file-tools exec would `cd` to it (exit 126).
  250. if not _is_unusable_container_cwd(new_cwd, mounted_host=mounted):
  251. return new_cwd
  252. if mounted:
  253. # The bind may sit at a fallback mount when /workspace is claimed.
  254. container_mount = getattr(env, "host_cwd_mount", None) or "/workspace"
  255. if _is_mounted_host_cwd(new_cwd, mounted):
  256. return container_mount
  257. translated = translate_mounted_host_path(new_cwd, mounted, container_mount)
  258. if translated:
  259. return translated
  260. return None
  261. def register_task_env_overrides(task_id: str, overrides: dict[str, Any]):
  262. """Register per-task sandbox overrides (``docker_image``/``modal_image``/
  263. ``daytona_image`` or a plugin backend's ``<backend>_image``, ``env_type``,
  264. ``cwd``) before the agent loop runs.
  265. A ``cwd`` override takes effect immediately: it becomes the session's
  266. recorded cwd (until a ``cd`` changes it) and any live env's cwd is updated
  267. too, so env-side seeding stays consistent (ACP switching project root
  268. mid-session via ``session/load``). The session record keeps the RAW path
  269. (host workspaces are tracked there on purpose); only the live-env write is
  270. sanitized, since a host cwd can never be a container workdir.
  271. ``host_workspace`` is host-side only: a Desktop session's own folder when its ``cwd`` is left
  272. empty because an unmounted Docker sandbox cannot see it (project tools read it; the terminal
  273. never does).
  274. Keyed like ``_session_cwd`` — routed-profile qualified (:func:`_qualify_task_key`) — so two
  275. profiles registering the same task id never read each other's image/cwd (#123989).
  276. """
  277. _task_env_overrides[_qualify_task_key(task_id)] = overrides
  278. new_cwd = overrides.get("cwd")
  279. if isinstance(new_cwd, str) and new_cwd.strip():
  280. record_session_cwd(task_id, new_cwd)
  281. # Live env may be cached under the raw task_id (per-session surfaces)
  282. # or the collapsed container id (isolation-keyed rollouts); try both so
  283. # a CWD-only override (which collapses to "default") still finds it.
  284. container_id = _resolve_container_task_id(task_id)
  285. with _env_lock:
  286. env = _active_environments.get(task_id) or _active_environments.get(container_id)
  287. if env is not None and getattr(env, "cwd", None) is not None:
  288. sanitized = _sanitize_cwd_for_live_env(env, new_cwd)
  289. if sanitized is not None:
  290. env.cwd = sanitized
  291. def clear_task_env_overrides(task_id: str):
  292. """Drop a task's overrides, cwd record and container alias (rollout cleanup)."""
  293. _task_env_overrides.pop(_qualify_task_key(task_id), None)
  294. clear_session_cwd(task_id)
  295. with _container_alias_lock:
  296. _container_aliases.pop(task_id, None)
  297. def register_container_alias(child_task_id: str, parent_task_id: Optional[str]) -> None:
  298. """Make *child_task_id* resolve to *parent_task_id*'s container (called at
  299. delegate_task spawn). A missing parent id aliases to ``"default"``."""
  300. if not child_task_id:
  301. return
  302. with _container_alias_lock:
  303. _container_aliases[child_task_id] = str(parent_task_id or "default")
  304. def _resolve_container_alias(task_id: str) -> str:
  305. """Follow the child→parent alias chain (cycle-safe) for *task_id*."""
  306. seen = set()
  307. key = task_id
  308. with _container_alias_lock:
  309. while key in _container_aliases and key not in seen:
  310. seen.add(key)
  311. key = _container_aliases[key]
  312. return key
  313. def _is_isolation_override_key(key: str) -> bool:
  314. """An image (``docker_image``, a plugin backend's ``<backend>_image``) or ``env_type`` override."""
  315. return key == "env_type" or key.endswith("_image")
  316. def _has_isolation_overrides(task_id: Optional[str]) -> bool:
  317. """True when *task_id* registered image/env_type overrides — the single
  318. "isolated RL/benchmark rollout" predicate shared by key resolution and
  319. container creation so the two can't drift."""
  320. overrides = _task_env_overrides.get(_qualify_task_key(task_id)) if task_id else None
  321. return bool(overrides and any(_is_isolation_override_key(str(key)) for key in overrides))
  322. @dataclass(frozen=True)
  323. class _SessionScope:
  324. """Backend identity + scoping predicates for one call, read once.
  325. ``env_type`` is the scope-aware TERMINAL_ENV; ``persistent`` is
  326. ``TERMINAL_CONTAINER_PERSISTENT``. Derived predicates:
  327. * ``session_isolated`` — non-persistent sandboxes get per-session identities:
  328. ``container_persistent: false`` means state must not survive or be shared
  329. across sessions, so one shared sandbox contradicts it. Docker, plus plugin
  330. backends declaring ``session_isolated_when_nonpersistent`` (sandboxes resumed
  331. by name, where a shared deterministic name would let two ephemeral runs
  332. attach one VM and delete it under each other).
  333. * ``docker_session_isolated`` — docker-only view: the workspace mount and
  334. session-scoped teardown paths must not fire for other backends.
  335. * ``docker_profile_scoped`` — docker + ``container_persistent: true``: ONE
  336. long-lived container per profile shared by every session (CLI, gateway,
  337. WebUI). The session-key fallback in :func:`_resolve_container_task_id` stops
  338. cross-profile SSH reuse; ungated it fragmented persistent Docker into one
  339. container per gateway session, so this restores profile scoping for exactly
  340. this backend/mode.
  341. """
  342. env_type: str
  343. persistent: bool
  344. @property
  345. def session_isolated(self) -> bool:
  346. if self.env_type != "docker" and not _plugin_env_flag(
  347. self.env_type, "session_isolated_when_nonpersistent"
  348. ):
  349. return False
  350. return not self.persistent
  351. @property
  352. def docker_session_isolated(self) -> bool:
  353. return self.env_type == "docker" and self.session_isolated
  354. @property
  355. def docker_profile_scoped(self) -> bool:
  356. return self.env_type == "docker" and self.persistent
  357. def _session_scope() -> _SessionScope:
  358. """Bridge config → env once, then snapshot the backend scope for this call."""
  359. _ensure_terminal_env_bridged()
  360. return _SessionScope(
  361. env_type=_tenv("TERMINAL_ENV", "local"),
  362. persistent=_tenv_bool("TERMINAL_CONTAINER_PERSISTENT", "true"),
  363. )
  364. def _docker_session_isolation_enabled() -> bool:
  365. """See :attr:`_SessionScope.docker_session_isolated` (used by the docker builder)."""
  366. return _session_scope().docker_session_isolated
  367. def _routed_home_task_key(profile_scoped: bool) -> Optional[str]:
  368. """Key for a session-less task serving a routed (non-launch) profile home, else None.
  369. A multiplexed host runs every profile's cron jobs without a session key; collapsing them all onto
  370. ``"default"`` made profile B's cron tool calls reuse the environment the launch profile's job
  371. created (its ``.env`` residue, its bridged ``TERMINAL_*``, its shell), so B ran with A's settings.
  372. Persistent Docker keys the profile name exactly like B's session-bound work, so B keeps ONE
  373. container instead of a second one per home path.
  374. """
  375. from hermes_constants import get_hermes_home_override, profile_name_for_home
  376. from tools.environments.local import _is_routed_home
  377. override = get_hermes_home_override()
  378. if not override or not _is_routed_home(override):
  379. return None
  380. profile = profile_name_for_home(override) if profile_scoped else None
  381. if profile:
  382. return profile_container_key(profile)
  383. try:
  384. return f"home:{os.path.realpath(override)}"
  385. except OSError:
  386. return f"home:{override}"
  387. def _sessionless_container_key(profile_scoped: bool) -> str:
  388. """Container key when no session is bound (CLI, cron, kanban dispatcher workers).
  389. A routed home override keeps its existing key. Persistent Docker whose
  390. process home is a named profile reuses ``profile:<name>`` — the same key
  391. session-bound work of that profile uses. Dispatcher workers set
  392. ``HERMES_HOME`` and clear session vars, so they are not a routed override;
  393. without this they collapsed onto ``default`` and missed the profile sandbox.
  394. The default profile, non-persistent backends, and a home outside the Hermes
  395. root stay ``default`` (:func:`~tools.terminal_tool_config.home_profile_name` names the profile).
  396. """
  397. routed = _routed_home_task_key(profile_scoped)
  398. if routed:
  399. return routed
  400. return profile_container_key(home_profile_name()) if profile_scoped else "default"
  401. def _qualify_task_key(key: str) -> str:
  402. """Prefix a session-derived key with the routed profile/home, or return it unchanged.
  403. A multiplexed host serves every profile in one process, and session ids are not
  404. profile-unique (a header-less API client's ``api-<digest>`` fingerprint, a DM chat id
  405. shared by two bots), so raw session keys in ``_active_environments`` / ``_session_cwd``
  406. would hand profile B the sandbox and cwd profile A created (#123989). Persistent Docker
  407. already keys the profile (branches 3/4); this covers the per-session keys. CLI and
  408. single-profile gateways (no routed home) keep the historical raw key.
  409. """
  410. scope = _routed_home_task_key(profile_scoped=True)
  411. return f"{scope}:{key}" if scope else key
  412. def _resolve_container_task_id(task_id: Optional[str]) -> str:
  413. """Map a tool-call ``task_id`` to the ``_active_environments`` key. Order matters —
  414. earlier branches are authoritative where they apply:
  415. 1. Image/``env_type`` overrides (RL/benchmark rollouts) key their own sandbox;
  416. CWD-only overrides (ACP workspace tracking) are NOT isolation signals.
  417. 2. Per-session isolation (docker + ``container_persistent: false``): each
  418. session's task_id is its own key (a fresh chat gets a fresh sandbox with only
  419. ITS mounts); delegate_task children follow the alias registry to the parent.
  420. Routed profiles qualify the key (:func:`_qualify_task_key`, #123989).
  421. 3. Session key present (WebUI per-session, gateway per-message): persistent
  422. Docker is PROFILE-scoped — ``shared:<key>`` opt-in, else ``profile:<name>``
  423. (a session naming no profile takes the process home's), with the default
  424. profile staying literally ``"default"`` so CLI and
  425. default-profile gateway sessions share ONE container; other backends key
  426. ``session:<key>`` so switching profiles can't reuse another profile's
  427. SSHEnvironment on the wrong host.
  428. 4. No session key (CLI, cron, kanban workers): ``shared:<key>`` when opted in
  429. (else a CLI run of a keyed profile would split from its gateway sessions); a
  430. routed multiplexed profile keys its own home; persistent Docker on a
  431. named-profile process reuses ``profile:<name>`` (same sandbox as that
  432. profile's session-bound work); else ``"default"``.
  433. """
  434. if task_id and _has_isolation_overrides(task_id):
  435. return _qualify_task_key(task_id)
  436. scope = _session_scope()
  437. if task_id and scope.session_isolated:
  438. return _qualify_task_key(_resolve_container_alias(task_id))
  439. # Per-session isolation: when a session key is present (the WebUI streaming layer sets it per-session,
  440. # the gateway per-message via contextvars), scope the container to it so switching profiles can't reuse
  441. # a previous profile's SSHEnvironment and silently run commands on the wrong remote host. Subagents
  442. # inherit the same session key, so they still collapse onto the parent's container (the #16177
  443. # shared-container intent). CLI mode has no session key. A named-profile process
  444. # under persistent Docker reuses ``profile:<name>`` (:func:`_sessionless_container_key`);
  445. # the default profile still falls through to "default". See commit e00f940a9. This runs *after* the isolation-override and
  446. # docker/container_persistent branches above: those paths already key containers per task_id, so they
  447. # stay authoritative where they apply and this only covers the cases that would otherwise collapse to
  448. # the shared "default" key (notably SSH).
  449. session_key = _current_session_key()
  450. shared = _tenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() if scope.docker_profile_scoped else ""
  451. if shared:
  452. # Explicit opt-in: trusted profiles configuring the same terminal.docker_shared_container_key share
  453. # ONE container/cache slot (and sandbox dir) regardless of profile name (#84671).
  454. return f"shared:{shared}"
  455. if not session_key:
  456. return _sessionless_container_key(scope.docker_profile_scoped)
  457. if not scope.docker_profile_scoped:
  458. return _qualify_task_key(f"session:{session_key}")
  459. return _workspace_container_key(profile_container_key(_container_profile()), task_id)
  460. def workspace_key_for(cwd: str) -> str:
  461. """The ``:ws:`` digest of a host workspace: computed ONCE when the session binds it
  462. (``gateway.session_policy.bind_turn_workspace``), so container keying never depends on live
  463. filesystem state at lookup time."""
  464. import hashlib
  465. resolved = os.path.realpath(os.path.expanduser(cwd))
  466. return hashlib.sha256(resolved.encode("utf-8")).hexdigest()[:16]
  467. def _workspace_container_key(base: str, task_id: Optional[str]) -> str:
  468. """Persistent Docker key for a session that registered a workspace: one container per
  469. (profile, workspace), not per profile (#140665).
  470. A gateway-owned CLI/Desktop session registers its client cwd as a ``cwd_source: "session"``
  471. override (``gateway.session_policy.bind_turn_workspace``). Keyed by profile alone, every
  472. concurrent session shared the container bound to whichever workspace created it first; sessions
  473. in the same folder still share one. Sessions with no registered workspace keep the profile key,
  474. and so does every session while ``docker_mount_cwd_to_workspace`` is off: nothing is bound into
  475. the container then, so a workspace must not split the profile's sandbox from its CLI files.
  476. Idempotent: an already-resolved ``<base>:ws:<digest>`` key maps to itself (``ensure_task_env``
  477. hands the resolved key to ``get_active_env``, which resolves again). A ``delegate_task`` child has
  478. only a container alias to its parent (``register_container_alias``), so the alias chain is
  479. followed: the child runs in its parent's workspace container, never the profile's.
  480. Reads the raw table (not :func:`resolve_task_overrides`, which calls back into key resolution).
  481. """
  482. if not task_id:
  483. return base
  484. if task_id.startswith(f"{base}:ws:"):
  485. return task_id
  486. # Without the cwd mount nothing is bound into the container: a registered workspace (a
  487. # Desktop session's folder) must not split the profile's sandbox from its CLI files.
  488. if not _tenv_bool("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false"):
  489. return base
  490. for key in dict.fromkeys((task_id, _resolve_container_alias(task_id))):
  491. overrides = _task_env_overrides.get(_qualify_task_key(key))
  492. if not overrides or overrides.get("cwd_source") != "session":
  493. continue
  494. digest = overrides.get("workspace_key")
  495. cwd = overrides.get("cwd")
  496. if not digest and isinstance(cwd, str) and cwd.strip() and "\0" not in cwd:
  497. digest = workspace_key_for(cwd)
  498. if digest:
  499. return f"{base}:ws:{digest}"
  500. return base
  501. def resolve_task_overrides(task_id: Optional[str]) -> dict[str, Any]:
  502. """Return the env overrides for *task_id*, raw key first then collapsed.
  503. ``register_task_env_overrides`` writes under the *raw* task/session id, but
  504. a CWD-only override collapses (:func:`_resolve_container_task_id`) to the
  505. shared ``"default"`` container. Callers must therefore read the raw id
  506. FIRST and only fall back to the collapsed container id, or the originating
  507. session's override is silently dropped. Single source of that lookup so
  508. the terminal and file layers can't drift apart.
  509. """
  510. raw = task_id or "default"
  511. return (
  512. _task_env_overrides.get(_qualify_task_key(raw))
  513. or _task_env_overrides.get(_resolve_container_task_id(raw))
  514. or {}
  515. )
  516. # Backends that take an image, keyed to the override/config key carrying it.
  517. _IMAGE_KEY_BY_BACKEND = {
  518. "docker": "docker_image",
  519. "modal": "modal_image",
  520. "daytona": "daytona_image",
  521. }
  522. def _select_image(env_type: str, overrides: dict[str, Any], config: dict[str, Any]) -> str:
  523. """Image for *env_type*: per-task override first, then config; "" for imageless backends. A plugin
  524. backend gets only its per-task ``<backend>_image`` override (batch runs register one); otherwise it
  525. reads its own image setting."""
  526. key = _IMAGE_KEY_BY_BACKEND.get(env_type)
  527. if key is None:
  528. return overrides.get(f"{env_type}_image") or ""
  529. return overrides.get(key) or config[key]
  530. def _lookup_active_env(effective_task_id: str, task_id: Optional[str]):
  531. """Return the cached env for the collapsed id, else for the raw task_id, else None.
  532. Caller holds ``_env_lock``. Per-session surfaces (ACP/gateway/dashboard)
  533. with a CWD-only override collapse to ``"default"`` for container sharing,
  534. yet an env may already be cached under the originating task_id; honor it
  535. instead of spawning a duplicate. Refreshes ``_last_activity`` on a hit.
  536. """
  537. for key in (effective_task_id, task_id):
  538. if key and key in _active_environments:
  539. _last_activity[key] = time.time()
  540. return _active_environments[key]
  541. return None
  542. def _resolve_task_host_cwd(config: dict[str, Any], task_id: Optional[str]) -> Optional[str]:
  543. """Host directory to bind into *task_id*'s container.
  544. Single owner of the cwd-mount policy for every creation site. Shared-
  545. container mode: the ``TERMINAL_CWD``-derived ``config["host_cwd"]``.
  546. Per-session isolation (docker + ``container_persistent: false``): only
  547. the SESSION's own registered workspace may mount — the process env var is
  548. a launch artifact that outlives the session that set it, so deriving a
  549. fresh session's mount from it would leak the previous session's directory.
  550. Overrides tagged ``cwd_source: "process"`` are refused for the same reason;
  551. ``cwd_source: "session"`` or untagged (ACP/RL) overrides mount.
  552. A Windows drive path is not a mount source while the cwd-to-/workspace flag
  553. is off. A raw host override must stay out of ``docker run -w`` and fall
  554. back to the sanitized config cwd. The Windows bind, including when
  555. ``/workspace`` is already claimed, is the volume mount, not this override.
  556. """
  557. if config.get("env_type") != "docker" or not config.get("docker_mount_cwd_to_workspace"):
  558. return None
  559. # Top-level CLI parent ("default") is a single-session process — legacy behavior.
  560. if not _docker_session_isolation_enabled() or _resolve_container_task_id(task_id) == "default":
  561. return config.get("host_cwd")
  562. overrides = resolve_task_overrides(task_id)
  563. candidate = overrides.get("cwd")
  564. if overrides.get("cwd_source") == "process" or not isinstance(candidate, str) or not candidate.strip():
  565. return None
  566. candidate = os.path.abspath(os.path.expanduser(candidate))
  567. # Must exist on the host and not already be an in-container path.
  568. if not os.path.isdir(candidate) or candidate.startswith(("/workspace", "/root")):
  569. return None
  570. return candidate
  571. # One-shot guard for the config-fallback bridge: after the first attempt
  572. # either TERMINAL_ENV is set or the import failed, so retrying is wasted work.
  573. _terminal_config_bridge_attempted = False
  574. def _ensure_terminal_env_bridged() -> None:
  575. """Backfill TERMINAL_* env vars from config.yaml when no launcher did.
  576. CLI, gateway and TUI/dashboard PTY launches bridge ``terminal.*`` into env vars
  577. at startup; processes that skip those paths (``hermes serve``, Desktop
  578. in-process agents, desktop cron ticker, ACP) would otherwise fall back to the
  579. local backend even when config selects docker — running on the host the user
  580. meant to sandbox. Explicit keys in the ``terminal`` section override matching
  581. env values (possibly stale from ``hermes setup``); env values for omitted keys
  582. are preserved. Without a terminal section an existing TERMINAL_ENV is kept and
  583. defaults are backfilled only when none is set. A per-turn terminal scope
  584. suppresses the bridge entirely: writing scope values into the process-global
  585. env would re-create the first-writer-wins cross-profile leak the scope fixes.
  586. Ambient ``os.environ`` is the *launch* profile's authority only. Under a
  587. context-local ``HERMES_HOME`` override (multiplexed dashboard / gateway
  588. secondary profile), this bridge is a no-op — otherwise the first unscoped
  589. call under that override would latch the secondary profile's ``terminal.*``
  590. into process-global env and poison later unscoped launch-profile turns
  591. (#107422 residual of #68559). Routed profiles must bind a terminal scope
  592. instead (same rule as ``env_loader._reapply_terminal_config_bridge``).
  593. terminal_tool reads ALL terminal settings from os.environ (TERMINAL_*). See #61115, #65696.
  594. """
  595. from tools.terminal_scope import get_terminal_scope
  596. if get_terminal_scope() is not None:
  597. return
  598. # Never write a secondary profile's terminal.* into process-global env.
  599. from hermes_constants import get_hermes_home_override
  600. if get_hermes_home_override() is not None:
  601. return
  602. global _terminal_config_bridge_attempted
  603. if _terminal_config_bridge_attempted:
  604. return
  605. _terminal_config_bridge_attempted = True
  606. # Never let a config problem take the terminal tool down.
  607. with _quiet("terminal config → env fallback bridge failed"):
  608. from hermes_cli.config import apply_terminal_config_to_env, read_raw_config
  609. raw_config = read_raw_config()
  610. if isinstance(raw_config.get("terminal"), dict):
  611. apply_terminal_config_to_env(env=None, override=True)
  612. elif "TERMINAL_ENV" not in os.environ:
  613. apply_terminal_config_to_env(env=None, override=False)
  614. # Default cwd per backend; plugins may declare ``default_cwd``; anything else (container backends) is "/root".
  615. _DEFAULT_CWD_BY_BACKEND = {"ssh": "~", "vercel_sandbox": _VERCEL_SANDBOX_DEFAULT_CWD}
  616. def _backend_default_cwd(env_type: str) -> str:
  617. if env_type == "local":
  618. return _safe_getcwd()
  619. if env_type in _DEFAULT_CWD_BY_BACKEND:
  620. return _DEFAULT_CWD_BY_BACKEND[env_type]
  621. declared = _plugin_env_flag(env_type, "default_cwd", None)
  622. # "~" is each sandbox's own home, expanded inside it; otherwise only an absolute in-sandbox path
  623. # (a relative value would be anchored on the host).
  624. if isinstance(declared, str) and (declared == "~" or declared.startswith("/")):
  625. return declared
  626. return "/root"
  627. def _resolve_config_cwd(env_type: str, mount_docker_cwd: bool) -> tuple:
  628. """``(cwd, host_cwd)`` from TERMINAL_CWD for *env_type*.
  629. Container backends are sanity-checked: with Docker cwd passthrough the host
  630. path is remapped to /workspace and tracked as host_cwd; otherwise host paths
  631. are discarded in favor of the backend default.
  632. """
  633. default_cwd = _backend_default_cwd(env_type)
  634. cwd = _tenv("TERMINAL_CWD", default_cwd)
  635. from hermes_cli.config import _is_ssh_remote_tilde_cwd
  636. # The backend's own default is an in-sandbox path: its ``~`` is the sandbox home, not ours.
  637. if cwd and cwd != default_cwd and not _is_ssh_remote_tilde_cwd(env_type, cwd):
  638. cwd = os.path.expanduser(cwd)
  639. host_cwd = None
  640. if env_type == "docker" and mount_docker_cwd:
  641. candidate = os.path.abspath(os.path.expanduser(_tenv("TERMINAL_CWD") or _safe_getcwd()))
  642. if (
  643. _is_host_cwd(candidate)
  644. or (os.path.isabs(candidate) and os.path.isdir(candidate) and not candidate.startswith(("/workspace", "/root")))
  645. ):
  646. host_cwd = candidate
  647. cwd = "/workspace"
  648. elif env_type == "docker" and _is_windows_drive_path(cwd):
  649. # A Windows workspace cannot exist inside the Linux container. Bind it
  650. # even when docker_mount_cwd_to_workspace is off; the env retargets cwd
  651. # to the mount (which may not be /workspace).
  652. candidate = os.path.expanduser(cwd)
  653. if os.name == "nt":
  654. candidate = os.path.abspath(candidate)
  655. if os.path.isdir(candidate):
  656. host_cwd = candidate
  657. cwd = candidate
  658. else:
  659. logger.info("Ignoring TERMINAL_CWD=%r for %s backend "
  660. "(host/relative path won't work in sandbox). Using %r instead.",
  661. cwd, env_type, default_cwd)
  662. cwd = default_cwd
  663. elif _is_container_backend(env_type) and cwd and _is_unusable_container_cwd(cwd) and cwd != default_cwd:
  664. logger.info("Ignoring TERMINAL_CWD=%r for %s backend "
  665. "(host/relative path won't work in sandbox). Using %r instead.",
  666. cwd, env_type, default_cwd)
  667. cwd = default_cwd
  668. return cwd, host_cwd
  669. def _get_env_config() -> dict[str, Any]:
  670. """Resolve the terminal configuration dict from TERMINAL_* env vars."""
  671. from hermes_cli.config_defaults import DEFAULT_SANDBOX_IMAGE as default_image
  672. _ensure_terminal_env_bridged()
  673. env_type = _tenv("TERMINAL_ENV", "local")
  674. mount_docker_cwd = _tenv_bool("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false")
  675. # Container/docker-only payloads are parsed only when such a backend is
  676. # selected: a stale or invalid Docker value bridged from config.yaml must
  677. # not make local terminal/execute_code unusable.
  678. if _is_container_backend(env_type):
  679. container_cpu = _parse_env_var("TERMINAL_CONTAINER_CPU", "1", float, "number")
  680. container_memory = _parse_env_var("TERMINAL_CONTAINER_MEMORY", "5120")
  681. container_disk = _parse_env_var("TERMINAL_CONTAINER_DISK", "51200")
  682. else:
  683. container_cpu, container_memory, container_disk = 1.0, 5120, 51200
  684. if env_type == "docker":
  685. docker_forward_env = _parse_env_var("TERMINAL_DOCKER_FORWARD_ENV", "[]", json.loads, "valid JSON")
  686. docker_volumes = _parse_env_var("TERMINAL_DOCKER_VOLUMES", "[]", json.loads, "valid JSON")
  687. docker_env = _parse_env_var("TERMINAL_DOCKER_ENV", "{}", json.loads, "valid JSON")
  688. docker_extra_args = _parse_env_var("TERMINAL_DOCKER_EXTRA_ARGS", "[]", json.loads, "valid JSON")
  689. docker_shm_size = _tenv("TERMINAL_DOCKER_SHM_SIZE", "1g")
  690. else:
  691. docker_forward_env, docker_volumes, docker_env, docker_extra_args, docker_shm_size = [], [], {}, [], "1g"
  692. cwd, host_cwd = _resolve_config_cwd(env_type, mount_docker_cwd)
  693. return {
  694. "env_type": env_type,
  695. "modal_mode": coerce_modal_mode(_tenv("TERMINAL_MODAL_MODE", "auto")),
  696. "docker_image": _tenv("TERMINAL_DOCKER_IMAGE", default_image),
  697. "docker_image_pinned": _tenv("TERMINAL_DOCKER_IMAGE_PINNED", "0") == "1",
  698. "docker_forward_env": docker_forward_env,
  699. "modal_image": _tenv("TERMINAL_MODAL_IMAGE", default_image),
  700. "daytona_image": _tenv("TERMINAL_DAYTONA_IMAGE", default_image),
  701. "vercel_runtime": _tenv("TERMINAL_VERCEL_RUNTIME", "").strip(),
  702. "vercel_image": _tenv("TERMINAL_VERCEL_IMAGE", "").strip(),
  703. "cwd": cwd,
  704. "host_cwd": host_cwd,
  705. "docker_mount_cwd_to_workspace": mount_docker_cwd,
  706. "timeout": _parse_env_var("TERMINAL_TIMEOUT", "180"),
  707. "lifetime_seconds": _parse_env_var("TERMINAL_LIFETIME_SECONDS", "300"),
  708. # SSH-specific config
  709. "ssh_host": _tenv("TERMINAL_SSH_HOST", ""),
  710. "ssh_user": _tenv("TERMINAL_SSH_USER", ""),
  711. "ssh_port": _parse_env_var("TERMINAL_SSH_PORT", "22"),
  712. "ssh_key": _tenv("TERMINAL_SSH_KEY", ""),
  713. # Exported env carries across commands (cwd always does). SSH follows
  714. # terminal.persistent_shell; local has its own knob. Both default on.
  715. "ssh_persistent": _tenv_bool(
  716. "TERMINAL_SSH_PERSISTENT", _tenv("TERMINAL_PERSISTENT_SHELL", "true"),
  717. ),
  718. "local_persistent": _tenv_bool("TERMINAL_LOCAL_PERSISTENT", "true"),
  719. # Container resources (MB); ignored for local/ssh.
  720. "container_cpu": container_cpu,
  721. "container_memory": container_memory,
  722. "container_disk": container_disk,
  723. "container_persistent": _tenv_bool("TERMINAL_CONTAINER_PERSISTENT", "true"),
  724. "docker_volumes": docker_volumes,
  725. "docker_env": docker_env,
  726. "docker_run_as_host_user": _tenv_bool("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false"),
  727. "docker_snap_compat": _tenv_bool("TERMINAL_DOCKER_SNAP_COMPAT", "false"),
  728. "docker_network": _tenv_bool("TERMINAL_DOCKER_NETWORK", "true"),
  729. "docker_extra_args": docker_extra_args,
  730. "docker_shm_size": docker_shm_size,
  731. # Cross-process reuse: attach to a labeled container at startup
  732. # instead of starting fresh; false = per-process isolation.
  733. "docker_persist_across_processes": _tenv_bool("TERMINAL_DOCKER_PERSIST_ACROSS_PROCESSES", "true"),
  734. "docker_shared_container_key": _tenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip(),
  735. "docker_orphan_reaper": _tenv_bool("TERMINAL_DOCKER_ORPHAN_REAPER", "true"),
  736. }
  737. def _cleanup_thread_worker():
  738. """Background thread worker that periodically cleans up inactive environments."""
  739. while _cleanup_running:
  740. with _quiet("Error in cleanup thread", level=logging.WARNING):
  741. _cleanup_inactive_envs(_get_env_config()["lifetime_seconds"])
  742. for _ in range(60):
  743. if not _cleanup_running:
  744. break
  745. time.sleep(1)
  746. def _start_cleanup_thread():
  747. """Start the background cleanup thread if not already running."""
  748. global _cleanup_thread, _cleanup_running
  749. with _env_lock:
  750. if _cleanup_thread is None or not _cleanup_thread.is_alive():
  751. _cleanup_running = True
  752. _cleanup_thread = threading.Thread(target=_cleanup_thread_worker, daemon=True)
  753. _cleanup_thread.start()
  754. def _stop_cleanup_thread():
  755. """Stop the background cleanup thread."""
  756. global _cleanup_running
  757. _cleanup_running = False
  758. if _cleanup_thread is not None:
  759. try:
  760. _cleanup_thread.join(timeout=5)
  761. except (SystemExit, KeyboardInterrupt):
  762. pass
  763. def _atexit_cleanup():
  764. """Stop the cleanup thread and shut down all remaining sandboxes on exit."""
  765. _stop_cleanup_thread()
  766. if _active_environments:
  767. logger.info("Shutting down %d remaining sandbox(es)...", len(_active_environments))
  768. # Snapshot BEFORE cleanup_all_environments empties the dict, then
  769. # block briefly so docker stop/rm completes before the interpreter
  770. # exits — otherwise daemon cleanup threads die mid-`docker stop` and
  771. # Exited containers pile up on the host.
  772. envs_to_wait = list(_active_environments.values())
  773. cleanup_all_environments()
  774. for env in envs_to_wait:
  775. wait_fn = getattr(env, "wait_for_cleanup", None)
  776. if wait_fn is not None:
  777. with _quiet("wait_for_cleanup raised on exit"): # never block shutdown on a bad backend
  778. wait_fn(timeout=15.0)
  779. # Workers of envs the idle reaper already detached are not in the registry (#86317).
  780. if "tools.environments.docker" in sys.modules:
  781. with _quiet("teardown drain raised on exit"):
  782. sys.modules["tools.environments.docker"].DockerEnvironment.wait_for_all_teardowns(timeout=15.0)
  783. atexit.register(_atexit_cleanup)
  784. def _command_requires_pipe_stdin(command: str) -> bool:
  785. """True when PTY mode would break a stdin-driven command: `gh auth login
  786. --with-token` waits for EOF on piped stdin, and under a PTY
  787. `process.submit()` only sends a newline, so it hangs forever."""
  788. normalized = " ".join(command.lower().split())
  789. return normalized.startswith("gh auth login") and "--with-token" in normalized
  790. from tools.terminal_tool_guards import (
  791. _foreground_background_guidance, _safe_command_preview, _validate_workdir,
  792. gateway_lifecycle_block, locked_skill_block, self_repo_block, trailing_amp_background_command,
  793. )
  794. from tools.terminal_tool_background import _YIELDED_NOTE, spawn_background_process, yield_to_background_handler
  795. from tools.terminal_tool_result import finalize_foreground_result
  796. def _resolve_notification_flag_conflict(*, notify_on_complete: bool, watch_patterns, background: bool) -> tuple:
  797. """Resolve notify_on_complete + watch_patterns both set: drop watch_patterns
  798. (combined they produce duplicate async notifications — one per match plus
  799. one on exit — that can spam the user long after the process ends).
  800. Returns ``(watch_patterns_to_use, conflict_note)``; note is "" without conflict."""
  801. if background and notify_on_complete and watch_patterns:
  802. return None, (
  803. "watch_patterns ignored because notify_on_complete=True; "
  804. "these two flags produce duplicate notifications when combined"
  805. )
  806. return watch_patterns, ""
  807. def _rewrite_via_env_mount(path: str, env) -> str | None:
  808. """Container path for *path* when *env* bind-mounted that host directory."""
  809. if env is None or not path:
  810. return None
  811. host = getattr(env, "host_cwd", None)
  812. mount = getattr(env, "host_cwd_mount", None)
  813. if not isinstance(host, str) or not host or not mount:
  814. return None
  815. return translate_mounted_host_path(path, host, mount)
  816. def _mount_envs(env):
  817. if env is not None:
  818. return [env]
  819. return list(_active_environments.values())
  820. def _container_visible_cwd(path: str, env_type: str | None, env=None) -> str:
  821. if not path or not _is_container_backend(env_type or ""):
  822. return path
  823. for item in _mount_envs(env):
  824. translated = _rewrite_via_env_mount(path, item)
  825. if translated:
  826. return translated
  827. return path
  828. def _container_visible_default(default_cwd: str, env_type: str | None, env=None) -> str:
  829. """Point a planner ``/workspace`` assumption at the mount that actually holds the host cwd."""
  830. if not _is_container_backend(env_type or ""):
  831. return default_cwd
  832. for item in _mount_envs(env):
  833. mount = getattr(item, "host_cwd_mount", None)
  834. host = getattr(item, "host_cwd", None)
  835. if not mount or not host or mount == default_cwd:
  836. continue
  837. if default_cwd == "/workspace" or _is_unusable_container_cwd(default_cwd):
  838. return mount
  839. return default_cwd
  840. def _resolve_command_cwd(
  841. *,
  842. workdir: Optional[str],
  843. default_cwd: str,
  844. session_key: Optional[str] = None,
  845. env_type: Optional[str] = None,
  846. mounted_host: Optional[str] = None,
  847. env=None,
  848. ) -> str:
  849. """cwd for a command: explicit ``workdir`` > the session's own cwd record >
  850. ``default_cwd``.
  851. The record is written after every completed command of THIS session, so
  852. it is the session's ``cd`` state with no shared-env ambiguity. On
  853. container backends a recorded HOST path (a desktop/TUI surface registering
  854. its workspace) is unusable in the sandbox — ``cd <host path>`` fails with
  855. exit 126 — so it is discarded in favor of ``default_cwd``.
  856. A recorded cwd that IS the host directory mounted at ``/workspace`` is
  857. classified unusable before the ``/Users`` / ``/home`` / drive-letter
  858. heuristic (``/mnt/...``, ``/srv/...`` are absolute and miss that heuristic)
  859. and remapped to ``/workspace`` so the session wrapper does not ``cd`` to it.
  860. Same guard class as the env-creation sanitizers (#50636, #54447); this is the per-command sibling site.
  861. """
  862. if workdir:
  863. return coerce_ssh_remote_cwd(_container_visible_cwd(workdir, env_type, env), env_type)
  864. recorded = get_session_cwd(session_key)
  865. if recorded and _is_container_backend(env_type) and _is_unusable_container_cwd(
  866. recorded, mounted_host=mounted_host
  867. ) and not _in_sandbox_home(recorded, env):
  868. visible = _container_visible_cwd(recorded, env_type, env)
  869. if visible != recorded:
  870. return visible
  871. if _is_mounted_host_cwd(recorded, mounted_host):
  872. logger.info(
  873. "Remapping recorded session cwd %r for %s backend "
  874. "(mounted host directory). Using '/workspace' instead.",
  875. recorded, env_type,
  876. )
  877. return "/workspace"
  878. logger.info(
  879. "Ignoring recorded session cwd %r for %s backend "
  880. "(host/relative path won't work in sandbox). Using %r instead.",
  881. recorded, env_type, default_cwd,
  882. )
  883. return _container_visible_default(default_cwd, env_type, env)
  884. return coerce_ssh_remote_cwd(recorded or _container_visible_default(default_cwd, env_type, env), env_type)
  885. def _error_json(error: str, *, exit_code: int = -1, status: Optional[str] = None, **extra) -> str:
  886. """The terminal error envelope: ``output``/``exit_code``/``error`` (+ ``status``, extras)."""
  887. body: dict[str, Any] = {"output": "", "exit_code": exit_code, "error": error}
  888. if status is not None:
  889. body["status"] = status
  890. body.update(extra)
  891. return json.dumps(body, ensure_ascii=False)
  892. def _fatal_error_json(e: BaseException) -> str:
  893. """Log the traceback and return the redacted error+traceback envelope.
  894. Exception text can embed the failing command line (and any secrets inline
  895. in it), so both fields are force-redacted before reaching the model.
  896. """
  897. import traceback
  898. tb_str = traceback.format_exc()
  899. logger.error("terminal_tool exception:\n%s", tb_str)
  900. return json.dumps({
  901. "output": "",
  902. "exit_code": -1,
  903. "error": _redact_terminal_error_text(f"Failed to execute command: {e}"),
  904. "traceback": _redact_terminal_error_text(tb_str),
  905. "status": "error"
  906. }, ensure_ascii=False)
  907. class _Rejected(Exception):
  908. """Carries a finished tool-result JSON out of the planning/guard helpers, so
  909. each early-return site is one ``raise`` instead of an isinstance-checked
  910. ``str | plan`` union at the caller."""
  911. def __init__(self, result_json: str):
  912. super().__init__(result_json)
  913. self.result_json = result_json
  914. @dataclass
  915. class _ApprovalVerdict:
  916. """Outcome of the pre-exec guard pass.
  917. ``note`` is the audit note attached to the result. ``approved_run`` is True
  918. when the user explicitly approved (or pre-confirmed via ``force``); it drives
  919. the clean-interrupt-slate clear before ``env.execute`` so an approved command
  920. can't be SIGINT-killed by a bit that landed during the approval-wait.
  921. """
  922. note: Optional[str] = None
  923. approved_run: bool = False
  924. def _run_approval_guards(command: str, env_type: str, config: dict[str, Any], *, force: bool) -> _ApprovalVerdict:
  925. """Run the command guards; ``force`` skips them entirely.
  926. Raises :class:`_Rejected` when the command may not run (denied, or pending
  927. gateway approval)."""
  928. if force:
  929. return _ApprovalVerdict(approved_run=True)
  930. approval = _check_all_guards(command, env_type, has_host_access=_docker_has_host_access(config))
  931. if not approval["approved"]:
  932. if approval.get("status") == "pending_approval": # gateway ask mode
  933. raise _Rejected(_error_json(
  934. "", status="pending_approval",
  935. approval_pending=True,
  936. command=approval.get("command", command),
  937. description=approval.get("description", "command flagged"),
  938. pattern_key=approval.get("pattern_key", ""),
  939. smart_denied=approval.get("smart_denied", False),
  940. allow_permanent=approval.get("allow_permanent", True),
  941. ))
  942. desc = approval.get("description", "command flagged")
  943. fallback_msg = (
  944. f"Command denied: {desc}. "
  945. "Use the approval prompt to allow it, or rephrase the command."
  946. )
  947. raise _Rejected(_error_json(approval.get("message", fallback_msg), status="blocked",
  948. **({"user_summary": approval["user_summary"]} if approval.get("user_summary") else {})))
  949. desc = approval.get("description", "flagged as dangerous")
  950. if approval.get("user_approved"):
  951. return _ApprovalVerdict(
  952. note=f"Command required approval ({desc}) and was approved by the user.",
  953. approved_run=True,
  954. )
  955. if approval.get("smart_approved"):
  956. return _ApprovalVerdict(note=f"Command was flagged ({desc}) and auto-approved by smart approval.")
  957. if approval.get("pre_approved"):
  958. from tools.approval_provenance import preapproval_note
  959. return _ApprovalVerdict(note=preapproval_note(approval["pre_approved"]))
  960. return _ApprovalVerdict()
  961. @dataclass
  962. class _ExecPlan:
  963. """Per-call execution parameters resolved before any environment is touched."""
  964. config: dict[str, Any]
  965. env_type: str
  966. effective_task_id: str
  967. image: str
  968. cwd: str
  969. host_cwd: Optional[str]
  970. effective_timeout: int
  971. # Set when a foreground call asked for more than FOREGROUND_MAX_TIMEOUT and was promoted to a
  972. # tracked background process instead of being refused (the requested seconds, for the note).
  973. promoted_from_foreground_timeout: Optional[int] = None
  974. # Set when a foreground `cmd &` (the trailing `&` its only backgrounding) is run as
  975. # terminal(command="cmd", background=true) instead of being refused: the command to run.
  976. auto_background_command: Optional[str] = None
  977. _PROMOTED_NOTE = (
  978. "Requested foreground timeout {requested}s exceeds the {cap}s cap, so this command was started as a "
  979. "tracked background process with notify_on_complete=true instead of being refused. Do NOT re-run it. "
  980. "Its completion (exit code + output tail) arrives as a notification; poll with "
  981. "process(action=\"poll\", session_id=...) if you need it sooner."
  982. )
  983. def _plan_execution(
  984. command: Any, *, task_id: Optional[str], timeout: Optional[int],
  985. background: bool, _host_local: bool,
  986. ) -> _ExecPlan:
  987. """Resolve backend, env-cache key, image, cwd and timeout for one call.
  988. Raises :class:`_Rejected` when the call is rejected up front (non-string
  989. command, non-positive or over-cap timeout, a foreground command that must
  990. run in the background).
  991. """
  992. if not isinstance(command, str):
  993. logger.warning("Rejected invalid terminal command value: %s", type(command).__name__)
  994. raise _Rejected(_error_json(
  995. f"Invalid command: expected string, got {type(command).__name__}", status="error",
  996. ))
  997. config = _get_env_config()
  998. env_type = "local" if _host_local else config["env_type"]
  999. # Fail closed under a refusal scope: the routed profile's terminal
  1000. # policy could not be resolved, so running with the launch process's
  1001. # ambient policy is forbidden.
  1002. # See #68559.
  1003. if not _host_local:
  1004. from tools.terminal_scope import enforce_no_refusal
  1005. enforce_no_refusal()
  1006. effective_task_id = _resolve_container_task_id(task_id)
  1007. if _host_local:
  1008. # Control-plane children run beside this interpreter, never inside
  1009. # the configured Docker/SSH backend; keep their env cache separate.
  1010. effective_task_id = f"host-local-{effective_task_id}"
  1011. # Per-task overrides (RL/benchmark envs, ACP workspace cwd) win over
  1012. # the global env-var config; ``resolve_task_overrides`` reads the raw
  1013. # task id first, then the collapsed container id.
  1014. overrides = resolve_task_overrides(task_id)
  1015. image = _select_image(env_type, overrides, config)
  1016. cwd = coerce_ssh_remote_cwd(
  1017. overrides.get("cwd") or get_session_cwd(task_id) or config["cwd"], env_type)
  1018. host_cwd = _resolve_task_host_cwd(config, task_id)
  1019. # config["cwd"] was sanitized for container backends in _get_env_config
  1020. # but an override / session record is raw: a host path would reach
  1021. # `docker run -w` and fail with exit 125. Re-apply the guard to the
  1022. # resolved cwd; when the host path IS this session's mounted workspace,
  1023. # remap to /workspace instead of discarding it. Mount equality is part of
  1024. # the unusable check so /mnt and /srv are not left as the container cwd.
  1025. if _is_container_backend(env_type) and _is_unusable_container_cwd(cwd, mounted_host=host_cwd):
  1026. remapped = "/workspace" if host_cwd else config["cwd"]
  1027. if cwd != remapped:
  1028. logger.info(
  1029. "Remapping host/relative cwd override %r for %s backend "
  1030. "(won't exist in sandbox). Using %r instead.",
  1031. cwd, env_type, remapped,
  1032. )
  1033. cwd = remapped
  1034. # Reject non-positive timeouts before deadline math: ``timeout or
  1035. # default`` would silently turn 0 into the default, and a negative
  1036. # value is truthy and would fire an immediate "-Ns" timeout.
  1037. if timeout is not None and timeout <= 0:
  1038. raise _Rejected(tool_error(f"timeout must be a positive number of seconds (got {timeout})."))
  1039. promoted = None
  1040. if not background:
  1041. # An over-cap foreground timeout is a bounded job the caller wants to wait for (test suites,
  1042. # builds). Refusing it only bought a mechanical retry: 454 refusals in one run, every one
  1043. # re-sent lower/split/background. Promote to a tracked background process instead; the
  1044. # caller is told in the result. The `&`/nohup/server guidance below stays a refusal: those
  1045. # need the command itself rewritten, which the tool cannot do safely.
  1046. # The detachment guidance applies whether or not the call is promoted: a promoted `cmd &`
  1047. # would start a tracked shell that exits at once while its payload runs untracked.
  1048. # Exception: a lone TRAILING `&` is unambiguous (the caller asked for exactly what
  1049. # background=true does), so run it that way instead of spending a model turn on a resend.
  1050. auto_bg = trailing_amp_background_command(command)
  1051. if auto_bg is None:
  1052. guidance = _foreground_background_guidance(command)
  1053. if guidance:
  1054. raise _Rejected(_error_json(guidance, status="error"))
  1055. if timeout and timeout > FOREGROUND_MAX_TIMEOUT:
  1056. promoted = timeout
  1057. else:
  1058. auto_bg = None
  1059. return _ExecPlan(
  1060. config=config, env_type=env_type, effective_task_id=effective_task_id,
  1061. image=image, cwd=cwd, host_cwd=host_cwd, effective_timeout=timeout or config["timeout"],
  1062. promoted_from_foreground_timeout=promoted, auto_background_command=auto_bg,
  1063. )
  1064. _PROMOTED_NOTE_POLL_ONLY = (
  1065. "Requested foreground timeout {requested}s exceeds the {cap}s cap, so this command was started as a "
  1066. "tracked background process instead of being refused. Do NOT re-run it. This session cannot receive "
  1067. "completion notifications, so poll it with process(action=\"poll\", session_id=...) until it exits."
  1068. )
  1069. def _with_promoted_note(result_json: str, requested_timeout: int) -> str:
  1070. """Attach the foreground->background promotion note to a spawn result (unchanged on error). The
  1071. note only promises a notification when the spawn actually kept notify_on_complete (finite sessions
  1072. such as one-shot runners cannot route one back; the spawn already said so and cleared the flag)."""
  1073. try:
  1074. data = json.loads(result_json)
  1075. except (TypeError, ValueError):
  1076. return result_json
  1077. if not isinstance(data, dict) or data.get("error"):
  1078. return result_json
  1079. template = _PROMOTED_NOTE if data.get("notify_on_complete") else _PROMOTED_NOTE_POLL_ONLY
  1080. data["promoted_from_foreground"] = template.format(requested=requested_timeout, cap=FOREGROUND_MAX_TIMEOUT)
  1081. return json.dumps(data, ensure_ascii=False)
  1082. def _acquire_env(plan: _ExecPlan, task_id: Optional[str]) -> Any:
  1083. """Cached env for the task, else create it under the per-task creation lock.
  1084. Concurrent calls for the same task_id wait for the first sandbox instead
  1085. of each creating their own; the cache is re-checked under that lock.
  1086. Raises :class:`_Rejected` with the ``"disabled"`` envelope when creation
  1087. raises ImportError.
  1088. """
  1089. _start_cleanup_thread()
  1090. env_type, eff = plan.env_type, plan.effective_task_id
  1091. with _env_lock:
  1092. env: Any = _lookup_active_env(eff, task_id)
  1093. if env is not None:
  1094. return env
  1095. with _creation_locks_lock:
  1096. task_lock = _creation_locks.setdefault(eff, threading.Lock())
  1097. with task_lock:
  1098. with _env_lock:
  1099. env = _lookup_active_env(eff, task_id)
  1100. if env is not None:
  1101. return env
  1102. logger.info("Creating new %s environment for task %s...", env_type, eff[:8])
  1103. try:
  1104. new_env = _create_configured_env(
  1105. plan.config, env_type, image=plan.image, cwd=plan.cwd,
  1106. timeout=plan.effective_timeout, task_id=eff, host_cwd=plan.host_cwd,
  1107. )
  1108. except ImportError as e:
  1109. raise _Rejected(_error_json(
  1110. _redact_terminal_error_text(f"Terminal tool disabled: environment creation failed ({e})"),
  1111. status="disabled",
  1112. ))
  1113. with _env_lock:
  1114. _active_environments[eff] = new_env
  1115. _last_activity[eff] = time.time()
  1116. logger.info("%s environment ready for task %s", env_type, eff[:8])
  1117. return new_env
  1118. def _yield_kwargs(command: str, **ctx) -> dict:
  1119. """``env.execute`` kwargs enabling yield-to-background (local backend only)."""
  1120. handler = yield_to_background_handler(command=command, **ctx)
  1121. return {"yield_handler": handler} if handler is not None else {}
  1122. def _run_foreground(
  1123. command: str, env: Any, plan: _ExecPlan, *,
  1124. task_id: Optional[str], session_id: Optional[str], session_key: str,
  1125. workdir: Optional[str], approval_note: Optional[str], clear_interrupt: bool,
  1126. metered: bool = True,
  1127. ) -> str:
  1128. """Execute in the foreground with retry on transient errors, then finalize. ``metered``
  1129. is False for Hermes' own control-plane commands (``_host_local``)."""
  1130. from hermes_cli.observability.shared_metrics_harness import record_terminal_outcome
  1131. max_retries = 3
  1132. env_type, eff, effective_timeout = plan.env_type, plan.effective_task_id, plan.effective_timeout
  1133. # Clean interrupt slate for an approved command, ONCE before the retry
  1134. # loop: drop a stale bit that landed during the approval-wait so it
  1135. # can't SIGINT the just-approved run. Do NOT re-clear inside the loop —
  1136. # a genuine interrupt during the backoff sleep must survive and abort
  1137. # the next attempt (rc 130).
  1138. if clear_interrupt:
  1139. from tools.interrupt import clear_current_thread_interrupt
  1140. clear_current_thread_interrupt()
  1141. for retry_count in range(max_retries + 1):
  1142. try:
  1143. command_cwd = _resolve_command_cwd(
  1144. workdir=workdir, default_cwd=plan.cwd, session_key=session_key, env_type=env_type,
  1145. mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,
  1146. env=env,
  1147. )
  1148. # bounded_capture: model-facing output keeps a head/tail window
  1149. # while streaming so a verbose command can't OOM the gateway;
  1150. # internal env.execute() consumers stay unbounded.
  1151. result = env.execute(
  1152. command, timeout=effective_timeout, cwd=command_cwd, bounded_capture=True,
  1153. **_yield_kwargs(command, env_type=env_type, cwd=command_cwd, effective_task_id=eff,
  1154. task_id=task_id, session_key=session_key),
  1155. )
  1156. break
  1157. except Exception as e:
  1158. # A backend exception (e.g. an SSH connect timeout) never reached an exit status, so it
  1159. # is not a terminal outcome; Hermes' own deadline arrives as ``hermes_timed_out``.
  1160. if "timeout" in str(e).lower():
  1161. return _error_json(f"Command timed out after {effective_timeout} seconds", exit_code=124)
  1162. # Retry on transient errors
  1163. if retry_count < max_retries:
  1164. wait_time = 2 ** (retry_count + 1)
  1165. logger.warning("Execution error, retrying in %ds (attempt %d/%d) - Command: %s - Error: %s: %s - Task: %s, Backend: %s",
  1166. wait_time, retry_count + 1, max_retries, _safe_command_preview(command), type(e).__name__, e, eff, env_type)
  1167. time.sleep(wait_time)
  1168. continue
  1169. logger.error("Execution failed after %d retries - Command: %s - Error: %s: %s - Task: %s, Backend: %s",
  1170. max_retries, _safe_command_preview(command), type(e).__name__, e, eff, env_type)
  1171. return _error_json(_redact_terminal_error_text(f"Command execution failed: {type(e).__name__}: {e}"))
  1172. if result.get("yielded_session_id"): # handed to the background: no exit status yet
  1173. return json.dumps({
  1174. "output": result.get("output", ""), "exit_code": None, "error": None,
  1175. "status": "yielded_to_background", "session_id": result["yielded_session_id"],
  1176. "pid": result.get("pid"), "notify_on_complete": True, "note": _YIELDED_NOTE,
  1177. }, ensure_ascii=False)
  1178. if metered:
  1179. record_terminal_outcome(command, env_type, result)
  1180. return finalize_foreground_result(
  1181. command=command, result=result, env=env, env_type=env_type, effective_task_id=eff,
  1182. task_id=task_id, session_id=session_id, session_key=session_key, workdir=workdir,
  1183. command_cwd=command_cwd, approval_note=approval_note,
  1184. )
  1185. # Floor for the pre-exec guard's share of the command deadline: a short command timeout
  1186. # (1s in tests, a few seconds in practice) must not turn the guard's own cold-start cost
  1187. # (module imports, git probes under load) into a refusal; the wedge it bounds lasted an hour.
  1188. _PRE_EXEC_GUARD_MIN_TIMEOUT_S = 30
  1189. def _pre_exec_block(
  1190. command: str, *, env: Any, env_type: str, cwd: str,
  1191. workdir: Optional[str], session_key: str,
  1192. ) -> None:
  1193. """Raise :class:`_Rejected` with the blocked-result JSON when the command must not run.
  1194. Order matters: gateway lifecycle first (protects the running gateway),
  1195. then the dangerous-workdir check, then the self-repo guard (local only).
  1196. """
  1197. blocked = gateway_lifecycle_block(
  1198. command=command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,
  1199. )
  1200. if blocked:
  1201. raise _Rejected(blocked)
  1202. if workdir:
  1203. workdir_error = _validate_workdir(workdir)
  1204. if workdir_error:
  1205. logger.warning("Blocked dangerous workdir: %s (command: %s)",
  1206. workdir[:200], _safe_command_preview(command))
  1207. raise _Rejected(_error_json(workdir_error, status="blocked"))
  1208. if env_type == "local":
  1209. blocked = (self_repo_block(command=command, cwd=cwd, workdir=workdir, session_key=session_key)
  1210. or locked_skill_block(command=command, cwd=cwd, workdir=workdir, session_key=session_key))
  1211. if blocked:
  1212. raise _Rejected(blocked)
  1213. _PTY_DISABLED_REASON = (
  1214. "PTY disabled for this command because it expects piped stdin/EOF "
  1215. "(for example gh auth login --with-token). For local background "
  1216. "processes, call process(action='close') after writing so it receives "
  1217. "EOF."
  1218. )
  1219. def _degraded_result(e: EnvironmentConnectionError, task_id: Optional[str]) -> str:
  1220. """Infrastructure failure (SSH host down, Docker daemon unreachable), distinct
  1221. from a nonzero exit. ``terminal.degraded_mode``: warn (default) returns a
  1222. structured degraded result with a retry hint; fail preserves the historical
  1223. error+traceback result."""
  1224. if _tenv("TERMINAL_DEGRADED_MODE", "warn").strip().lower() == "fail":
  1225. return _fatal_error_json(e)
  1226. logger.warning("terminal backend degraded: %s", e.reason)
  1227. # Evict the possibly-broken backend so the next call re-creates it.
  1228. with _quiet("degraded-env eviction failed"):
  1229. _evict_environment_for_task(task_id)
  1230. return json.dumps({
  1231. "output": "",
  1232. "exit_code": -1,
  1233. "status": "degraded",
  1234. "reason": e.reason,
  1235. "retry_hint": e.retry_hint,
  1236. "error": f"Terminal backend degraded: {e.reason}",
  1237. }, ensure_ascii=False)
  1238. def terminal_tool(
  1239. command: str,
  1240. background: bool = False,
  1241. timeout: Optional[int] = None,
  1242. task_id: Optional[str] = None,
  1243. session_id: Optional[str] = None,
  1244. force: bool = False,
  1245. workdir: Optional[str] = None,
  1246. pty: bool = False,
  1247. notify_on_complete: bool = False,
  1248. watch_patterns: Optional[list[str]] = None,
  1249. _host_local: bool = False,
  1250. _completion_output_chars: int = 0,
  1251. heartbeat: int = 0,
  1252. persist_on_release: bool = False,
  1253. ) -> str:
  1254. """Execute *command* in the configured terminal environment; returns a JSON string.
  1255. ``force`` (internal, not in the model schema) skips the dangerous-command
  1256. check after the user confirmed. ``workdir`` is per-command and never
  1257. recorded as the session cwd. ``pty`` applies to the local backend only.
  1258. ``notify_on_complete`` and ``watch_patterns`` are mutually exclusive
  1259. background-only flags: on conflict watch_patterns is dropped. watch_patterns
  1260. is hard rate-limited (1 notification / 15s / process) and auto-disabled
  1261. after repeated strikes or a lifetime cap, promoting to notify_on_complete —
  1262. use it only for rare one-shot signals on long-lived processes. ``heartbeat`` (seconds,
  1263. background-only, implies notify_on_complete) emits a "still running + output since last
  1264. time" event every N seconds so the agent stays current on a long job without polling.
  1265. ``persist_on_release`` (background-only) keeps the process alive across agent-lifecycle
  1266. cleanup — session end, context compression, error recovery, max-iteration stop — all of
  1267. which kill the task's background processes; the user can still stop it on purpose via
  1268. process_manage kill (#41225).
  1269. ``_completion_output_chars`` (internal) sizes the completion notification's output for a
  1270. spawner whose output is the payload (a bot DM's reply); 0 keeps the usual tail.
  1271. ``_host_local`` forces the local backend for Hermes-owned control-plane
  1272. children (kept in a separate env cache from the configured backend).
  1273. """
  1274. from hermes_cli.observability.shared_metrics_loop import record_terminal_backend as _metered
  1275. plan = None
  1276. try:
  1277. plan = _plan_execution(
  1278. command, task_id=task_id, timeout=timeout, background=background, _host_local=_host_local,
  1279. )
  1280. auto_bg = getattr(plan, "auto_background_command", None)
  1281. if auto_bg is not None:
  1282. # From here on this IS a background=true call for the stripped command: every guard
  1283. # below (lifecycle, workdir, self-repo, dangerous-command approval) sees what runs.
  1284. command, background = auto_bg, True
  1285. env = _acquire_env(plan, task_id)
  1286. # _lookup_active_env stamps whichever of the two keys holds the env.
  1287. with _call_in_flight(plan.effective_task_id, task_id):
  1288. env_type, cwd, effective_task_id = plan.env_type, plan.cwd, plan.effective_task_id
  1289. # Session key for cwd records: the contextvar doesn't cross tool-worker
  1290. # threads, so fall back to the raw task_id (the top-level agent's
  1291. # session_key) as a stable anchor.
  1292. from tools.approval import get_current_session_key
  1293. session_key = get_current_session_key(default="") or (task_id or "")
  1294. # The supervised-gateway identity probe ends in a kernel process query
  1295. # (psutil create_time) that has wedged for the better part of an hour on
  1296. # macOS; ``env.execute`` is already behind ``run_bounded_sync`` but this
  1297. # chain ran ahead of it, so the tool call never returned and the cron
  1298. # slot stayed occupied (#111922). Share the command's own deadline. A
  1299. # guard that never rendered a verdict fails CLOSED: these checks apply
  1300. # unconditionally (``force`` cannot bypass them), so the command is
  1301. # refused with a retryable error instead of running unguarded.
  1302. from agent.deadline import run_bounded_sync
  1303. from tools.interrupt import acting_for_tid
  1304. # The guard chain runs on the deadline worker; keep it answerable to /stop
  1305. # aimed at this tool thread (a remote-backend script read polls is_interrupted()).
  1306. guard_timeout = max(plan.effective_timeout, _PRE_EXEC_GUARD_MIN_TIMEOUT_S)
  1307. _acting_token = acting_for_tid.set(threading.current_thread().ident)
  1308. try:
  1309. bounded_guard = run_bounded_sync(
  1310. lambda: _pre_exec_block(
  1311. command, env=env, env_type=env_type, cwd=cwd, workdir=workdir, session_key=session_key,
  1312. ),
  1313. guard_timeout,
  1314. label="terminal.pre-exec-guard",
  1315. )
  1316. finally:
  1317. acting_for_tid.reset(_acting_token)
  1318. if bounded_guard.timed_out:
  1319. raise _Rejected(_error_json(
  1320. f"Terminal pre-execution guard did not finish within {guard_timeout}s "
  1321. "(process-identity probe wedged); the command was not run. Retry the call.",
  1322. status="error",
  1323. ))
  1324. # Pre-exec security checks (floors + dangerous command detection);
  1325. # force=True means the user already confirmed.
  1326. verdict = _run_approval_guards(command, env_type, plan.config, force=force)
  1327. pty_disabled = pty and _command_requires_pipe_stdin(command)
  1328. if plan.promoted_from_foreground_timeout is not None:
  1329. # Promotion implies notify_on_complete; watch_patterns is a background-only flag the
  1330. # caller could not have meant for a foreground call, and the two are exclusive anyway.
  1331. background, notify_on_complete, watch_patterns = True, True, None
  1332. if background:
  1333. result = spawn_background_process(
  1334. command=command, env=env, env_type=env_type, effective_task_id=effective_task_id,
  1335. task_id=task_id, session_key=session_key, workdir=workdir, cwd=cwd,
  1336. mounted_host=getattr(env, "host_cwd", None) or plan.host_cwd,
  1337. effective_pty=pty and not pty_disabled, notify_on_complete=notify_on_complete,
  1338. watch_patterns=watch_patterns, approval_note=verdict.note,
  1339. pty_disabled_reason=_PTY_DISABLED_REASON if pty_disabled else None,
  1340. completion_output_chars=_completion_output_chars,
  1341. heartbeat_seconds=heartbeat,
  1342. persist_on_release=persist_on_release,
  1343. auto_backgrounded=auto_bg is not None and plan.promoted_from_foreground_timeout is None,
  1344. )
  1345. if plan.promoted_from_foreground_timeout is not None:
  1346. result = _with_promoted_note(result, plan.promoted_from_foreground_timeout)
  1347. return _metered(None if _host_local else plan, result)
  1348. return _metered(None if _host_local else plan, _run_foreground(
  1349. command, env, plan,
  1350. task_id=task_id, session_id=session_id, session_key=session_key,
  1351. workdir=workdir, approval_note=verdict.note, clear_interrupt=verdict.approved_run,
  1352. metered=not _host_local,
  1353. ))
  1354. except _Rejected as r:
  1355. return r.result_json
  1356. except EnvironmentConnectionError as e:
  1357. return _metered(None if _host_local else plan, _degraded_result(e, task_id), error_class="tool_error")
  1358. except Exception as e:
  1359. return _metered(None if _host_local else plan, _fatal_error_json(e), error_class="exception")
  1360. def check_terminal_requirements() -> bool:
  1361. """Check if all requirements for the terminal tool are met. The reason for a failure is kept for
  1362. :func:`terminal_backend_unavailable_reason` (CLI startup notice / doctor)."""
  1363. try:
  1364. config = _get_env_config()
  1365. checker = _REQUIREMENT_CHECKERS.get(config["env_type"], _check_plugin_requirements)
  1366. return checker(config)
  1367. except Exception as e:
  1368. logger.error("Terminal requirements check failed: %s", e, exc_info=True)
  1369. _record_unavailable_reason(f"the requirements check failed: {e}")
  1370. return False
  1371. from tools.registry import registry
  1372. TERMINAL_SCHEMA = {
  1373. "name": "terminal",
  1374. "description": TERMINAL_TOOL_DESCRIPTION,
  1375. "parameters": {
  1376. "type": "object",
  1377. "properties": {
  1378. "command": {
  1379. "type": "string",
  1380. "description": "The shell command to execute"
  1381. },
  1382. "background": {
  1383. "type": "boolean",
  1384. "description": "Keep running after this call returns; returns a session_id.",
  1385. "default": False
  1386. },
  1387. "timeout": {
  1388. "type": "integer",
  1389. "description": f"Max seconds (default 180, foreground max {FOREGROUND_MAX_TIMEOUT}). Finishing early returns at once, so set it generously. Above {FOREGROUND_MAX_TIMEOUT}s the command runs as a tracked background process with notify_on_complete=true (the result says so; do not re-run it).",
  1390. "minimum": 1
  1391. },
  1392. "workdir": {
  1393. "type": "string",
  1394. "description": "Absolute working directory for this command only."
  1395. },
  1396. "pty": {
  1397. "type": "boolean",
  1398. "description": "With background=true: pseudo-terminal for interactive CLIs (REPLs, coding agents); drive it with process_manage write/submit. Local backend only.",
  1399. "default": False
  1400. },
  1401. "notify": {
  1402. "description": "With background=true. true: one notification on exit; use it for anything that ends (builds, tests, deploys), or the job runs silently. ['pattern', ...]: notify when a line matches, only for one-shot readiness on processes that never exit; rate-limited. Omit for servers/daemons.",
  1403. "anyOf": [
  1404. {"type": "boolean"},
  1405. {"type": "array", "items": {"type": "string"}}
  1406. ]
  1407. },
  1408. "heartbeat": {
  1409. "type": "integer",
  1410. "minimum": 0,
  1411. "default": 0,
  1412. "description": "With background=true: also notify every N seconds (min 60) with new output; quiet ticks are skipped. For bounded jobs you must react to mid-run, never servers. Implies notify=true; 0 = off.",
  1413. },
  1414. "persist_on_release": {
  1415. "type": "boolean",
  1416. "default": False,
  1417. "description": "With background=true: survive session end, /new, compression and other cleanup (not host exit). Only for jobs the user explicitly wants to outlive the conversation."
  1418. }
  1419. # Legacy aliases (unadvertised, still accepted): notify_on_complete
  1420. # (bool) and watch_patterns (list). notify=true|[...] maps onto
  1421. # them in the dispatch wrapper; explicit notify wins on conflict.
  1422. },
  1423. "required": ["command"]
  1424. }
  1425. }
  1426. def _handle_terminal(args, **kw):
  1427. from agent.terminal_approval_batch import validate_prepared_terminal
  1428. validate_prepared_terminal(args)
  1429. # Models sometimes send execute_code's ``code`` here; name the stray
  1430. # argument and the right tool instead of failing on command=None.
  1431. if "command" not in args and "code" in args:
  1432. return tool_error(
  1433. "terminal received a 'code' parameter, but it requires a shell "
  1434. "command in 'command'. Use execute_code(code=...) for Python; "
  1435. "for shell, retry as terminal(command=...)."
  1436. )
  1437. # `notify` is the advertised interface (true → notify_on_complete,
  1438. # [...] → watch_patterns); the legacy args stay accepted, explicit
  1439. # `notify` wins. Notification INTENT on a foreground call is refused with
  1440. # the corrected call. A foreground `heartbeat` carries no intent: it has
  1441. # no execution meaning without a tracked process, and providers that
  1442. # materialize every schema property (or models copying a background call
  1443. # shape) send `heartbeat=60` on ordinary commands — refusing it produced
  1444. # identical-call retry loops (46 subagent sessions, 4 refusals each, 38 of
  1445. # which never ran a single command), so it is normalized away instead.
  1446. notify = args.get("notify")
  1447. notify_on_complete = args.get("notify_on_complete", False)
  1448. watch_patterns = args.get("watch_patterns")
  1449. heartbeat = args.get("heartbeat") or 0
  1450. persist_on_release = bool(args.get("persist_on_release", False))
  1451. if not isinstance(heartbeat, int) or isinstance(heartbeat, bool) or heartbeat < 0:
  1452. return tool_error("heartbeat must be a whole number of seconds (0 disables; positive values are clamped to min 60).")
  1453. if not args.get("background", False):
  1454. if notify or watch_patterns or notify_on_complete:
  1455. return tool_error(
  1456. "notify only applies to background commands (foreground "
  1457. "results return directly). Either drop it, or run as "
  1458. "terminal(command=..., background=true, notify=...)."
  1459. )
  1460. heartbeat = 0
  1461. if args.get("pty", False):
  1462. return tool_error(
  1463. "pty requires background=true (a PTY session is interacted "
  1464. "with via process(action='write'/'submit'), which needs a "
  1465. "tracked background process). Retry as terminal(command=..., "
  1466. "background=true, pty=true)."
  1467. )
  1468. if persist_on_release:
  1469. return tool_error(
  1470. "persist_on_release only applies to background commands (a foreground "
  1471. "process is awaited inline and has nothing to persist). Retry as "
  1472. "terminal(command=..., background=true, persist_on_release=true)."
  1473. )
  1474. if notify is not None:
  1475. if isinstance(notify, bool):
  1476. notify_on_complete = notify
  1477. watch_patterns = None
  1478. elif isinstance(notify, list):
  1479. watch_patterns = notify
  1480. notify_on_complete = False
  1481. else:
  1482. return tool_error(
  1483. "notify must be true/false (notify on exit) or a list of "
  1484. "strings (notify on output pattern match)."
  1485. )
  1486. if heartbeat:
  1487. notify_on_complete = True # the heartbeat rides the completion delivery path
  1488. return terminal_tool(
  1489. command=args.get("command"),
  1490. background=args.get("background", False),
  1491. timeout=args.get("timeout"),
  1492. task_id=kw.get("task_id"),
  1493. session_id=kw.get("session_id"),
  1494. workdir=args.get("workdir"),
  1495. pty=args.get("pty", False),
  1496. notify_on_complete=notify_on_complete,
  1497. watch_patterns=watch_patterns,
  1498. heartbeat=heartbeat,
  1499. persist_on_release=persist_on_release,
  1500. )
  1501. registry.register(
  1502. name="terminal",
  1503. toolset="terminal",
  1504. schema=TERMINAL_SCHEMA,
  1505. handler=_handle_terminal,
  1506. check_fn=check_terminal_requirements,
  1507. emoji="💻",
  1508. max_result_size_chars=100_000,
  1509. )