| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394 |
- """Docker execution environment for sandboxed command execution.
- Security hardened (cap-drop ALL, no-new-privileges, PID limits), configurable
- resource limits (CPU, memory, disk), and optional filesystem persistence via
- bind mounts.
- """
- import datetime
- import hashlib
- import json
- import logging
- import os
- import re
- import shlex
- import shutil
- import subprocess
- import sys
- import tempfile
- import threading
- import time
- import uuid
- from pathlib import Path
- from typing import Optional
- from hermes_constants import get_hermes_home
- from tools.environments.base import BaseEnvironment, EnvironmentConnectionError, _SHELL_ENV_NAME_RE, get_sandbox_dir
- from tools.terminal_tool_config import (
- _host_path_key, _is_windows_drive_path, cwd_follows_host_mount, home_profile_name,
- )
- from tools.environments.base_output import _popen_bash
- from tools.environments.docker_egress import (
- _EGRESS_LABEL_KEY, _critical_egress_env_names, _egress_enforce_on_docker, _egress_proxy_args_for_docker,
- _egress_reuse_fingerprint, check_docker_env_collisions, check_extra_args_collisions,
- check_forward_env_collisions, merge_egress_env,
- )
- from tools.environments.docker_ownership import (
- hand_back_sandbox_files, reclaim_sandbox_files, strip_sandbox_privileges)
- from tools.environments.path_utils import sanitize_task_id_for_path
- from tools.environments.remote_common import (
- bash_argv, client_env_with, load_hermes_env_vars, prepend_unset, resolve_passthrough_env, run_capture)
- logger = logging.getLogger(__name__)
- # Docker Desktop install paths checked when 'docker' is not in PATH
- # (macOS Intel / Apple Silicon Homebrew / app bundle).
- _DOCKER_SEARCH_PATHS = [
- "/usr/local/bin/docker", "/opt/homebrew/bin/docker", "/Applications/Docker.app/Contents/Resources/bin/docker",
- ]
- _docker_executable: Optional[str] = None # resolved once, cached
- _ENV_VAR_NAME_RE = _SHELL_ENV_NAME_RE
- _ENVIRONMENT_LABEL_KEY = "hermes-environment"
- # Stamped at create time on per-workspace persistent containers (task key ``<base>:ws:<digest>``)
- # with the digest as its value. The orphan reaper reads it to spare a container the idle reaper
- # stopped on purpose; ``hermes-task-id`` cannot serve, it is truncated to 63 chars.
- _WORKSPACE_LABEL_KEY = "hermes-workspace"
- _WORKSPACE_TASK_RE = re.compile(r":ws:([0-9a-f]{16})$")
- def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]:
- """Return a deduplicated list of valid environment variable names."""
- normalized: list[str] = []
- for item in forward_env or []:
- if not isinstance(item, str):
- logger.warning("Ignoring non-string docker_forward_env entry: %r", item)
- continue
- key = item.strip()
- if not key:
- continue
- if not _ENV_VAR_NAME_RE.match(key):
- logger.warning("Ignoring invalid docker_forward_env entry: %r", item)
- elif key not in normalized:
- normalized.append(key)
- return normalized
- def _normalize_env_dict(env: dict | None) -> dict[str, str]:
- """Validate a docker_env dict to {str: str}; scalars are coerced, other values dropped."""
- if not env:
- return {}
- if not isinstance(env, dict):
- logger.warning("docker_env is not a dict: %s", type(env).__name__)
- return {}
- normalized: dict[str, str] = {}
- for key, value in env.items():
- if not isinstance(key, str) or not _ENV_VAR_NAME_RE.match(key.strip()):
- logger.warning("Ignoring invalid docker_env key: %r", key)
- continue
- if not isinstance(value, (str, int, float, bool)):
- logger.warning("Ignoring non-string docker_env value for %r: %s", key.strip(), type(value).__name__)
- continue
- normalized[key.strip()] = value if isinstance(value, str) else str(value)
- return normalized
- # Module-level binding: tests patch ``docker._load_hermes_env_vars`` to fake the .env file.
- _load_hermes_env_vars = load_hermes_env_vars
- # Docker label values must match [a-zA-Z0-9_.-] and stay <=63 chars to round-trip
- # through `docker ps --filter label=key=value`.
- _LABEL_VALUE_OK_RE = re.compile(r"[^A-Za-z0-9_.-]")
- def _sanitize_label_value(value: str) -> str:
- """Lossy coercion into a Docker label-safe form; empty/invalid input becomes ``"unknown"``."""
- if not isinstance(value, str) or not value:
- return "unknown"
- return _LABEL_VALUE_OK_RE.sub("_", value)[:63] or "unknown"
- # task_id -> host directory name; shared with every backend persisting per-task
- # state on the host (the singularity plugin's overlays) so the mapping is fixed in one place.
- _sandbox_dir_name = sanitize_task_id_for_path
- def _get_active_profile_name() -> str:
- """Active Hermes profile name (the container key's own lookup), or ``"default"`` on error. Resolved
- at container-create time so a container stays tagged with its creator even if the process switches
- profiles."""
- return home_profile_name()
- def _container_identity(shared_key: str = "") -> str:
- """Profile label used for container reuse and orphan reaping. Profiles are isolated by default; an
- explicit shared key lets trusted profiles share one Docker identity. Label sanitization is lossy
- and reuse is label-keyed, so a digest of the raw key disambiguates colliding keys. Plain profile
- names keep their historical un-suffixed labels."""
- if not shared_key:
- return _sanitize_label_value(_get_active_profile_name())
- digest = hashlib.sha256(shared_key.encode("utf-8")).hexdigest()[:12]
- return f"{_sanitize_label_value(shared_key)[:50]}-{digest}"
- def _is_volatile_mount_spec(spec: str) -> bool:
- """True when *spec* is a ``host:container[:mode]`` mount whose host source is a
- per-process tempdir (a ``mkdtemp`` under the system temp), so its path is random
- per process and must not be hashed into the reuse label.
- The known source is the symlink-safe skills copy from
- ``credential_files._safe_skills_path``: any symlink under ``skills/`` makes it a
- fresh ``mkdtemp`` per process. Stable host paths — including a symlink-free
- skills dir, which mounts directly — never sit under the process tempdir.
- """
- if spec in ("-v", "--mount") or ":" not in spec:
- return False # the flag element itself, or a non-bind arg (tmpfs modes etc.)
- parsed = _split_volume_spec(spec)
- source = parsed[0] if parsed is not None else spec.split(":", 1)[0]
- if _is_windows_drive_path(source):
- return False # drive-letter hosts can never be the POSIX process tempdir
- try:
- temp_root = os.path.realpath(tempfile.gettempdir())
- source_abs = os.path.realpath(os.path.abspath(os.path.expanduser(source)))
- return source_abs == temp_root or source_abs.startswith(temp_root + os.sep)
- except OSError: # unreadable source — treat as stable, fail the safe way
- return False
- def _reuse_environment_fingerprint(*, image: str, mount_args: list[str], hermes_home: str) -> str:
- """Hash immutable configuration so reuse cannot silently attach to stale mounts.
- Hash requested values rather than exposing profile paths and volume sources in labels.
- Keep mount order: later arguments can override earlier mount destinations.
- Per-process tempdir-sourced mounts (the symlink-safe skills copy) have their volatile
- host path replaced by a stable placeholder: the path is random per process, so hashing
- it made the label differ across processes and cross-process container reuse never
- matched for users with any symlink under ``skills/``. The container path stays in the
- hash, so moving where that mount lands still forces a fresh container.
- """
- normalized_home = os.path.normcase(os.path.abspath(os.path.expanduser(hermes_home)))
- canonical_mounts = [
- (f"<volatile-tempdir-mount>:{spec.split(':', 1)[1]}"
- if _is_volatile_mount_spec(spec) else spec)
- for spec in mount_args]
- payload = json.dumps(
- {"image": image, "mount_args": canonical_mounts, "hermes_home": normalized_home},
- sort_keys=True, separators=(",", ":"))
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
- def reap_orphan_containers(
- *, max_age_seconds: int = 600, profile_filter: str | None = None, docker_exe: str | None = None,
- ) -> int:
- """Remove stale hermes-tagged containers left behind by prior processes (SIGKILL/OOM
- exits that bypass atexit). Only ``status=exited`` containers (running ones may belong
- to a sibling process), only the caller's profile, and only if ``FinishedAt`` is older
- than *max_age_seconds* (a just-exited sibling may be about to reuse its container).
- Best-effort and idempotent: failures log at debug and the count removed so far is returned.
- """
- docker = docker_exe or find_docker() or "docker"
- filters = ["--filter", "label=hermes-agent=1", "--filter", "status=exited"]
- if profile_filter:
- filters.extend(["--filter", f"label=hermes-profile={_sanitize_label_value(profile_filter)}"])
- listing = _docker_query(
- [docker, "ps", "-a", *filters, "--format", "{{.ID}}"], timeout=15,
- fail="orphan reaper docker ps failed: %s", nonzero="orphan reaper docker ps returned %d: %s")
- if listing is None:
- return 0
- # Per-container inspect keeps the failure blast radius to one container.
- now = datetime.datetime.now(datetime.UTC)
- removed = 0
- for cid in (ln.strip() for ln in listing.stdout.splitlines() if ln.strip()):
- finished_at = _container_finished_at(docker, cid)
- if finished_at is None: # unknown age — be conservative
- continue
- age = (now - finished_at).total_seconds()
- if age < max_age_seconds:
- continue
- strip_sandbox_privileges(docker, cid) # its teardown never ran; owners stay (it may restart)
- if _is_idle_workspace_container(docker, cid):
- continue
- # No -f: a sibling may have restarted the container between the ps snapshot
- # and now (FinishedAt still reports the previous exit), and the daemon refuses
- # a plain rm on a running container, which is the atomic recheck this sweep needs.
- result = _docker_query(
- [docker, "rm", cid], timeout=30, fail="orphan reaper docker rm %s failed: %s", fail_args=(cid[:12],))
- if result is None:
- continue
- if result.returncode == 0:
- removed += 1
- logger.info("Reaped orphan container %s (exited %d seconds ago)", cid[:12], int(age))
- else:
- logger.debug("docker rm %s failed: %s", cid[:12], result.stderr.strip())
- return removed
- def _is_idle_workspace_container(docker_exe: str, container_id: str) -> bool:
- """A persistent per-workspace container (task key ``<base>:ws:<digest>``, stamped with the
- ``hermes-workspace=<digest>`` label at create time) that the idle reaper STOPPED on purpose
- (``DockerEnvironment.stop_idle``): it is that workspace's sandbox, restarted on next use, not an
- orphan. Reads the dedicated label, never the 63-char-truncated ``hermes-task-id``, whose
- ``_ws_<digest>`` suffix a long profile name cuts. Unknown (inspect failed) reads as not one."""
- result = _docker_query(
- [docker_exe, "inspect", "--format", f'{{{{index .Config.Labels "{_WORKSPACE_LABEL_KEY}"}}}}', container_id],
- timeout=10, fail="orphan reaper docker inspect %s failed: %s", fail_args=(container_id[:12],))
- if result is None or result.returncode != 0:
- return False
- return re.fullmatch(r"[0-9a-f]{16}", result.stdout.strip()) is not None
- def _container_finished_at(docker_exe: str, container_id: str):
- """``docker inspect`` FinishedAt as an aware datetime; ``None`` (= don't reap) when
- missing/unparseable or Docker's never-finished zero value ``0001-01-01T00:00:00Z``."""
- result = _docker_query(
- [docker_exe, "inspect", "--format", "{{.State.FinishedAt}}", container_id], timeout=10,
- fail="orphan reaper docker inspect %s failed: %s", fail_args=(container_id[:12],))
- if result is None or result.returncode != 0:
- return None
- raw = result.stdout.strip()
- if not raw or raw.startswith("0001-01-01"):
- return None
- # Docker emits RFC3339 with nanoseconds; fromisoformat only takes microseconds.
- raw = re.sub(r"(\.\d{6})\d+", r"\1", raw).replace("Z", "+00:00")
- try:
- return datetime.datetime.fromisoformat(raw)
- except ValueError as e:
- logger.debug("could not parse FinishedAt %r for %s: %s", raw, container_id[:12], e)
- return None
- def _is_executable(path: str) -> bool:
- return os.path.isfile(path) and os.access(path, os.X_OK)
- def _docker_query(
- argv: list[str], *, timeout: float, fail: str, fail_args: tuple = (), nonzero: str | None = None,
- exc_types: tuple = (subprocess.TimeoutExpired, OSError),
- ):
- """Best-effort ``run_capture`` for probes: logs *fail* (``*fail_args, exc``) at debug and
- returns ``None`` when the CLI raises; with *nonzero* set, a nonzero exit also logs
- (``*fail_args, returncode, stderr``) and returns ``None``."""
- try:
- result = run_capture(argv, timeout=timeout)
- except exc_types as e:
- logger.debug(fail, *fail_args, e)
- return None
- if nonzero is not None and result.returncode != 0:
- logger.debug(nonzero, *fail_args, result.returncode, result.stderr.strip())
- return None
- return result
- def find_docker() -> Optional[str]:
- """Locate the docker/podman CLI (cached): ``HERMES_DOCKER_BINARY`` override, ``docker``
- on PATH, ``podman`` on PATH, then macOS Docker Desktop locations; ``None`` if absent."""
- global _docker_executable
- if _docker_executable is not None:
- return _docker_executable
- override = os.getenv("HERMES_DOCKER_BINARY")
- if override and _is_executable(override):
- logger.info("Using HERMES_DOCKER_BINARY override: %s", override)
- found = override
- elif found := shutil.which("docker"):
- pass
- elif found := shutil.which("podman"):
- logger.info("Using podman as container runtime: %s", found)
- elif found := next((p for p in _DOCKER_SEARCH_PATHS if _is_executable(p)), None):
- logger.info("Found docker at non-PATH location: %s", found)
- if found:
- _docker_executable = found
- return found
- def docker_runtime_name(executable: str) -> str:
- """User-facing runtime name (``"Podman"`` / ``"Docker"``) for the CLI at *executable*, so
- diagnostics and pickers name the runtime actually in use."""
- return "Podman" if "podman" in os.path.basename(executable).lower() else "Docker"
- def docker_runtime_start_hint(executable: str) -> str:
- """How to bring the runtime at *executable* back up, for a "not reachable" message. Docker has
- a daemon to start; Podman is daemonless (outside Linux it runs inside a VM)."""
- if docker_runtime_name(executable) != "Podman":
- return "start Docker and retry"
- return "run `podman machine start` and retry"
- # Security flags applied to every container. The container is the security
- # boundary; all caps are dropped and the minimum added back:
- # DAC_OVERRIDE - root can write to bind-mounted dirs owned by the host user
- # CHOWN/FOWNER - package managers need to set file ownership
- # /tmp is size-limited and nosuid but allows exec (pip/npm builds need it).
- # ``--pids-limit``/``--cpus``/``--memory`` live in resource_args instead, gated
- # on ``_cgroup_limits_available`` (unprivileged LXCs lack the controllers).
- _BASE_SECURITY_ARGS = [
- "--cap-drop", "ALL",
- "--cap-add", "DAC_OVERRIDE",
- "--cap-add", "CHOWN",
- "--cap-add", "FOWNER",
- "--tmpfs", "/tmp:rw,nosuid,size=512m", # no-tmp: ok — container tmpfs mount spec
- "--tmpfs", "/var/tmp:rw,noexec,nosuid,size=256m"]
- # Fork-bomb guard, applied only when the pids cgroup controller is available. The pids cgroup counts
- # THREADS, and a sandbox that hosts the Bot Screen runs a desktop in here: measured on
- # hermes-sandbox:desktop, the idle container is 2 tasks, Xvnc + Xfce + dbus 44, one Chromium with one
- # tab 212, plus the agent's own Chromium with two tabs 488. The old 256 was hit in normal
- # use and every further `docker exec` (browser command, cua-driver, thumbnail, CDP forward) died with
- # runc's "procReady not received". 2048 leaves room for a working browser and is still three orders of
- # magnitude under the host's pid_max.
- _DEFAULT_PIDS_LIMIT = "2048"
- # Docker's 64 MB /dev/shm default crashes Chromium/Playwright tabs and PyTorch
- # DataLoader workers. tmpfs is lazily allocated so a 1g ceiling costs nothing
- # until used (and still counts against --memory). Empty/"0" config omits the flag.
- # Docker's built-in default is a tiny 64 MB, which silently breaks shared-memory-hungry workloads inside the
- # sandbox: Chromium / Playwright renderers crash tabs, and PyTorch DataLoader workers die with "bus error" /
- # "insufficient shared memory" once they exceed it. tmpfs is lazily allocated, so a 1g ceiling costs nothing
- # until actually used (and usage still counts against the container's --memory cgroup limit). Configurable
- # via ``terminal.docker_shm_size`` in config.yaml; an empty value (or "0") omits the flag and falls back to
- # Docker's 64 MB default. Ported from nanocoai/nanoclaw#2748.
- _DEFAULT_SHM_SIZE = "1g"
- def _extra_args_set_shm_size(extra_args: list) -> bool:
- """True when docker_extra_args already set ``--shm-size`` (then our default is skipped)."""
- return any(
- isinstance(a, str) and (a == "--shm-size" or a.startswith("--shm-size="))
- for a in (extra_args or []))
- _NETWORK_FLAGS = ("--network", "--net")
- def _extra_args_network_mode(extra_args: list) -> Optional[str]:
- """Network mode requested by ``docker_extra_args`` (``--network none`` / ``--network=none`` /
- ``--net``), or None when the operator set no network flag. Docker rejects a repeated
- ``--network`` outright (exit 125), so the implicit ``docker_network: false`` flag and an
- operator-supplied one must be reconciled before the command line is built."""
- args = [a for a in (extra_args or []) if isinstance(a, str)]
- for i, arg in enumerate(args):
- for flag in _NETWORK_FLAGS:
- if arg == flag:
- return args[i + 1] if i + 1 < len(args) else ""
- if arg.startswith(f"{flag}="):
- return arg.split("=", 1)[1]
- return None
- # /run is separate from _BASE_SECURITY_ARGS: s6-overlay images exec
- # /run/s6/basedir/bin/init at stage0 and die (exit 126) on a noexec mount.
- _RUN_TMPFS_NOEXEC = "--tmpfs", "/run:rw,noexec,nosuid,size=64m"
- _RUN_TMPFS_EXEC = "--tmpfs", "/run:rw,exec,nosuid,size=64m"
- # SETUID/SETGID let a root-started init drop privileges (s6-setuidgid, gosu,
- # su). Combined with no-new-privileges the dropped process cannot escalate
- # back. Skipped when --user is passed: the container already starts unprivileged.
- _PRIVDROP_CAP_ARGS = ["--cap-add", "SETUID", "--cap-add", "SETGID"]
- # Every ``cleanup()`` worker, independent of terminal_tool's active-env registry (see
- # ``wait_for_all_teardowns``).
- _TEARDOWN_THREADS: set[threading.Thread] = set()
- _TEARDOWN_LOCK = threading.Lock()
- _S6_INIT_ENTRYPOINTS = ("/init", "/package/admin/s6-overlay/command/init")
- _NO_NEW_PRIVILEGES_ARGS = ["--security-opt", "no-new-privileges"]
- def _build_security_args(run_as_host_user: bool, run_exec: bool = False, snap_compat: bool = False) -> list[str]:
- """Security/cap/tmpfs args for the privilege mode; ``run_exec`` mounts /run exec for s6 images.
- ``snap_compat`` drops no-new-privileges: snap-packaged Docker's AppArmor profile turns it into
- "exec: operation not permitted" for every process in the container (#9730, LP#1908448)."""
- args = list(_BASE_SECURITY_ARGS) + ([] if snap_compat else list(_NO_NEW_PRIVILEGES_ARGS))
- args += list(_RUN_TMPFS_EXEC if run_exec else _RUN_TMPFS_NOEXEC)
- return args if run_as_host_user else args + list(_PRIVDROP_CAP_ARGS)
- def _image_uses_init_entrypoint(docker_exe: str, image: str) -> bool:
- """True if ``image``'s entrypoint is the s6-overlay ``/init`` (own PID 1: incompatible
- with ``--init`` and a noexec /run). Any inspection failure, including an image not yet
- pulled, returns False and keeps the hardened defaults."""
- result = _docker_query(
- [docker_exe, "image", "inspect", image, "--format", "{{json .Config.Entrypoint}}"], timeout=15,
- fail="Docker: could not inspect entrypoint for %s: %s", fail_args=(image,),
- nonzero="Docker: image inspect for %s returned %d (stderr=%s)",
- exc_types=(subprocess.SubprocessError, OSError))
- raw = (result.stdout or "").strip() if result is not None else ""
- if not raw or raw == "null":
- return False
- try:
- entrypoint = json.loads(raw)
- except (ValueError, TypeError):
- return False
- if isinstance(entrypoint, str):
- entrypoint = [entrypoint]
- if not isinstance(entrypoint, list) or not entrypoint:
- return False
- return str(entrypoint[0]).strip() in _S6_INIT_ENTRYPOINTS
- def _resolve_host_user_spec() -> Optional[str]:
- """``<uid>:<gid>`` of the host user, or ``None`` without POSIX ids. Uses os.getuid/getgid
- directly (not pwd/getpass) so nameless UIDs inside sandboxed launchers never raise."""
- get_uid = getattr(os, "getuid", None)
- get_gid = getattr(os, "getgid", None)
- if get_uid is None or get_gid is None:
- return None
- try:
- return f"{get_uid()}:{get_gid()}"
- except Exception: # pragma: no cover - defensive
- return None
- _storage_opt_ok: Optional[bool] = None # cached result across instances
- _cgroup_limits_ok: Optional[bool] = None # cached result across instances
- def _cgroup_limits_available(image: str) -> bool:
- """Probe once per process whether ``--cpus``/``--memory``/``--pids-limit`` work here, via a
- throwaway ``sleep 0`` container from *image* (no extra pull). Without delegated cgroup
- controllers (unprivileged LXCs, rootless) these flags fail every start with exit 126;
- the result is host-wide, so it is cached. Only DEFINITIVE answers are cached: a probe
- that could not run (auto-pull past the timeout, daemon cold-start, manifest/pull error)
- says nothing about cgroup support, so it degrades this spawn and is retried on the next."""
- global _cgroup_limits_ok
- if _cgroup_limits_ok is not None:
- return _cgroup_limits_ok
- docker_exe = find_docker()
- if not docker_exe or not image:
- return False # not cached: docker may appear later in this process
- try:
- result = run_capture(
- [docker_exe, "run", "--rm", "--cpus", "0.5", "--memory", "64m", "--pids-limit", "32",
- image, "sleep", "0"],
- timeout=60)
- except Exception as e:
- logger.warning("Cgroup limit probe failed; containers run without "
- "CPU/memory/PID limits until a probe succeeds: %s", e)
- return False
- if result.returncode == 0:
- _cgroup_limits_ok = True
- return True
- stderr = (result.stderr or "").strip()
- if "cgroup" not in stderr.lower():
- # Pull/manifest/daemon errors say nothing about cgroup support: not cached.
- logger.warning(
- "Cgroup limit probe could not determine support (docker exited %d: %s). "
- "Containers run without CPU/memory/PID limits until a probe succeeds.",
- result.returncode, stderr[:500])
- return False
- _cgroup_limits_ok = False
- logger.warning(
- "Cgroup resource limits (--cpus/--memory/--pids-limit) not "
- "available in this environment. Containers will run without "
- "CPU, memory or PID limits. To enable, delegate the cpu, "
- "memory and pids cgroup controllers to this container. Probe stderr: %s",
- stderr[:500])
- return False
- def _docker_unavailable(log_msg: str, *log_args, error: str, hint: str, exc_info: bool = False):
- logger.error(log_msg, *log_args, exc_info=exc_info)
- return EnvironmentConnectionError(error, retry_hint=hint)
- def _ensure_docker_available() -> None:
- """Fail fast with an ``EnvironmentConnectionError`` when the docker CLI/daemon is unusable."""
- docker_exe = find_docker()
- if not docker_exe:
- raise _docker_unavailable(
- "Docker backend selected but no docker executable was found in PATH "
- "or known install locations. Install Docker Desktop and ensure the CLI is available.",
- error="Docker executable not found in PATH or known install locations. "
- "Install Docker and ensure the 'docker' command is available.",
- hint="Install Docker (or fix PATH) and retry, or run `hermes setup terminal` to switch to Local.")
- try:
- result = run_capture([docker_exe, "version"], timeout=5)
- except FileNotFoundError:
- raise _docker_unavailable(
- "Docker backend selected but the resolved docker executable '%s' could not be executed.",
- docker_exe, exc_info=True,
- error="Docker executable could not be executed. Check your Docker installation.",
- hint="Repair the Docker installation and retry.")
- except subprocess.TimeoutExpired:
- raise _docker_unavailable(
- "Docker backend selected but '%s version' timed out. The Docker daemon may not be running.",
- docker_exe, exc_info=True,
- error="Docker daemon is not responding. Ensure Docker is running and try again.",
- hint="Start Docker (e.g. `systemctl start docker` or launch Docker Desktop), then retry — "
- "or run `hermes setup terminal` to switch to Local.")
- except Exception:
- logger.error("Unexpected error while checking Docker availability.", exc_info=True)
- raise
- if result.returncode != 0:
- raise _docker_unavailable(
- "Docker backend selected but '%s version' failed (exit code %d, stderr=%s)",
- docker_exe, result.returncode, result.stderr.strip(),
- error="Docker command is available but 'docker version' failed. Check your Docker installation.",
- hint="Start Docker, or add your user to the docker group, then retry — "
- "or run `hermes setup terminal` to switch to Local.")
- def _name_only_env_args(names) -> list[str]:
- """``-e KEY`` flags (no values): the docker CLI resolves values from its own env,
- so secrets live in owner-readable /proc/*/environ instead of world-readable cmdline."""
- return [arg for key in sorted(names) for arg in ("-e", key)]
- # Mount kinds declared by skills/credential_files: (getter, expects_file, log noun).
- _RO_MOUNT_SOURCES = (
- ("get_credential_file_mounts", True, "credential"),
- ("get_skills_directory_mount", False, "skills dir"),
- ("get_cache_directory_mounts", False, "cache dir"))
- def _readonly_skill_mount_args() -> list[str]:
- """``-v host:container:ro`` args for credential files, skill dirs and cache dirs. Read-only so the
- container can authenticate/read but never modify host state. Missing or wrong-kind sources are
- skipped with a warning (Docker-in-Docker auto-creates a missing file source as a directory,
- which would exit 125)."""
- args: list[str] = []
- try:
- import tools.credential_files as cf
- for getter, expects_file, noun in _RO_MOUNT_SOURCES:
- for entry in getattr(cf, getter)():
- src = Path(entry["host_path"])
- if expects_file:
- problem = ("source is a directory (likely Docker-in-Docker auto-creation)" if src.is_dir()
- else None if src.is_file() else "source not found")
- else:
- problem = None if src.is_dir() else "source is not a directory"
- if problem:
- logger.warning("Docker: skipping %s mount — %s: %s", noun.split()[0], problem, src)
- continue
- args.extend(["-v", f"{entry['host_path']}:{entry['container_path']}:ro"])
- logger.info("Docker: mounting %s %s -> %s", noun, entry["host_path"], entry["container_path"])
- except Exception as e:
- logger.debug("Docker: could not load credential file mounts: %s", e)
- return args
- def _host_user_args(run_as_host_user: bool) -> list[str]:
- """``--user uid:gid`` so bind-mount writes are owned by the host user, not root. Without
- POSIX ids fall back to the full cap set — the image's init may still need to drop privileges."""
- if not run_as_host_user:
- return []
- user_spec = _resolve_host_user_spec()
- if user_spec is not None:
- logger.info("Docker: running container as host user %s", user_spec)
- return ["--user", user_spec]
- logger.warning(
- "docker_run_as_host_user is enabled but this platform does "
- "not expose POSIX uid/gid; container will start as its image default user.")
- return []
- _VOLUME_SPEC_RE = re.compile(r"^(?P<host>.+):(?P<container>/[^:]+)(?::[^:]*)?$")
- # Second mount when a user volume already owns /workspace. Not a username.
- _HOST_CWD_FALLBACK_MOUNTS = ("/host-cwd", "/host-cwd-2", "/host-cwd-3")
- def _split_volume_spec(spec: str) -> tuple[str, str] | None:
- """``host:container[:mode]`` → ``(host, container)``. Drive-letter hosts keep their colon."""
- if not isinstance(spec, str):
- return None
- match = _VOLUME_SPEC_RE.match(spec.strip())
- if not match:
- return None
- return match.group("host"), match.group("container")
- def _container_mount_taken(volume_args: list[str], mount: str) -> bool:
- target = mount.rstrip("/") or "/"
- for arg in volume_args:
- parsed = _split_volume_spec(arg)
- if parsed and (parsed[1].rstrip("/") or "/") == target:
- return True
- return False
- def _existing_host_mount(volume_args: list[str], host_cwd_abs: str) -> str | None:
- """Container path if a user volume already bind-mounts this host directory."""
- want = _host_path_key(host_cwd_abs)
- if not want:
- return None
- for arg in volume_args:
- parsed = _split_volume_spec(arg)
- if parsed and _host_path_key(parsed[0]) == want:
- return parsed[1]
- return None
- def _free_host_cwd_mount(volume_args: list[str]) -> str:
- for candidate in _HOST_CWD_FALLBACK_MOUNTS:
- if not _container_mount_taken(volume_args, candidate):
- return candidate
- return _HOST_CWD_FALLBACK_MOUNTS[-1]
- def _abs_host_cwd(host_cwd: str) -> str:
- """Absolute host path. A Windows drive path is not prefixed with the POSIX process cwd."""
- expanded = os.path.expanduser(host_cwd)
- if _is_windows_drive_path(expanded) and os.name != "nt":
- return expanded
- return os.path.abspath(expanded)
- class DockerEnvironment(BaseEnvironment):
- """Hardened Docker container execution (caps dropped, no-new-privileges, PID limits,
- size-limited tmpfs). The container is the security boundary — its filesystem stays
- writable so agents can install packages. Persistence bind-mounts /workspace and /root."""
- _passwordless_sudo_probe_supported = True
- _profile_scoped_passthrough = True
- def _additional_profile_scoped_passthrough_names(self) -> tuple[str, ...]:
- """Keep explicit docker_forward_env values out of shared snapshots."""
- return tuple(self._forward_env)
- def __init__(
- self,
- image: str,
- cwd: str = "/root",
- timeout: int = 60,
- cpu: float = 0,
- memory: int = 0,
- disk: int = 0,
- persistent_filesystem: bool = False,
- task_id: str = "default",
- volumes: list | None = None,
- forward_env: list[str] | None = None,
- env: dict | None = None,
- network: bool = True,
- host_cwd: Optional[str] = None,
- auto_mount_cwd: bool = False,
- run_as_host_user: bool = False,
- extra_args: list | None = None,
- persist_across_processes: bool = True,
- shm_size: str = _DEFAULT_SHM_SIZE,
- shared_container_key: str = "",
- snap_compat: bool = False,
- image_pinned: bool = False):
- if cwd == "~":
- cwd = "/root"
- super().__init__(cwd=cwd, timeout=timeout)
- self._persistent = persistent_filesystem
- self._persist_across_processes = persist_across_processes
- # Set by terminal_tool._create_environment for session-scoped containers
- # (docker + container_persistent: false): removed at session close/idle timeout.
- self._session_scoped = False
- self._task_id = task_id
- self._forward_env = _normalize_forward_env_names(forward_env)
- self._env = _normalize_env_dict(env)
- self._init_unset_passthrough_names: tuple[str, ...] = ()
- self._container_id: Optional[str] = None
- self._init_env_values: dict[str, str] = {}
- self._workspace_dir: Optional[str] = None
- self._home_dir: Optional[str] = None
- # (host_dir, container_path) of the Hermes-created sandbox bind mounts; owner handback targets.
- self._sandbox_mounts: list[tuple[str, str]] = []
- logger.info("DockerEnvironment volumes: %s", volumes)
- if volumes is not None and not isinstance(volumes, list):
- logger.warning("docker_volumes config is not a list: %r", volumes)
- volumes = []
- _ensure_docker_available()
- resource_args = self._resource_args(image, cpu, memory, disk, network, shm_size, extra_args)
- volume_args, writable_args = self._mount_args(volumes, host_cwd, auto_mount_cwd, task_id)
- mount = getattr(self, "host_cwd_mount", None)
- if mount and cwd_follows_host_mount(cwd, mount):
- logger.info(
- "Container cwd follows configured host workspace at %s (requested %s)",
- mount, cwd)
- cwd = mount
- self.cwd = mount
- volume_args.extend(_readonly_skill_mount_args())
- egress_label, egress_volume_args, egress_host_args, env_args, validated_extra = (
- self._egress_and_env_args(extra_args))
- volume_args.extend(egress_volume_args)
- user_args = _host_user_args(run_as_host_user)
- # Resolved once so it works when /usr/local/bin is not in PATH (macOS services).
- self._docker_exe = find_docker() or "docker"
- # s6-overlay images (e.g. hermes-agent:latest) already use /init as PID 1 and exec
- # /run/s6/basedir/bin/init during startup. For those images we must (a) skip Docker's --init (two
- # competing PID-1 inits) and (b) mount /run with exec instead of noexec, or s6 stage0 dies with exit
- # 126 "Permission denied". Detected once here; defaults are kept on any inspection failure. See
- # issue #34628.
- image_uses_s6_init = _image_uses_init_entrypoint(self._docker_exe, image)
- if image_uses_s6_init:
- logger.info(
- "Docker: image %s uses /init (s6-overlay) as entrypoint — "
- "skipping --init and mounting /run with exec.",
- image)
- security_args = _build_security_args(
- run_as_host_user and bool(user_args), run_exec=image_uses_s6_init, snap_compat=snap_compat)
- self._snap_compat = snap_compat
- if snap_compat:
- logger.warning(
- "docker_snap_compat: running without --init and no-new-privileges (snap Docker under AppArmor)")
- logger.info("Docker volume_args: %s", volume_args)
- # docker_extra_args go last so they can override defaults.
- all_run_args = (
- security_args + user_args + writable_args + resource_args
- + egress_host_args + volume_args + env_args + validated_extra)
- logger.info("Docker run_args: %s", all_run_args)
- # Labels identify hermes containers to the orphan reaper (hermes-agent=1),
- # cross-process reuse (task-id/profile) and operators. The reuse identity
- # is captured at start and never changes for the container's lifetime.
- # Egress posture gets its own label: env/CA mounts are immutable after
- # creation, so reusing a pre-egress container would bypass the firewall.
- profile_name = _container_identity(shared_container_key)
- task_label = _sanitize_label_value(task_id)
- self._labels = {
- "hermes-agent": "1",
- "hermes-task-id": task_label,
- "hermes-profile": profile_name,
- _EGRESS_LABEL_KEY: egress_label}
- if workspace := _WORKSPACE_TASK_RE.search(task_id or ""):
- self._labels[_WORKSPACE_LABEL_KEY] = workspace.group(1)
- # Explicit sharing opts into the first creator's settings. Otherwise,
- # changed image/mount/home configuration must start a fresh container.
- if not shared_container_key:
- self._labels[_ENVIRONMENT_LABEL_KEY] = _reuse_environment_fingerprint(
- image=image, mount_args=[*writable_args, *volume_args],
- hermes_home=str(get_hermes_home()))
- # Saved for container recreation on "No such container" recovery.
- self._image = image
- self._image_pinned = image_pinned
- self._image_uses_s6_init = image_uses_s6_init
- self._all_run_args = all_run_args
- reused = persist_across_processes and self._attach_existing_container(
- task_label, profile_name, egress_label, network)
- if not reused:
- self._container_id = self._docker_run(cwd)
- else: # it never stopped, so no teardown stripped what it left
- strip_sandbox_privileges(self._docker_exe, self._container_id)
- reclaim_sandbox_files(self._docker_exe, self._container_id, self._sandbox_mounts)
- # Init-time env forwarding args seed the snapshot.
- self._init_env_args = self._build_init_env_args()
- self.init_session()
- # --- __init__ helpers ---
- def _egress_and_env_args(self, extra_args) -> tuple[str, list[str], list[str], list[str], list[str]]:
- """Egress credential-injection proxy plumbing (CA mount + HTTPS_PROXY/CA-bundle env so
- outbound traffic routes through the host-side proxy and the sandbox receives proxy tokens
- instead of real API keys), merged with docker_env into name-only ``-e`` args, plus the
- validated docker_extra_args. Returns ``(egress_label, volume_args, host_args, env_args,
- validated_extra)``; sets ``self._run_env_values`` (injected into the docker-client
- subprocess env at run time and reused verbatim by container-recreation recovery)."""
- egress_volume_args, egress_env_overrides, egress_host_args = _egress_proxy_args_for_docker()
- egress_label = _egress_reuse_fingerprint(egress_volume_args, egress_env_overrides, egress_host_args)
- enforce_egress = _egress_enforce_on_docker() if egress_env_overrides else True
- critical_egress_names = _critical_egress_env_names(egress_env_overrides)
- if egress_env_overrides:
- check_forward_env_collisions(self._forward_env, critical_egress_names, enforce_egress)
- check_docker_env_collisions(self._env, egress_env_overrides, enforce_egress)
- merged_env = merge_egress_env(self._env, egress_env_overrides, enforce_egress)
- self._run_env_values = dict(merged_env)
- validated_extra = []
- for arg in (extra_args or []):
- if not isinstance(arg, str):
- logger.warning("Ignoring non-string docker_extra_args entry: %r", arg)
- continue
- validated_extra.append(arg)
- if egress_env_overrides:
- check_extra_args_collisions(validated_extra, critical_egress_names, enforce_egress)
- return egress_label, egress_volume_args, egress_host_args, _name_only_env_args(merged_env), validated_extra
- def _resource_args(self, image, cpu, memory, disk, network, shm_size, extra_args) -> list[str]:
- """cgroup-gated CPU/memory/pids limits, shm size, disk quota and network mode."""
- args: list[str] = []
- if _cgroup_limits_available(image):
- if cpu > 0:
- args.extend(["--cpus", str(cpu)])
- if memory > 0:
- args.extend(["--memory", f"{memory}m"])
- args.extend(["--pids-limit", _DEFAULT_PIDS_LIMIT])
- # --shm-size is a tmpfs option, not cgroup-gated. Skipped when the user
- # sets it in docker_extra_args or opts out with empty/"0".
- shm = str(shm_size or "").strip()
- if shm and shm != "0" and not _extra_args_set_shm_size(extra_args):
- args.extend(["--shm-size", shm])
- if disk > 0 and sys.platform != "darwin":
- if self._storage_opt_supported():
- args.extend(["--storage-opt", f"size={disk}m"])
- else:
- logger.warning(
- "Docker storage driver does not support per-container disk limits "
- "(requires overlay2 on XFS with pquota). Container will run without disk quota.")
- if not network:
- extra_network = _extra_args_network_mode(extra_args)
- if extra_network is None:
- args.append("--network=none")
- elif extra_network != "none":
- # Contradictory intent: honouring the extra arg would hand the agent a networked
- # container despite the configured lockdown, and the reuse guard (NetworkMode ==
- # "none") would churn it every startup. Fail closed and name both keys.
- raise RuntimeError(
- f"terminal.docker_network is false (air-gapped) but terminal.docker_extra_args "
- f"requests --network={extra_network!r}. Remove the --network entry from "
- f"docker_extra_args, or set docker_network: true to opt into that network.")
- # extra_network == "none": same intent stated twice; the extra arg carries it once.
- return args
- def _mount_args(self, volumes, host_cwd, auto_mount_cwd, task_id) -> tuple[list[str], list[str]]:
- """``(volume_args, writable_args)`` for user volumes, host cwd and /workspace,/root.
- Persistent mode bind-mounts from TERMINAL_SANDBOX_DIR (default ~/.hermes/sandboxes/).
- A configured host working directory is bound even when another volume already
- claims ``/workspace``: at ``/workspace`` when that path is free, otherwise at
- a second mount. ``host_cwd`` / ``host_cwd_mount`` tell tools which container
- path is that directory. A Windows drive path is bound whenever it exists on
- the host — it can never be a path inside the Linux container, and the check
- is the drive shape, not a username.
- """
- volume_args: list[str] = []
- for vol in (volumes or []):
- if not isinstance(vol, str):
- logger.warning("Docker volume entry is not a string: %r", vol)
- continue
- vol = vol.strip()
- if not vol:
- continue
- if ":" not in vol:
- logger.warning("Docker volume '%s' missing colon, skipping", vol)
- continue
- volume_args.extend(["-v", vol])
- workspace_explicitly_mounted = any(":/workspace" in v for v in volume_args)
- host_cwd_abs = _abs_host_cwd(host_cwd) if host_cwd else ""
- windows_cwd = _is_windows_drive_path(host_cwd or "") or _is_windows_drive_path(host_cwd_abs)
- host_dir_exists = bool(host_cwd_abs) and os.path.isdir(host_cwd_abs)
- should_bind = host_dir_exists and (auto_mount_cwd or windows_cwd)
- if (auto_mount_cwd or windows_cwd) and host_cwd and not host_dir_exists:
- logger.debug("Skipping docker cwd mount: host_cwd is not a valid directory: %s", host_cwd)
- existing_mount = _existing_host_mount(volume_args, host_cwd_abs) if should_bind else None
- if existing_mount:
- # Already bind-mounted (often the volume that claimed /workspace). Point
- # tools at that container path instead of adding a second -v.
- self.host_cwd = host_cwd_abs
- self.host_cwd_mount = existing_mount
- bind_target = None
- elif should_bind:
- bind_target = (
- "/workspace" if not workspace_explicitly_mounted
- else _free_host_cwd_mount(volume_args))
- self.host_cwd = host_cwd_abs
- self.host_cwd_mount = bind_target
- else:
- self.host_cwd = None
- self.host_cwd_mount = None
- bind_target = None
- bind_at_workspace = bind_target == "/workspace"
- mount_workspace = not bind_at_workspace and not workspace_explicitly_mounted
- writable_args: list[str] = []
- if self._persistent:
- # _sandbox_dir_name(): a raw session-key task_id carries colons,
- # which `-v` reads as extra spec fields (exit 125).
- sandbox = get_sandbox_dir() / "docker" / _sandbox_dir_name(task_id)
- self._home_dir = str(sandbox / "home")
- os.makedirs(self._home_dir, exist_ok=True)
- self._sandbox_mounts = [(self._home_dir, "/root")]
- if mount_workspace:
- self._workspace_dir = str(sandbox / "workspace")
- os.makedirs(self._workspace_dir, exist_ok=True)
- self._sandbox_mounts.append((self._workspace_dir, "/workspace"))
- for host_dir, container_path in self._sandbox_mounts:
- writable_args += ["-v", f"{host_dir}:{container_path}"]
- else:
- writable_args += ["--tmpfs", "/workspace:rw,exec,size=10g"] if mount_workspace else []
- writable_args += ["--tmpfs", "/home:rw,exec,size=1g", "--tmpfs", "/root:rw,exec,size=1g"]
- if bind_target:
- logger.info("Mounting configured host cwd to %s: %s", bind_target, host_cwd_abs)
- volume_args = ["-v", f"{host_cwd_abs}:{bind_target}", *volume_args]
- elif workspace_explicitly_mounted and not existing_mount:
- logger.debug("Skipping docker cwd mount: /workspace already mounted by user config")
- return volume_args, writable_args
- def _attach_existing_container(self, task_label, profile_name, egress_label, network: bool) -> bool:
- """Attach to a prior process's labeled container ("ONE long-lived container shared
- across sessions"; opt out via ``docker_persist_across_processes: false``).
- Network guard is lockdown-only: a bridge container under ``docker_network: false``
- is removed and recreated, but a ``none`` container under default config is kept so
- ``--network=none`` in extra args doesn't churn containers every startup."""
- existing = self._find_reusable_container(task_label, profile_name, egress_label)
- if existing is None:
- return False
- container_id, state = existing
- # A container built from another image. Explicitly configured image (config.yaml /
- # TERMINAL_DOCKER_IMAGE): the user changed it, so the old container is not their sandbox any
- # more — recreate (the image is immutable after creation). Default image: a default flip
- # (nikolaik base -> hermes-sandbox:desktop) must not replace a sandbox someone has state in;
- # keep it and let the CLI / Screen pane ask. Same rule Modal (snapshot wins) and Daytona
- # (labeled sandbox wins) already apply.
- actual_image = self._container_image(container_id)
- if actual_image is not None and actual_image != self._image:
- if not self._image_pinned:
- logger.warning(
- "Existing container %s runs image %s; the default docker_image is now %s. Keeping "
- "the existing sandbox — approve the switch with `hermes config set "
- "terminal.docker_image %s` (files in /root and /workspace carry over) or pin the "
- "current image to stop this notice (task=%s, profile=%s).",
- container_id[:12], actual_image, self._image, self._image, task_label, profile_name)
- elif not self._image_available_locally():
- # The replacement image cannot be had (private/misspelled tag, registry down, pull past
- # its timeout). Removing the old container first would throw away its writable layer with
- # nothing to put in its place, so keep running the sandbox the user has and retry the
- # switch next time the image resolves.
- logger.warning(
- "Existing container %s runs image %s; docker_image is %s but that image could not be "
- "pulled — keeping the current sandbox until it can (task=%s, profile=%s).",
- container_id[:12], actual_image, self._image, task_label, profile_name)
- else:
- logger.warning(
- "Existing container %s runs image %s but docker_image is %s — removing it and "
- "starting fresh (task=%s, profile=%s).",
- container_id[:12], actual_image, self._image, task_label, profile_name)
- try:
- run_capture([self._docker_exe, "rm", "-f", container_id], timeout=30)
- except (subprocess.TimeoutExpired, OSError) as e:
- logger.warning("Failed to remove mismatched container %s: %s", container_id[:12], e)
- return False
- if not network:
- actual_mode = self._container_network_mode(container_id)
- if actual_mode != "none":
- logger.warning(
- "Existing container %s has NetworkMode=%s but "
- "docker_network=false requests an air-gapped "
- "container — removing it and starting fresh (task=%s, profile=%s).",
- container_id[:12], actual_mode or "unknown", task_label, profile_name)
- try:
- run_capture([self._docker_exe, "rm", "-f", container_id], timeout=30)
- except (subprocess.TimeoutExpired, OSError) as e:
- logger.warning("Failed to remove mismatched container %s: %s", container_id[:12], e)
- return False
- if state != "running":
- err = self._start_container(container_id)
- if err is not None:
- logger.warning(
- "Failed to start existing container %s (state=%s): "
- "%s — falling back to a fresh container.",
- container_id[:12], state, err)
- return False
- self._container_id = container_id
- logger.info(
- "Reusing container %s (task=%s, profile=%s, prior state=%s)",
- container_id[:12], task_label, profile_name, state)
- return True
- def _image_available_locally(self) -> bool:
- """True once ``self._image`` is in the local image store, pulling it when it is not. Called BEFORE a
- container replacement removes anything: a pull that fails must leave the old sandbox intact."""
- try:
- probe = run_capture([self._docker_exe, "image", "inspect", self._image, "--format", "{{.Id}}"],
- timeout=30)
- if probe.returncode == 0:
- return True
- pull = run_capture([self._docker_exe, "pull", self._image], timeout=900)
- return pull.returncode == 0
- except (subprocess.SubprocessError, OSError) as e:
- logger.warning("Docker: could not pull %s: %s", self._image, e)
- return False
- def _start_container(self, container_id: str) -> Exception | None:
- """``docker start`` a stopped container; returns the failure instead of raising."""
- try:
- run_capture([self._docker_exe, "start", container_id], timeout=30, check=True)
- return None
- except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
- return e
- def _run_command(self, name: str, workdir: str) -> list[str]:
- """``docker run -d`` argv for a fresh ``sleep infinity`` container (idle reaper handles
- lifetime). s6-overlay images already provide PID 1, so ``--init`` is skipped for them."""
- label_args = [arg for k, v in self._labels.items() for arg in ("--label", f"{k}={v}")]
- return [
- # tini/catatonit as PID 1 reaps zombie children — but s6-overlay images already provide their
- # own /init PID 1, so adding --init there creates two competing inits and breaks startup
- # (#34628).
- self._docker_exe, "run", "-d",
- *([] if self._image_uses_s6_init or self._snap_compat else ["--init"]),
- "--name", name,
- *label_args,
- "-w", workdir,
- *self._all_run_args,
- self._image,
- "sleep", "infinity"]
- def _docker_run(self, cwd: str) -> str:
- """Start a fresh container and return its id. A failed ``docker run`` (exit 125, timeout
- mid-pull) can leave a "Created" orphan the exited-only reaper never catches, so it is
- removed by name before re-raising."""
- container_name = f"hermes-{uuid.uuid4().hex[:8]}"
- run_cmd = self._run_command(container_name, cwd)
- logger.debug("Starting container: %s", ' '.join(run_cmd))
- try:
- result = run_capture(
- run_cmd, timeout=120, check=True, # image pull may take a while
- env=self._docker_client_env(self._run_env_values))
- except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
- logger.warning("docker run failed for %s, cleaning up orphaned container: %s", container_name, e)
- subprocess.run(
- [self._docker_exe, "rm", "-f", container_name],
- capture_output=True, timeout=10, stdin=subprocess.DEVNULL, check=False)
- raise
- container_id = result.stdout.strip()
- logger.info("Started container %s (%s)", container_name, container_id[:12])
- return container_id
- # --- Env forwarding ---
- def _docker_client_env(self, values: dict[str, str]) -> dict[str, str] | None:
- """Env for the docker-client subprocess carrying forwarded values (pairs with name-only
- ``-e KEY`` flags to keep secrets out of cmdline); ``None`` = inherit when empty.
- Name-only ``-e KEY`` flags make the docker CLI read each value from its own process environment,
- keeping secrets out of the client's world-readable ``/proc/<pid>/cmdline`` (issue #96268). Values
- live in ``/proc/<pid>/environ`` instead, which is owner/root-only. Returns ``None`` (inherit as
- before) when there is nothing to add.
- """
- return client_env_with(values)
- def _build_init_env_args(self) -> list[str]:
- """Name-only ``-e`` args for init_session so ``export -p`` captures docker_env
- plus the current profile's forwarded values (values travel via the client env).
- The VALUES intentionally do not appear in the argv — they are passed via the docker client
- subprocess env (see _docker_client_env and issue #96268); the flags here are name-only ``-e KEY``.
- """
- passthrough_env, unset_names = self._resolve_passthrough_env()
- exec_env = {**self._env, **passthrough_env}
- for name in unset_names:
- exec_env.pop(name, None)
- self._init_unset_passthrough_names = tuple(sorted(unset_names))
- self._init_env_values = dict(exec_env)
- return _name_only_env_args(exec_env)
- def _resolve_passthrough_env(self) -> tuple[dict[str, str], set[str]]:
- """See ``remote_common.resolve_passthrough_env``; explicit docker_forward_env bypasses the blocklist."""
- return resolve_passthrough_env(self._forward_env, hermes_env_loader=_load_hermes_env_vars)
- def _build_runtime_env_args_with_unsets(self) -> tuple[list[str], tuple[str, ...], dict[str, str]]:
- """Runtime name-only forwarding args, names absent from scope, and the values
- to inject into the docker client subprocess env.
- See #96268.
- """
- passthrough_env, unset_names = self._resolve_passthrough_env()
- return _name_only_env_args(passthrough_env), tuple(sorted(unset_names)), dict(passthrough_env)
- def _run_bash(self, cmd_string: str, *, login: bool = False,
- timeout: int = 120,
- stdin_data: str | None = None) -> subprocess.Popen:
- """Spawn bash inside the container. Init seeds the snapshot; profile-scoped passthrough
- values are re-injected on every command because one container can be shared by
- multiple routed profiles in a gateway process."""
- assert self._container_id, "Container not started"
- cmd = [self._docker_exe, "exec"]
- if stdin_data is not None:
- cmd.append("-i")
- # Init seeds the snapshot. Profile-scoped passthrough values are also injected on every later
- # command because this container can be shared by multiple routed profiles in one gateway process.
- # Env flags are name-only; values travel via the client subprocess env so they never hit
- # world-readable /proc/*/cmdline (#96268).
- unset_names: tuple[str, ...] = ()
- env_values: dict[str, str] = {}
- if login:
- cmd.extend(self._init_env_args)
- env_values = dict(getattr(self, "_init_env_values", {}))
- unset_names = getattr(self, "_init_unset_passthrough_names", ())
- elif self._profile_scoped_passthrough:
- runtime_args, unset_names, env_values = self._build_runtime_env_args_with_unsets()
- cmd.extend(runtime_args)
- cmd += [self._container_id, *bash_argv(prepend_unset(cmd_string, unset_names), login)]
- client_env = self._docker_client_env(env_values)
- return _popen_bash(cmd, stdin_data, env=client_env) if client_env is not None else _popen_bash(cmd, stdin_data)
- # --- "No such container" recovery ---
- _NO_CONTAINER_PATTERNS = ("No such container", "is not running", "no such container")
- def _is_container_gone(self, output: str) -> bool:
- return any(p in output for p in self._NO_CONTAINER_PATTERNS)
- def _recreate_container(self) -> bool:
- """Recreate a container removed out-of-band: label-based reuse first (another process
- may have recreated it), else a fresh one from the saved image/run-args. False when
- recovery fails so the caller surfaces the original error."""
- logger.warning("Container %s appears to be gone — attempting recovery", (self._container_id or "")[:12])
- self._container_id = None
- existing = self._find_reusable_container(
- self._labels.get("hermes-task-id", ""),
- self._labels.get("hermes-profile", ""),
- self._labels.get(_EGRESS_LABEL_KEY, "off"))
- if existing is not None:
- cid, state = existing
- if state == "running":
- self._container_id = cid
- logger.info("Recovery: reusing running container %s", cid[:12])
- elif (err := self._start_container(cid)) is None:
- self._container_id = cid
- logger.info("Recovery: restarted container %s", cid[:12])
- else:
- logger.warning("Recovery: failed to start container %s: %s", cid[:12], err)
- if not self._container_id:
- if not self._image:
- logger.error("Recovery: no saved image name, cannot recreate container")
- return False
- try:
- new_name = f"hermes-{uuid.uuid4().hex[:8]}"
- result = run_capture(
- self._run_command(new_name, self.cwd), timeout=120, check=True,
- env=self._docker_client_env(self._run_env_values))
- self._container_id = result.stdout.strip()
- logger.info("Recovery: created fresh container %s (%s)", new_name, self._container_id[:12])
- except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e:
- logger.error("Recovery: failed to create new container: %s", e)
- return False
- try:
- self._snapshot_ready = False
- self.init_session()
- except Exception as e:
- logger.error("Recovery: init_session failed in new container: %s", e)
- return False
- logger.info("Recovery successful — new container %s", (self._container_id or "")[:12])
- self._mark_recreated()
- return True
- def execute(self, command: str, cwd: str = "", **kwargs) -> dict:
- """Execute a command; if the container was removed out-of-band (idle reaper,
- docker prune, OOM, daemon restart) recreate it and retry once."""
- result = super().execute(command, cwd, **kwargs)
- if (
- result.get("returncode", 0) != 0
- and self._is_container_gone(result.get("output", ""))
- and self._persist_across_processes
- and self._recreate_container()):
- result = super().execute(command, cwd, **kwargs)
- return result
- @staticmethod
- def _storage_opt_supported() -> bool:
- """Whether ``--storage-opt size=`` works (only overlay2 on XFS with pquota; ext4 errors out).
- Only definitive answers are cached: a probe that could not run (daemon cold-start,
- hello-world pull timeout) says nothing about pquota support and is retried next spawn."""
- global _storage_opt_ok
- if _storage_opt_ok is not None:
- return _storage_opt_ok
- try:
- docker = find_docker() or "docker"
- result = run_capture([docker, "info", "--format", "{{.Driver}}"], timeout=10)
- if result.returncode != 0:
- return False # daemon unreachable etc. is transient; retry next spawn
- if result.stdout.strip().lower() != "overlay2":
- _storage_opt_ok = False # storage driver is a host property
- return False
- # Probe with a real create — the fastest reliable check.
- probe = run_capture([docker, "create", "--storage-opt", "size=1m", "hello-world"], timeout=15)
- if probe.returncode == 0:
- _storage_opt_ok = True
- if probe.stdout.strip():
- subprocess.run([docker, "rm", probe.stdout.strip()],
- capture_output=True, timeout=5, stdin=subprocess.DEVNULL, check=False)
- elif "storage" in (probe.stderr or "").lower():
- _storage_opt_ok = False # daemon rejected --storage-opt: a host property
- # else: pull/daemon failure unrelated to storage-opt; not cached, retried next spawn
- except Exception:
- return False # TimeoutExpired, missing binary; transient, retried next spawn
- logger.debug("Docker --storage-opt support: %s", _storage_opt_ok)
- return _storage_opt_ok or False
- def _container_image(self, container_id: str) -> Optional[str]:
- """The image reference a container was created from (``Config.Image``: the tag as given
- to ``docker run``, so it compares directly with ``docker_image``), or ``None`` when
- inspection fails (callers then keep the container: a failed probe must not churn)."""
- result = _docker_query(
- [self._docker_exe, "inspect", "--format", "{{.Config.Image}}", container_id], timeout=10,
- fail="docker inspect Image failed: %s", nonzero="docker inspect Image returned %d: %s")
- return (result.stdout.strip() or None) if result is not None else None
- def _container_network_mode(self, container_id: str) -> Optional[str]:
- """``HostConfig.NetworkMode`` of a container, or ``None`` when inspection fails (callers
- treat ``None`` as a mismatch under lockdown, so a failed inspect fails closed)."""
- result = _docker_query(
- [self._docker_exe, "inspect", "--format", "{{.HostConfig.NetworkMode}}", container_id], timeout=10,
- fail="docker inspect NetworkMode failed: %s", nonzero="docker inspect NetworkMode returned %d: %s")
- return (result.stdout.strip() or None) if result is not None else None
- def _find_reusable_container(
- self, task_label: str, profile_label: str, egress_label: str) -> Optional[tuple[str, str]]:
- """``(container_id, state)`` of an existing container labeled for this task/profile/
- egress posture and immutable environment, or ``None`` on miss or any failure.
- Explicit shared keys opt out of the environment filter. The egress posture is a label
- FILTER for every posture, "off" included: a container built with egress on must not be
- reused after ``hermes egress disable`` (baked-in proxy env and CA mounts), and every
- container this class creates carries the label. The ``{{.Label "key"}}`` template
- function is Docker-only — podman ps exits 125 on it — so the probe never uses it (#99213)."""
- filters = [
- "--filter", "label=hermes-agent=1",
- "--filter", f"label=hermes-task-id={task_label}",
- "--filter", f"label=hermes-profile={profile_label}",
- "--filter", f"label={_EGRESS_LABEL_KEY}={egress_label}"]
- if environment_label := self._labels.get(_ENVIRONMENT_LABEL_KEY):
- filters.extend(["--filter", f"label={_ENVIRONMENT_LABEL_KEY}={environment_label}"])
- result = _docker_query(
- [self._docker_exe, "ps", "-a", *filters, "--format", "{{.ID}}\t{{.State}}"], timeout=10,
- fail="docker ps probe failed: %s — will start a fresh container",
- nonzero="docker ps probe returned %d: %s — will start a fresh container")
- if result is None:
- return None
- # Multiple matches can happen after a crash mid-cleanup: prefer a running
- # one, else the first listed; stale duplicates are the orphan reaper's job.
- running = first = None
- for ln in (ln for ln in result.stdout.splitlines() if ln.strip()):
- parts = ln.split("\t", 1)
- if len(parts) != 2:
- continue
- cid, state = parts[0], parts[1].strip().lower()
- if first is None:
- first = (cid, state)
- if state == "running" and running is None:
- running = (cid, state)
- return running or first
- def _remove_bind_dirs(self) -> None:
- for d in (self._workspace_dir, self._home_dir):
- if d:
- shutil.rmtree(d, ignore_errors=True)
- def cleanup(self, *, force_remove: bool = False):
- """Tear down per persist mode. Persist mode (default) leaves the container RUNNING —
- stopping it on every exit would kill background processes and add a ``docker start``
- delay per session; reclamation is ``reap_orphan_containers()`` at next startup.
- ``persist_across_processes=False`` or ``force_remove=True`` (explicit-teardown hook,
- unused so far) does ``docker stop`` + ``docker rm -f`` on a daemon thread that the
- atexit hook joins via ``wait_for_cleanup`` so the work completes before exit.
- Cleanup runs on a daemon thread with bounded ``subprocess.run`` calls (not the racy ``Popen(... &)``
- pattern from before PR #33645). The atexit hook in ``tools/terminal_tool.py`` waits up to 15s for
- the thread to finish before the interpreter exits, so ``docker stop`` / ``docker rm`` actually
- completes when we do trigger it.
- """
- container_id = self._container_id
- if not container_id:
- # Bind-mount dirs are still dropped in non-persistent mode.
- if not self._persistent:
- self._remove_bind_dirs()
- return
- # Capture what the worker needs — the thread can outlive ``self``.
- docker_exe = self._docker_exe
- log_id = container_id[:12]
- sandbox_mounts = list(getattr(self, "_sandbox_mounts", []))
- if not force_remove and self._persist_across_processes:
- # Drop the in-process handle so a fresh __init__ re-probes via
- # labels instead of reusing a stale Python reference.
- self._container_id = None
- # The host user may run what the container left before it is reattached.
- self._start_teardown(lambda: strip_sandbox_privileges(docker_exe, container_id), log_id)
- return
- def _do_cleanup() -> None:
- for argv, fail_msg in ((["stop", "-t", "10"], "docker stop %s timed out / failed: %s"),
- (["rm", "-f"], "docker rm -f %s failed: %s")):
- if argv[0] == "rm" and sandbox_mounts: # stopped: no writer left, volumes still reachable
- hand_back_sandbox_files(docker_exe, container_id, sandbox_mounts)
- try:
- subprocess.run(
- [docker_exe, *argv, container_id],
- capture_output=True, timeout=30, stdin=subprocess.DEVNULL, check=False)
- except (subprocess.TimeoutExpired, OSError) as e:
- logger.warning(fail_msg, log_id, e)
- self._start_teardown(_do_cleanup, log_id)
- self._container_id = None
- # Bind-mount dirs are the container's filesystem state; only drop them
- # once the container itself is removed.
- if not self._persistent:
- self._remove_bind_dirs()
- def stop_idle(self) -> None:
- """Idle reclaim of a per-workspace persistent container (#140665): ``docker stop``, never
- ``rm``. Its writable layer and ``/root`` sandbox stay, and the next use in that workspace
- re-attaches through :meth:`_attach_existing_container`, which ``docker start``s a stopped
- container. Without this, persistent mode left one RUNNING container per workspace a user
- ever opened (each with its own ``container_memory``) until the process exited. Like a
- persist-mode cleanup, the host user may touch what it left before it is reattached, so its
- sandbox privileges are stripped once it has stopped."""
- container_id = self._container_id
- if not container_id:
- return
- docker_exe, log_id = self._docker_exe, container_id[:12]
- def _do_stop() -> None:
- try:
- subprocess.run([docker_exe, "stop", "-t", "10", container_id], capture_output=True,
- timeout=30, stdin=subprocess.DEVNULL, check=False)
- except (subprocess.TimeoutExpired, OSError) as e:
- logger.warning("docker stop %s (idle workspace) failed: %s", log_id, e)
- strip_sandbox_privileges(docker_exe, container_id)
- self._start_teardown(_do_stop, log_id)
- self._container_id = None
- def _start_teardown(self, work, log_id: str) -> None:
- """Run *work* on a daemon thread the atexit drain joins."""
- t = threading.Thread(target=work, daemon=True, name=f"hermes-cleanup-{log_id}")
- with _TEARDOWN_LOCK:
- _TEARDOWN_THREADS.add(t)
- t.start()
- self._cleanup_thread = t
- @staticmethod
- def wait_for_all_teardowns(timeout: float = 15.0) -> bool:
- """Join every in-flight teardown worker, including those of envs the idle reaper already
- popped out of the active registry (their handles are otherwise unreachable at exit, so
- ``docker rm`` died with the interpreter and left a stopped container behind — #86317).
- Re-snapshots each pass: the reaper may start a worker while the drain runs."""
- deadline = time.monotonic() + timeout
- while True:
- with _TEARDOWN_LOCK:
- _TEARDOWN_THREADS.difference_update({t for t in _TEARDOWN_THREADS if not t.is_alive()})
- pending = list(_TEARDOWN_THREADS)
- if not pending:
- return True
- remaining = deadline - time.monotonic()
- if remaining <= 0:
- return False
- pending[0].join(timeout=remaining)
- def wait_for_cleanup(self, timeout: float = 30.0) -> bool:
- """Block up to *timeout* seconds for the cleanup thread (atexit hook). True if it
- finished or none was started, False on timeout."""
- thread = getattr(self, "_cleanup_thread", None)
- if thread is None or not thread.is_alive():
- return True
- thread.join(timeout=timeout)
- return not thread.is_alive()
|