docker.py 71 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394
  1. """Docker execution environment for sandboxed command execution.
  2. Security hardened (cap-drop ALL, no-new-privileges, PID limits), configurable
  3. resource limits (CPU, memory, disk), and optional filesystem persistence via
  4. bind mounts.
  5. """
  6. import datetime
  7. import hashlib
  8. import json
  9. import logging
  10. import os
  11. import re
  12. import shlex
  13. import shutil
  14. import subprocess
  15. import sys
  16. import tempfile
  17. import threading
  18. import time
  19. import uuid
  20. from pathlib import Path
  21. from typing import Optional
  22. from hermes_constants import get_hermes_home
  23. from tools.environments.base import BaseEnvironment, EnvironmentConnectionError, _SHELL_ENV_NAME_RE, get_sandbox_dir
  24. from tools.terminal_tool_config import (
  25. _host_path_key, _is_windows_drive_path, cwd_follows_host_mount, home_profile_name,
  26. )
  27. from tools.environments.base_output import _popen_bash
  28. from tools.environments.docker_egress import (
  29. _EGRESS_LABEL_KEY, _critical_egress_env_names, _egress_enforce_on_docker, _egress_proxy_args_for_docker,
  30. _egress_reuse_fingerprint, check_docker_env_collisions, check_extra_args_collisions,
  31. check_forward_env_collisions, merge_egress_env,
  32. )
  33. from tools.environments.docker_ownership import (
  34. hand_back_sandbox_files, reclaim_sandbox_files, strip_sandbox_privileges)
  35. from tools.environments.path_utils import sanitize_task_id_for_path
  36. from tools.environments.remote_common import (
  37. bash_argv, client_env_with, load_hermes_env_vars, prepend_unset, resolve_passthrough_env, run_capture)
  38. logger = logging.getLogger(__name__)
  39. # Docker Desktop install paths checked when 'docker' is not in PATH
  40. # (macOS Intel / Apple Silicon Homebrew / app bundle).
  41. _DOCKER_SEARCH_PATHS = [
  42. "/usr/local/bin/docker", "/opt/homebrew/bin/docker", "/Applications/Docker.app/Contents/Resources/bin/docker",
  43. ]
  44. _docker_executable: Optional[str] = None # resolved once, cached
  45. _ENV_VAR_NAME_RE = _SHELL_ENV_NAME_RE
  46. _ENVIRONMENT_LABEL_KEY = "hermes-environment"
  47. # Stamped at create time on per-workspace persistent containers (task key ``<base>:ws:<digest>``)
  48. # with the digest as its value. The orphan reaper reads it to spare a container the idle reaper
  49. # stopped on purpose; ``hermes-task-id`` cannot serve, it is truncated to 63 chars.
  50. _WORKSPACE_LABEL_KEY = "hermes-workspace"
  51. _WORKSPACE_TASK_RE = re.compile(r":ws:([0-9a-f]{16})$")
  52. def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]:
  53. """Return a deduplicated list of valid environment variable names."""
  54. normalized: list[str] = []
  55. for item in forward_env or []:
  56. if not isinstance(item, str):
  57. logger.warning("Ignoring non-string docker_forward_env entry: %r", item)
  58. continue
  59. key = item.strip()
  60. if not key:
  61. continue
  62. if not _ENV_VAR_NAME_RE.match(key):
  63. logger.warning("Ignoring invalid docker_forward_env entry: %r", item)
  64. elif key not in normalized:
  65. normalized.append(key)
  66. return normalized
  67. def _normalize_env_dict(env: dict | None) -> dict[str, str]:
  68. """Validate a docker_env dict to {str: str}; scalars are coerced, other values dropped."""
  69. if not env:
  70. return {}
  71. if not isinstance(env, dict):
  72. logger.warning("docker_env is not a dict: %s", type(env).__name__)
  73. return {}
  74. normalized: dict[str, str] = {}
  75. for key, value in env.items():
  76. if not isinstance(key, str) or not _ENV_VAR_NAME_RE.match(key.strip()):
  77. logger.warning("Ignoring invalid docker_env key: %r", key)
  78. continue
  79. if not isinstance(value, (str, int, float, bool)):
  80. logger.warning("Ignoring non-string docker_env value for %r: %s", key.strip(), type(value).__name__)
  81. continue
  82. normalized[key.strip()] = value if isinstance(value, str) else str(value)
  83. return normalized
  84. # Module-level binding: tests patch ``docker._load_hermes_env_vars`` to fake the .env file.
  85. _load_hermes_env_vars = load_hermes_env_vars
  86. # Docker label values must match [a-zA-Z0-9_.-] and stay <=63 chars to round-trip
  87. # through `docker ps --filter label=key=value`.
  88. _LABEL_VALUE_OK_RE = re.compile(r"[^A-Za-z0-9_.-]")
  89. def _sanitize_label_value(value: str) -> str:
  90. """Lossy coercion into a Docker label-safe form; empty/invalid input becomes ``"unknown"``."""
  91. if not isinstance(value, str) or not value:
  92. return "unknown"
  93. return _LABEL_VALUE_OK_RE.sub("_", value)[:63] or "unknown"
  94. # task_id -> host directory name; shared with every backend persisting per-task
  95. # state on the host (the singularity plugin's overlays) so the mapping is fixed in one place.
  96. _sandbox_dir_name = sanitize_task_id_for_path
  97. def _get_active_profile_name() -> str:
  98. """Active Hermes profile name (the container key's own lookup), or ``"default"`` on error. Resolved
  99. at container-create time so a container stays tagged with its creator even if the process switches
  100. profiles."""
  101. return home_profile_name()
  102. def _container_identity(shared_key: str = "") -> str:
  103. """Profile label used for container reuse and orphan reaping. Profiles are isolated by default; an
  104. explicit shared key lets trusted profiles share one Docker identity. Label sanitization is lossy
  105. and reuse is label-keyed, so a digest of the raw key disambiguates colliding keys. Plain profile
  106. names keep their historical un-suffixed labels."""
  107. if not shared_key:
  108. return _sanitize_label_value(_get_active_profile_name())
  109. digest = hashlib.sha256(shared_key.encode("utf-8")).hexdigest()[:12]
  110. return f"{_sanitize_label_value(shared_key)[:50]}-{digest}"
  111. def _is_volatile_mount_spec(spec: str) -> bool:
  112. """True when *spec* is a ``host:container[:mode]`` mount whose host source is a
  113. per-process tempdir (a ``mkdtemp`` under the system temp), so its path is random
  114. per process and must not be hashed into the reuse label.
  115. The known source is the symlink-safe skills copy from
  116. ``credential_files._safe_skills_path``: any symlink under ``skills/`` makes it a
  117. fresh ``mkdtemp`` per process. Stable host paths — including a symlink-free
  118. skills dir, which mounts directly — never sit under the process tempdir.
  119. """
  120. if spec in ("-v", "--mount") or ":" not in spec:
  121. return False # the flag element itself, or a non-bind arg (tmpfs modes etc.)
  122. parsed = _split_volume_spec(spec)
  123. source = parsed[0] if parsed is not None else spec.split(":", 1)[0]
  124. if _is_windows_drive_path(source):
  125. return False # drive-letter hosts can never be the POSIX process tempdir
  126. try:
  127. temp_root = os.path.realpath(tempfile.gettempdir())
  128. source_abs = os.path.realpath(os.path.abspath(os.path.expanduser(source)))
  129. return source_abs == temp_root or source_abs.startswith(temp_root + os.sep)
  130. except OSError: # unreadable source — treat as stable, fail the safe way
  131. return False
  132. def _reuse_environment_fingerprint(*, image: str, mount_args: list[str], hermes_home: str) -> str:
  133. """Hash immutable configuration so reuse cannot silently attach to stale mounts.
  134. Hash requested values rather than exposing profile paths and volume sources in labels.
  135. Keep mount order: later arguments can override earlier mount destinations.
  136. Per-process tempdir-sourced mounts (the symlink-safe skills copy) have their volatile
  137. host path replaced by a stable placeholder: the path is random per process, so hashing
  138. it made the label differ across processes and cross-process container reuse never
  139. matched for users with any symlink under ``skills/``. The container path stays in the
  140. hash, so moving where that mount lands still forces a fresh container.
  141. """
  142. normalized_home = os.path.normcase(os.path.abspath(os.path.expanduser(hermes_home)))
  143. canonical_mounts = [
  144. (f"<volatile-tempdir-mount>:{spec.split(':', 1)[1]}"
  145. if _is_volatile_mount_spec(spec) else spec)
  146. for spec in mount_args]
  147. payload = json.dumps(
  148. {"image": image, "mount_args": canonical_mounts, "hermes_home": normalized_home},
  149. sort_keys=True, separators=(",", ":"))
  150. return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:24]
  151. def reap_orphan_containers(
  152. *, max_age_seconds: int = 600, profile_filter: str | None = None, docker_exe: str | None = None,
  153. ) -> int:
  154. """Remove stale hermes-tagged containers left behind by prior processes (SIGKILL/OOM
  155. exits that bypass atexit). Only ``status=exited`` containers (running ones may belong
  156. to a sibling process), only the caller's profile, and only if ``FinishedAt`` is older
  157. than *max_age_seconds* (a just-exited sibling may be about to reuse its container).
  158. Best-effort and idempotent: failures log at debug and the count removed so far is returned.
  159. """
  160. docker = docker_exe or find_docker() or "docker"
  161. filters = ["--filter", "label=hermes-agent=1", "--filter", "status=exited"]
  162. if profile_filter:
  163. filters.extend(["--filter", f"label=hermes-profile={_sanitize_label_value(profile_filter)}"])
  164. listing = _docker_query(
  165. [docker, "ps", "-a", *filters, "--format", "{{.ID}}"], timeout=15,
  166. fail="orphan reaper docker ps failed: %s", nonzero="orphan reaper docker ps returned %d: %s")
  167. if listing is None:
  168. return 0
  169. # Per-container inspect keeps the failure blast radius to one container.
  170. now = datetime.datetime.now(datetime.UTC)
  171. removed = 0
  172. for cid in (ln.strip() for ln in listing.stdout.splitlines() if ln.strip()):
  173. finished_at = _container_finished_at(docker, cid)
  174. if finished_at is None: # unknown age — be conservative
  175. continue
  176. age = (now - finished_at).total_seconds()
  177. if age < max_age_seconds:
  178. continue
  179. strip_sandbox_privileges(docker, cid) # its teardown never ran; owners stay (it may restart)
  180. if _is_idle_workspace_container(docker, cid):
  181. continue
  182. # No -f: a sibling may have restarted the container between the ps snapshot
  183. # and now (FinishedAt still reports the previous exit), and the daemon refuses
  184. # a plain rm on a running container, which is the atomic recheck this sweep needs.
  185. result = _docker_query(
  186. [docker, "rm", cid], timeout=30, fail="orphan reaper docker rm %s failed: %s", fail_args=(cid[:12],))
  187. if result is None:
  188. continue
  189. if result.returncode == 0:
  190. removed += 1
  191. logger.info("Reaped orphan container %s (exited %d seconds ago)", cid[:12], int(age))
  192. else:
  193. logger.debug("docker rm %s failed: %s", cid[:12], result.stderr.strip())
  194. return removed
  195. def _is_idle_workspace_container(docker_exe: str, container_id: str) -> bool:
  196. """A persistent per-workspace container (task key ``<base>:ws:<digest>``, stamped with the
  197. ``hermes-workspace=<digest>`` label at create time) that the idle reaper STOPPED on purpose
  198. (``DockerEnvironment.stop_idle``): it is that workspace's sandbox, restarted on next use, not an
  199. orphan. Reads the dedicated label, never the 63-char-truncated ``hermes-task-id``, whose
  200. ``_ws_<digest>`` suffix a long profile name cuts. Unknown (inspect failed) reads as not one."""
  201. result = _docker_query(
  202. [docker_exe, "inspect", "--format", f'{{{{index .Config.Labels "{_WORKSPACE_LABEL_KEY}"}}}}', container_id],
  203. timeout=10, fail="orphan reaper docker inspect %s failed: %s", fail_args=(container_id[:12],))
  204. if result is None or result.returncode != 0:
  205. return False
  206. return re.fullmatch(r"[0-9a-f]{16}", result.stdout.strip()) is not None
  207. def _container_finished_at(docker_exe: str, container_id: str):
  208. """``docker inspect`` FinishedAt as an aware datetime; ``None`` (= don't reap) when
  209. missing/unparseable or Docker's never-finished zero value ``0001-01-01T00:00:00Z``."""
  210. result = _docker_query(
  211. [docker_exe, "inspect", "--format", "{{.State.FinishedAt}}", container_id], timeout=10,
  212. fail="orphan reaper docker inspect %s failed: %s", fail_args=(container_id[:12],))
  213. if result is None or result.returncode != 0:
  214. return None
  215. raw = result.stdout.strip()
  216. if not raw or raw.startswith("0001-01-01"):
  217. return None
  218. # Docker emits RFC3339 with nanoseconds; fromisoformat only takes microseconds.
  219. raw = re.sub(r"(\.\d{6})\d+", r"\1", raw).replace("Z", "+00:00")
  220. try:
  221. return datetime.datetime.fromisoformat(raw)
  222. except ValueError as e:
  223. logger.debug("could not parse FinishedAt %r for %s: %s", raw, container_id[:12], e)
  224. return None
  225. def _is_executable(path: str) -> bool:
  226. return os.path.isfile(path) and os.access(path, os.X_OK)
  227. def _docker_query(
  228. argv: list[str], *, timeout: float, fail: str, fail_args: tuple = (), nonzero: str | None = None,
  229. exc_types: tuple = (subprocess.TimeoutExpired, OSError),
  230. ):
  231. """Best-effort ``run_capture`` for probes: logs *fail* (``*fail_args, exc``) at debug and
  232. returns ``None`` when the CLI raises; with *nonzero* set, a nonzero exit also logs
  233. (``*fail_args, returncode, stderr``) and returns ``None``."""
  234. try:
  235. result = run_capture(argv, timeout=timeout)
  236. except exc_types as e:
  237. logger.debug(fail, *fail_args, e)
  238. return None
  239. if nonzero is not None and result.returncode != 0:
  240. logger.debug(nonzero, *fail_args, result.returncode, result.stderr.strip())
  241. return None
  242. return result
  243. def find_docker() -> Optional[str]:
  244. """Locate the docker/podman CLI (cached): ``HERMES_DOCKER_BINARY`` override, ``docker``
  245. on PATH, ``podman`` on PATH, then macOS Docker Desktop locations; ``None`` if absent."""
  246. global _docker_executable
  247. if _docker_executable is not None:
  248. return _docker_executable
  249. override = os.getenv("HERMES_DOCKER_BINARY")
  250. if override and _is_executable(override):
  251. logger.info("Using HERMES_DOCKER_BINARY override: %s", override)
  252. found = override
  253. elif found := shutil.which("docker"):
  254. pass
  255. elif found := shutil.which("podman"):
  256. logger.info("Using podman as container runtime: %s", found)
  257. elif found := next((p for p in _DOCKER_SEARCH_PATHS if _is_executable(p)), None):
  258. logger.info("Found docker at non-PATH location: %s", found)
  259. if found:
  260. _docker_executable = found
  261. return found
  262. def docker_runtime_name(executable: str) -> str:
  263. """User-facing runtime name (``"Podman"`` / ``"Docker"``) for the CLI at *executable*, so
  264. diagnostics and pickers name the runtime actually in use."""
  265. return "Podman" if "podman" in os.path.basename(executable).lower() else "Docker"
  266. def docker_runtime_start_hint(executable: str) -> str:
  267. """How to bring the runtime at *executable* back up, for a "not reachable" message. Docker has
  268. a daemon to start; Podman is daemonless (outside Linux it runs inside a VM)."""
  269. if docker_runtime_name(executable) != "Podman":
  270. return "start Docker and retry"
  271. return "run `podman machine start` and retry"
  272. # Security flags applied to every container. The container is the security
  273. # boundary; all caps are dropped and the minimum added back:
  274. # DAC_OVERRIDE - root can write to bind-mounted dirs owned by the host user
  275. # CHOWN/FOWNER - package managers need to set file ownership
  276. # /tmp is size-limited and nosuid but allows exec (pip/npm builds need it).
  277. # ``--pids-limit``/``--cpus``/``--memory`` live in resource_args instead, gated
  278. # on ``_cgroup_limits_available`` (unprivileged LXCs lack the controllers).
  279. _BASE_SECURITY_ARGS = [
  280. "--cap-drop", "ALL",
  281. "--cap-add", "DAC_OVERRIDE",
  282. "--cap-add", "CHOWN",
  283. "--cap-add", "FOWNER",
  284. "--tmpfs", "/tmp:rw,nosuid,size=512m", # no-tmp: ok — container tmpfs mount spec
  285. "--tmpfs", "/var/tmp:rw,noexec,nosuid,size=256m"]
  286. # Fork-bomb guard, applied only when the pids cgroup controller is available. The pids cgroup counts
  287. # THREADS, and a sandbox that hosts the Bot Screen runs a desktop in here: measured on
  288. # hermes-sandbox:desktop, the idle container is 2 tasks, Xvnc + Xfce + dbus 44, one Chromium with one
  289. # tab 212, plus the agent's own Chromium with two tabs 488. The old 256 was hit in normal
  290. # use and every further `docker exec` (browser command, cua-driver, thumbnail, CDP forward) died with
  291. # runc's "procReady not received". 2048 leaves room for a working browser and is still three orders of
  292. # magnitude under the host's pid_max.
  293. _DEFAULT_PIDS_LIMIT = "2048"
  294. # Docker's 64 MB /dev/shm default crashes Chromium/Playwright tabs and PyTorch
  295. # DataLoader workers. tmpfs is lazily allocated so a 1g ceiling costs nothing
  296. # until used (and still counts against --memory). Empty/"0" config omits the flag.
  297. # Docker's built-in default is a tiny 64 MB, which silently breaks shared-memory-hungry workloads inside the
  298. # sandbox: Chromium / Playwright renderers crash tabs, and PyTorch DataLoader workers die with "bus error" /
  299. # "insufficient shared memory" once they exceed it. tmpfs is lazily allocated, so a 1g ceiling costs nothing
  300. # until actually used (and usage still counts against the container's --memory cgroup limit). Configurable
  301. # via ``terminal.docker_shm_size`` in config.yaml; an empty value (or "0") omits the flag and falls back to
  302. # Docker's 64 MB default. Ported from nanocoai/nanoclaw#2748.
  303. _DEFAULT_SHM_SIZE = "1g"
  304. def _extra_args_set_shm_size(extra_args: list) -> bool:
  305. """True when docker_extra_args already set ``--shm-size`` (then our default is skipped)."""
  306. return any(
  307. isinstance(a, str) and (a == "--shm-size" or a.startswith("--shm-size="))
  308. for a in (extra_args or []))
  309. _NETWORK_FLAGS = ("--network", "--net")
  310. def _extra_args_network_mode(extra_args: list) -> Optional[str]:
  311. """Network mode requested by ``docker_extra_args`` (``--network none`` / ``--network=none`` /
  312. ``--net``), or None when the operator set no network flag. Docker rejects a repeated
  313. ``--network`` outright (exit 125), so the implicit ``docker_network: false`` flag and an
  314. operator-supplied one must be reconciled before the command line is built."""
  315. args = [a for a in (extra_args or []) if isinstance(a, str)]
  316. for i, arg in enumerate(args):
  317. for flag in _NETWORK_FLAGS:
  318. if arg == flag:
  319. return args[i + 1] if i + 1 < len(args) else ""
  320. if arg.startswith(f"{flag}="):
  321. return arg.split("=", 1)[1]
  322. return None
  323. # /run is separate from _BASE_SECURITY_ARGS: s6-overlay images exec
  324. # /run/s6/basedir/bin/init at stage0 and die (exit 126) on a noexec mount.
  325. _RUN_TMPFS_NOEXEC = "--tmpfs", "/run:rw,noexec,nosuid,size=64m"
  326. _RUN_TMPFS_EXEC = "--tmpfs", "/run:rw,exec,nosuid,size=64m"
  327. # SETUID/SETGID let a root-started init drop privileges (s6-setuidgid, gosu,
  328. # su). Combined with no-new-privileges the dropped process cannot escalate
  329. # back. Skipped when --user is passed: the container already starts unprivileged.
  330. _PRIVDROP_CAP_ARGS = ["--cap-add", "SETUID", "--cap-add", "SETGID"]
  331. # Every ``cleanup()`` worker, independent of terminal_tool's active-env registry (see
  332. # ``wait_for_all_teardowns``).
  333. _TEARDOWN_THREADS: set[threading.Thread] = set()
  334. _TEARDOWN_LOCK = threading.Lock()
  335. _S6_INIT_ENTRYPOINTS = ("/init", "/package/admin/s6-overlay/command/init")
  336. _NO_NEW_PRIVILEGES_ARGS = ["--security-opt", "no-new-privileges"]
  337. def _build_security_args(run_as_host_user: bool, run_exec: bool = False, snap_compat: bool = False) -> list[str]:
  338. """Security/cap/tmpfs args for the privilege mode; ``run_exec`` mounts /run exec for s6 images.
  339. ``snap_compat`` drops no-new-privileges: snap-packaged Docker's AppArmor profile turns it into
  340. "exec: operation not permitted" for every process in the container (#9730, LP#1908448)."""
  341. args = list(_BASE_SECURITY_ARGS) + ([] if snap_compat else list(_NO_NEW_PRIVILEGES_ARGS))
  342. args += list(_RUN_TMPFS_EXEC if run_exec else _RUN_TMPFS_NOEXEC)
  343. return args if run_as_host_user else args + list(_PRIVDROP_CAP_ARGS)
  344. def _image_uses_init_entrypoint(docker_exe: str, image: str) -> bool:
  345. """True if ``image``'s entrypoint is the s6-overlay ``/init`` (own PID 1: incompatible
  346. with ``--init`` and a noexec /run). Any inspection failure, including an image not yet
  347. pulled, returns False and keeps the hardened defaults."""
  348. result = _docker_query(
  349. [docker_exe, "image", "inspect", image, "--format", "{{json .Config.Entrypoint}}"], timeout=15,
  350. fail="Docker: could not inspect entrypoint for %s: %s", fail_args=(image,),
  351. nonzero="Docker: image inspect for %s returned %d (stderr=%s)",
  352. exc_types=(subprocess.SubprocessError, OSError))
  353. raw = (result.stdout or "").strip() if result is not None else ""
  354. if not raw or raw == "null":
  355. return False
  356. try:
  357. entrypoint = json.loads(raw)
  358. except (ValueError, TypeError):
  359. return False
  360. if isinstance(entrypoint, str):
  361. entrypoint = [entrypoint]
  362. if not isinstance(entrypoint, list) or not entrypoint:
  363. return False
  364. return str(entrypoint[0]).strip() in _S6_INIT_ENTRYPOINTS
  365. def _resolve_host_user_spec() -> Optional[str]:
  366. """``<uid>:<gid>`` of the host user, or ``None`` without POSIX ids. Uses os.getuid/getgid
  367. directly (not pwd/getpass) so nameless UIDs inside sandboxed launchers never raise."""
  368. get_uid = getattr(os, "getuid", None)
  369. get_gid = getattr(os, "getgid", None)
  370. if get_uid is None or get_gid is None:
  371. return None
  372. try:
  373. return f"{get_uid()}:{get_gid()}"
  374. except Exception: # pragma: no cover - defensive
  375. return None
  376. _storage_opt_ok: Optional[bool] = None # cached result across instances
  377. _cgroup_limits_ok: Optional[bool] = None # cached result across instances
  378. def _cgroup_limits_available(image: str) -> bool:
  379. """Probe once per process whether ``--cpus``/``--memory``/``--pids-limit`` work here, via a
  380. throwaway ``sleep 0`` container from *image* (no extra pull). Without delegated cgroup
  381. controllers (unprivileged LXCs, rootless) these flags fail every start with exit 126;
  382. the result is host-wide, so it is cached. Only DEFINITIVE answers are cached: a probe
  383. that could not run (auto-pull past the timeout, daemon cold-start, manifest/pull error)
  384. says nothing about cgroup support, so it degrades this spawn and is retried on the next."""
  385. global _cgroup_limits_ok
  386. if _cgroup_limits_ok is not None:
  387. return _cgroup_limits_ok
  388. docker_exe = find_docker()
  389. if not docker_exe or not image:
  390. return False # not cached: docker may appear later in this process
  391. try:
  392. result = run_capture(
  393. [docker_exe, "run", "--rm", "--cpus", "0.5", "--memory", "64m", "--pids-limit", "32",
  394. image, "sleep", "0"],
  395. timeout=60)
  396. except Exception as e:
  397. logger.warning("Cgroup limit probe failed; containers run without "
  398. "CPU/memory/PID limits until a probe succeeds: %s", e)
  399. return False
  400. if result.returncode == 0:
  401. _cgroup_limits_ok = True
  402. return True
  403. stderr = (result.stderr or "").strip()
  404. if "cgroup" not in stderr.lower():
  405. # Pull/manifest/daemon errors say nothing about cgroup support: not cached.
  406. logger.warning(
  407. "Cgroup limit probe could not determine support (docker exited %d: %s). "
  408. "Containers run without CPU/memory/PID limits until a probe succeeds.",
  409. result.returncode, stderr[:500])
  410. return False
  411. _cgroup_limits_ok = False
  412. logger.warning(
  413. "Cgroup resource limits (--cpus/--memory/--pids-limit) not "
  414. "available in this environment. Containers will run without "
  415. "CPU, memory or PID limits. To enable, delegate the cpu, "
  416. "memory and pids cgroup controllers to this container. Probe stderr: %s",
  417. stderr[:500])
  418. return False
  419. def _docker_unavailable(log_msg: str, *log_args, error: str, hint: str, exc_info: bool = False):
  420. logger.error(log_msg, *log_args, exc_info=exc_info)
  421. return EnvironmentConnectionError(error, retry_hint=hint)
  422. def _ensure_docker_available() -> None:
  423. """Fail fast with an ``EnvironmentConnectionError`` when the docker CLI/daemon is unusable."""
  424. docker_exe = find_docker()
  425. if not docker_exe:
  426. raise _docker_unavailable(
  427. "Docker backend selected but no docker executable was found in PATH "
  428. "or known install locations. Install Docker Desktop and ensure the CLI is available.",
  429. error="Docker executable not found in PATH or known install locations. "
  430. "Install Docker and ensure the 'docker' command is available.",
  431. hint="Install Docker (or fix PATH) and retry, or run `hermes setup terminal` to switch to Local.")
  432. try:
  433. result = run_capture([docker_exe, "version"], timeout=5)
  434. except FileNotFoundError:
  435. raise _docker_unavailable(
  436. "Docker backend selected but the resolved docker executable '%s' could not be executed.",
  437. docker_exe, exc_info=True,
  438. error="Docker executable could not be executed. Check your Docker installation.",
  439. hint="Repair the Docker installation and retry.")
  440. except subprocess.TimeoutExpired:
  441. raise _docker_unavailable(
  442. "Docker backend selected but '%s version' timed out. The Docker daemon may not be running.",
  443. docker_exe, exc_info=True,
  444. error="Docker daemon is not responding. Ensure Docker is running and try again.",
  445. hint="Start Docker (e.g. `systemctl start docker` or launch Docker Desktop), then retry — "
  446. "or run `hermes setup terminal` to switch to Local.")
  447. except Exception:
  448. logger.error("Unexpected error while checking Docker availability.", exc_info=True)
  449. raise
  450. if result.returncode != 0:
  451. raise _docker_unavailable(
  452. "Docker backend selected but '%s version' failed (exit code %d, stderr=%s)",
  453. docker_exe, result.returncode, result.stderr.strip(),
  454. error="Docker command is available but 'docker version' failed. Check your Docker installation.",
  455. hint="Start Docker, or add your user to the docker group, then retry — "
  456. "or run `hermes setup terminal` to switch to Local.")
  457. def _name_only_env_args(names) -> list[str]:
  458. """``-e KEY`` flags (no values): the docker CLI resolves values from its own env,
  459. so secrets live in owner-readable /proc/*/environ instead of world-readable cmdline."""
  460. return [arg for key in sorted(names) for arg in ("-e", key)]
  461. # Mount kinds declared by skills/credential_files: (getter, expects_file, log noun).
  462. _RO_MOUNT_SOURCES = (
  463. ("get_credential_file_mounts", True, "credential"),
  464. ("get_skills_directory_mount", False, "skills dir"),
  465. ("get_cache_directory_mounts", False, "cache dir"))
  466. def _readonly_skill_mount_args() -> list[str]:
  467. """``-v host:container:ro`` args for credential files, skill dirs and cache dirs. Read-only so the
  468. container can authenticate/read but never modify host state. Missing or wrong-kind sources are
  469. skipped with a warning (Docker-in-Docker auto-creates a missing file source as a directory,
  470. which would exit 125)."""
  471. args: list[str] = []
  472. try:
  473. import tools.credential_files as cf
  474. for getter, expects_file, noun in _RO_MOUNT_SOURCES:
  475. for entry in getattr(cf, getter)():
  476. src = Path(entry["host_path"])
  477. if expects_file:
  478. problem = ("source is a directory (likely Docker-in-Docker auto-creation)" if src.is_dir()
  479. else None if src.is_file() else "source not found")
  480. else:
  481. problem = None if src.is_dir() else "source is not a directory"
  482. if problem:
  483. logger.warning("Docker: skipping %s mount — %s: %s", noun.split()[0], problem, src)
  484. continue
  485. args.extend(["-v", f"{entry['host_path']}:{entry['container_path']}:ro"])
  486. logger.info("Docker: mounting %s %s -> %s", noun, entry["host_path"], entry["container_path"])
  487. except Exception as e:
  488. logger.debug("Docker: could not load credential file mounts: %s", e)
  489. return args
  490. def _host_user_args(run_as_host_user: bool) -> list[str]:
  491. """``--user uid:gid`` so bind-mount writes are owned by the host user, not root. Without
  492. POSIX ids fall back to the full cap set — the image's init may still need to drop privileges."""
  493. if not run_as_host_user:
  494. return []
  495. user_spec = _resolve_host_user_spec()
  496. if user_spec is not None:
  497. logger.info("Docker: running container as host user %s", user_spec)
  498. return ["--user", user_spec]
  499. logger.warning(
  500. "docker_run_as_host_user is enabled but this platform does "
  501. "not expose POSIX uid/gid; container will start as its image default user.")
  502. return []
  503. _VOLUME_SPEC_RE = re.compile(r"^(?P<host>.+):(?P<container>/[^:]+)(?::[^:]*)?$")
  504. # Second mount when a user volume already owns /workspace. Not a username.
  505. _HOST_CWD_FALLBACK_MOUNTS = ("/host-cwd", "/host-cwd-2", "/host-cwd-3")
  506. def _split_volume_spec(spec: str) -> tuple[str, str] | None:
  507. """``host:container[:mode]`` → ``(host, container)``. Drive-letter hosts keep their colon."""
  508. if not isinstance(spec, str):
  509. return None
  510. match = _VOLUME_SPEC_RE.match(spec.strip())
  511. if not match:
  512. return None
  513. return match.group("host"), match.group("container")
  514. def _container_mount_taken(volume_args: list[str], mount: str) -> bool:
  515. target = mount.rstrip("/") or "/"
  516. for arg in volume_args:
  517. parsed = _split_volume_spec(arg)
  518. if parsed and (parsed[1].rstrip("/") or "/") == target:
  519. return True
  520. return False
  521. def _existing_host_mount(volume_args: list[str], host_cwd_abs: str) -> str | None:
  522. """Container path if a user volume already bind-mounts this host directory."""
  523. want = _host_path_key(host_cwd_abs)
  524. if not want:
  525. return None
  526. for arg in volume_args:
  527. parsed = _split_volume_spec(arg)
  528. if parsed and _host_path_key(parsed[0]) == want:
  529. return parsed[1]
  530. return None
  531. def _free_host_cwd_mount(volume_args: list[str]) -> str:
  532. for candidate in _HOST_CWD_FALLBACK_MOUNTS:
  533. if not _container_mount_taken(volume_args, candidate):
  534. return candidate
  535. return _HOST_CWD_FALLBACK_MOUNTS[-1]
  536. def _abs_host_cwd(host_cwd: str) -> str:
  537. """Absolute host path. A Windows drive path is not prefixed with the POSIX process cwd."""
  538. expanded = os.path.expanduser(host_cwd)
  539. if _is_windows_drive_path(expanded) and os.name != "nt":
  540. return expanded
  541. return os.path.abspath(expanded)
  542. class DockerEnvironment(BaseEnvironment):
  543. """Hardened Docker container execution (caps dropped, no-new-privileges, PID limits,
  544. size-limited tmpfs). The container is the security boundary — its filesystem stays
  545. writable so agents can install packages. Persistence bind-mounts /workspace and /root."""
  546. _passwordless_sudo_probe_supported = True
  547. _profile_scoped_passthrough = True
  548. def _additional_profile_scoped_passthrough_names(self) -> tuple[str, ...]:
  549. """Keep explicit docker_forward_env values out of shared snapshots."""
  550. return tuple(self._forward_env)
  551. def __init__(
  552. self,
  553. image: str,
  554. cwd: str = "/root",
  555. timeout: int = 60,
  556. cpu: float = 0,
  557. memory: int = 0,
  558. disk: int = 0,
  559. persistent_filesystem: bool = False,
  560. task_id: str = "default",
  561. volumes: list | None = None,
  562. forward_env: list[str] | None = None,
  563. env: dict | None = None,
  564. network: bool = True,
  565. host_cwd: Optional[str] = None,
  566. auto_mount_cwd: bool = False,
  567. run_as_host_user: bool = False,
  568. extra_args: list | None = None,
  569. persist_across_processes: bool = True,
  570. shm_size: str = _DEFAULT_SHM_SIZE,
  571. shared_container_key: str = "",
  572. snap_compat: bool = False,
  573. image_pinned: bool = False):
  574. if cwd == "~":
  575. cwd = "/root"
  576. super().__init__(cwd=cwd, timeout=timeout)
  577. self._persistent = persistent_filesystem
  578. self._persist_across_processes = persist_across_processes
  579. # Set by terminal_tool._create_environment for session-scoped containers
  580. # (docker + container_persistent: false): removed at session close/idle timeout.
  581. self._session_scoped = False
  582. self._task_id = task_id
  583. self._forward_env = _normalize_forward_env_names(forward_env)
  584. self._env = _normalize_env_dict(env)
  585. self._init_unset_passthrough_names: tuple[str, ...] = ()
  586. self._container_id: Optional[str] = None
  587. self._init_env_values: dict[str, str] = {}
  588. self._workspace_dir: Optional[str] = None
  589. self._home_dir: Optional[str] = None
  590. # (host_dir, container_path) of the Hermes-created sandbox bind mounts; owner handback targets.
  591. self._sandbox_mounts: list[tuple[str, str]] = []
  592. logger.info("DockerEnvironment volumes: %s", volumes)
  593. if volumes is not None and not isinstance(volumes, list):
  594. logger.warning("docker_volumes config is not a list: %r", volumes)
  595. volumes = []
  596. _ensure_docker_available()
  597. resource_args = self._resource_args(image, cpu, memory, disk, network, shm_size, extra_args)
  598. volume_args, writable_args = self._mount_args(volumes, host_cwd, auto_mount_cwd, task_id)
  599. mount = getattr(self, "host_cwd_mount", None)
  600. if mount and cwd_follows_host_mount(cwd, mount):
  601. logger.info(
  602. "Container cwd follows configured host workspace at %s (requested %s)",
  603. mount, cwd)
  604. cwd = mount
  605. self.cwd = mount
  606. volume_args.extend(_readonly_skill_mount_args())
  607. egress_label, egress_volume_args, egress_host_args, env_args, validated_extra = (
  608. self._egress_and_env_args(extra_args))
  609. volume_args.extend(egress_volume_args)
  610. user_args = _host_user_args(run_as_host_user)
  611. # Resolved once so it works when /usr/local/bin is not in PATH (macOS services).
  612. self._docker_exe = find_docker() or "docker"
  613. # s6-overlay images (e.g. hermes-agent:latest) already use /init as PID 1 and exec
  614. # /run/s6/basedir/bin/init during startup. For those images we must (a) skip Docker's --init (two
  615. # competing PID-1 inits) and (b) mount /run with exec instead of noexec, or s6 stage0 dies with exit
  616. # 126 "Permission denied". Detected once here; defaults are kept on any inspection failure. See
  617. # issue #34628.
  618. image_uses_s6_init = _image_uses_init_entrypoint(self._docker_exe, image)
  619. if image_uses_s6_init:
  620. logger.info(
  621. "Docker: image %s uses /init (s6-overlay) as entrypoint — "
  622. "skipping --init and mounting /run with exec.",
  623. image)
  624. security_args = _build_security_args(
  625. run_as_host_user and bool(user_args), run_exec=image_uses_s6_init, snap_compat=snap_compat)
  626. self._snap_compat = snap_compat
  627. if snap_compat:
  628. logger.warning(
  629. "docker_snap_compat: running without --init and no-new-privileges (snap Docker under AppArmor)")
  630. logger.info("Docker volume_args: %s", volume_args)
  631. # docker_extra_args go last so they can override defaults.
  632. all_run_args = (
  633. security_args + user_args + writable_args + resource_args
  634. + egress_host_args + volume_args + env_args + validated_extra)
  635. logger.info("Docker run_args: %s", all_run_args)
  636. # Labels identify hermes containers to the orphan reaper (hermes-agent=1),
  637. # cross-process reuse (task-id/profile) and operators. The reuse identity
  638. # is captured at start and never changes for the container's lifetime.
  639. # Egress posture gets its own label: env/CA mounts are immutable after
  640. # creation, so reusing a pre-egress container would bypass the firewall.
  641. profile_name = _container_identity(shared_container_key)
  642. task_label = _sanitize_label_value(task_id)
  643. self._labels = {
  644. "hermes-agent": "1",
  645. "hermes-task-id": task_label,
  646. "hermes-profile": profile_name,
  647. _EGRESS_LABEL_KEY: egress_label}
  648. if workspace := _WORKSPACE_TASK_RE.search(task_id or ""):
  649. self._labels[_WORKSPACE_LABEL_KEY] = workspace.group(1)
  650. # Explicit sharing opts into the first creator's settings. Otherwise,
  651. # changed image/mount/home configuration must start a fresh container.
  652. if not shared_container_key:
  653. self._labels[_ENVIRONMENT_LABEL_KEY] = _reuse_environment_fingerprint(
  654. image=image, mount_args=[*writable_args, *volume_args],
  655. hermes_home=str(get_hermes_home()))
  656. # Saved for container recreation on "No such container" recovery.
  657. self._image = image
  658. self._image_pinned = image_pinned
  659. self._image_uses_s6_init = image_uses_s6_init
  660. self._all_run_args = all_run_args
  661. reused = persist_across_processes and self._attach_existing_container(
  662. task_label, profile_name, egress_label, network)
  663. if not reused:
  664. self._container_id = self._docker_run(cwd)
  665. else: # it never stopped, so no teardown stripped what it left
  666. strip_sandbox_privileges(self._docker_exe, self._container_id)
  667. reclaim_sandbox_files(self._docker_exe, self._container_id, self._sandbox_mounts)
  668. # Init-time env forwarding args seed the snapshot.
  669. self._init_env_args = self._build_init_env_args()
  670. self.init_session()
  671. # --- __init__ helpers ---
  672. def _egress_and_env_args(self, extra_args) -> tuple[str, list[str], list[str], list[str], list[str]]:
  673. """Egress credential-injection proxy plumbing (CA mount + HTTPS_PROXY/CA-bundle env so
  674. outbound traffic routes through the host-side proxy and the sandbox receives proxy tokens
  675. instead of real API keys), merged with docker_env into name-only ``-e`` args, plus the
  676. validated docker_extra_args. Returns ``(egress_label, volume_args, host_args, env_args,
  677. validated_extra)``; sets ``self._run_env_values`` (injected into the docker-client
  678. subprocess env at run time and reused verbatim by container-recreation recovery)."""
  679. egress_volume_args, egress_env_overrides, egress_host_args = _egress_proxy_args_for_docker()
  680. egress_label = _egress_reuse_fingerprint(egress_volume_args, egress_env_overrides, egress_host_args)
  681. enforce_egress = _egress_enforce_on_docker() if egress_env_overrides else True
  682. critical_egress_names = _critical_egress_env_names(egress_env_overrides)
  683. if egress_env_overrides:
  684. check_forward_env_collisions(self._forward_env, critical_egress_names, enforce_egress)
  685. check_docker_env_collisions(self._env, egress_env_overrides, enforce_egress)
  686. merged_env = merge_egress_env(self._env, egress_env_overrides, enforce_egress)
  687. self._run_env_values = dict(merged_env)
  688. validated_extra = []
  689. for arg in (extra_args or []):
  690. if not isinstance(arg, str):
  691. logger.warning("Ignoring non-string docker_extra_args entry: %r", arg)
  692. continue
  693. validated_extra.append(arg)
  694. if egress_env_overrides:
  695. check_extra_args_collisions(validated_extra, critical_egress_names, enforce_egress)
  696. return egress_label, egress_volume_args, egress_host_args, _name_only_env_args(merged_env), validated_extra
  697. def _resource_args(self, image, cpu, memory, disk, network, shm_size, extra_args) -> list[str]:
  698. """cgroup-gated CPU/memory/pids limits, shm size, disk quota and network mode."""
  699. args: list[str] = []
  700. if _cgroup_limits_available(image):
  701. if cpu > 0:
  702. args.extend(["--cpus", str(cpu)])
  703. if memory > 0:
  704. args.extend(["--memory", f"{memory}m"])
  705. args.extend(["--pids-limit", _DEFAULT_PIDS_LIMIT])
  706. # --shm-size is a tmpfs option, not cgroup-gated. Skipped when the user
  707. # sets it in docker_extra_args or opts out with empty/"0".
  708. shm = str(shm_size or "").strip()
  709. if shm and shm != "0" and not _extra_args_set_shm_size(extra_args):
  710. args.extend(["--shm-size", shm])
  711. if disk > 0 and sys.platform != "darwin":
  712. if self._storage_opt_supported():
  713. args.extend(["--storage-opt", f"size={disk}m"])
  714. else:
  715. logger.warning(
  716. "Docker storage driver does not support per-container disk limits "
  717. "(requires overlay2 on XFS with pquota). Container will run without disk quota.")
  718. if not network:
  719. extra_network = _extra_args_network_mode(extra_args)
  720. if extra_network is None:
  721. args.append("--network=none")
  722. elif extra_network != "none":
  723. # Contradictory intent: honouring the extra arg would hand the agent a networked
  724. # container despite the configured lockdown, and the reuse guard (NetworkMode ==
  725. # "none") would churn it every startup. Fail closed and name both keys.
  726. raise RuntimeError(
  727. f"terminal.docker_network is false (air-gapped) but terminal.docker_extra_args "
  728. f"requests --network={extra_network!r}. Remove the --network entry from "
  729. f"docker_extra_args, or set docker_network: true to opt into that network.")
  730. # extra_network == "none": same intent stated twice; the extra arg carries it once.
  731. return args
  732. def _mount_args(self, volumes, host_cwd, auto_mount_cwd, task_id) -> tuple[list[str], list[str]]:
  733. """``(volume_args, writable_args)`` for user volumes, host cwd and /workspace,/root.
  734. Persistent mode bind-mounts from TERMINAL_SANDBOX_DIR (default ~/.hermes/sandboxes/).
  735. A configured host working directory is bound even when another volume already
  736. claims ``/workspace``: at ``/workspace`` when that path is free, otherwise at
  737. a second mount. ``host_cwd`` / ``host_cwd_mount`` tell tools which container
  738. path is that directory. A Windows drive path is bound whenever it exists on
  739. the host — it can never be a path inside the Linux container, and the check
  740. is the drive shape, not a username.
  741. """
  742. volume_args: list[str] = []
  743. for vol in (volumes or []):
  744. if not isinstance(vol, str):
  745. logger.warning("Docker volume entry is not a string: %r", vol)
  746. continue
  747. vol = vol.strip()
  748. if not vol:
  749. continue
  750. if ":" not in vol:
  751. logger.warning("Docker volume '%s' missing colon, skipping", vol)
  752. continue
  753. volume_args.extend(["-v", vol])
  754. workspace_explicitly_mounted = any(":/workspace" in v for v in volume_args)
  755. host_cwd_abs = _abs_host_cwd(host_cwd) if host_cwd else ""
  756. windows_cwd = _is_windows_drive_path(host_cwd or "") or _is_windows_drive_path(host_cwd_abs)
  757. host_dir_exists = bool(host_cwd_abs) and os.path.isdir(host_cwd_abs)
  758. should_bind = host_dir_exists and (auto_mount_cwd or windows_cwd)
  759. if (auto_mount_cwd or windows_cwd) and host_cwd and not host_dir_exists:
  760. logger.debug("Skipping docker cwd mount: host_cwd is not a valid directory: %s", host_cwd)
  761. existing_mount = _existing_host_mount(volume_args, host_cwd_abs) if should_bind else None
  762. if existing_mount:
  763. # Already bind-mounted (often the volume that claimed /workspace). Point
  764. # tools at that container path instead of adding a second -v.
  765. self.host_cwd = host_cwd_abs
  766. self.host_cwd_mount = existing_mount
  767. bind_target = None
  768. elif should_bind:
  769. bind_target = (
  770. "/workspace" if not workspace_explicitly_mounted
  771. else _free_host_cwd_mount(volume_args))
  772. self.host_cwd = host_cwd_abs
  773. self.host_cwd_mount = bind_target
  774. else:
  775. self.host_cwd = None
  776. self.host_cwd_mount = None
  777. bind_target = None
  778. bind_at_workspace = bind_target == "/workspace"
  779. mount_workspace = not bind_at_workspace and not workspace_explicitly_mounted
  780. writable_args: list[str] = []
  781. if self._persistent:
  782. # _sandbox_dir_name(): a raw session-key task_id carries colons,
  783. # which `-v` reads as extra spec fields (exit 125).
  784. sandbox = get_sandbox_dir() / "docker" / _sandbox_dir_name(task_id)
  785. self._home_dir = str(sandbox / "home")
  786. os.makedirs(self._home_dir, exist_ok=True)
  787. self._sandbox_mounts = [(self._home_dir, "/root")]
  788. if mount_workspace:
  789. self._workspace_dir = str(sandbox / "workspace")
  790. os.makedirs(self._workspace_dir, exist_ok=True)
  791. self._sandbox_mounts.append((self._workspace_dir, "/workspace"))
  792. for host_dir, container_path in self._sandbox_mounts:
  793. writable_args += ["-v", f"{host_dir}:{container_path}"]
  794. else:
  795. writable_args += ["--tmpfs", "/workspace:rw,exec,size=10g"] if mount_workspace else []
  796. writable_args += ["--tmpfs", "/home:rw,exec,size=1g", "--tmpfs", "/root:rw,exec,size=1g"]
  797. if bind_target:
  798. logger.info("Mounting configured host cwd to %s: %s", bind_target, host_cwd_abs)
  799. volume_args = ["-v", f"{host_cwd_abs}:{bind_target}", *volume_args]
  800. elif workspace_explicitly_mounted and not existing_mount:
  801. logger.debug("Skipping docker cwd mount: /workspace already mounted by user config")
  802. return volume_args, writable_args
  803. def _attach_existing_container(self, task_label, profile_name, egress_label, network: bool) -> bool:
  804. """Attach to a prior process's labeled container ("ONE long-lived container shared
  805. across sessions"; opt out via ``docker_persist_across_processes: false``).
  806. Network guard is lockdown-only: a bridge container under ``docker_network: false``
  807. is removed and recreated, but a ``none`` container under default config is kept so
  808. ``--network=none`` in extra args doesn't churn containers every startup."""
  809. existing = self._find_reusable_container(task_label, profile_name, egress_label)
  810. if existing is None:
  811. return False
  812. container_id, state = existing
  813. # A container built from another image. Explicitly configured image (config.yaml /
  814. # TERMINAL_DOCKER_IMAGE): the user changed it, so the old container is not their sandbox any
  815. # more — recreate (the image is immutable after creation). Default image: a default flip
  816. # (nikolaik base -> hermes-sandbox:desktop) must not replace a sandbox someone has state in;
  817. # keep it and let the CLI / Screen pane ask. Same rule Modal (snapshot wins) and Daytona
  818. # (labeled sandbox wins) already apply.
  819. actual_image = self._container_image(container_id)
  820. if actual_image is not None and actual_image != self._image:
  821. if not self._image_pinned:
  822. logger.warning(
  823. "Existing container %s runs image %s; the default docker_image is now %s. Keeping "
  824. "the existing sandbox — approve the switch with `hermes config set "
  825. "terminal.docker_image %s` (files in /root and /workspace carry over) or pin the "
  826. "current image to stop this notice (task=%s, profile=%s).",
  827. container_id[:12], actual_image, self._image, self._image, task_label, profile_name)
  828. elif not self._image_available_locally():
  829. # The replacement image cannot be had (private/misspelled tag, registry down, pull past
  830. # its timeout). Removing the old container first would throw away its writable layer with
  831. # nothing to put in its place, so keep running the sandbox the user has and retry the
  832. # switch next time the image resolves.
  833. logger.warning(
  834. "Existing container %s runs image %s; docker_image is %s but that image could not be "
  835. "pulled — keeping the current sandbox until it can (task=%s, profile=%s).",
  836. container_id[:12], actual_image, self._image, task_label, profile_name)
  837. else:
  838. logger.warning(
  839. "Existing container %s runs image %s but docker_image is %s — removing it and "
  840. "starting fresh (task=%s, profile=%s).",
  841. container_id[:12], actual_image, self._image, task_label, profile_name)
  842. try:
  843. run_capture([self._docker_exe, "rm", "-f", container_id], timeout=30)
  844. except (subprocess.TimeoutExpired, OSError) as e:
  845. logger.warning("Failed to remove mismatched container %s: %s", container_id[:12], e)
  846. return False
  847. if not network:
  848. actual_mode = self._container_network_mode(container_id)
  849. if actual_mode != "none":
  850. logger.warning(
  851. "Existing container %s has NetworkMode=%s but "
  852. "docker_network=false requests an air-gapped "
  853. "container — removing it and starting fresh (task=%s, profile=%s).",
  854. container_id[:12], actual_mode or "unknown", task_label, profile_name)
  855. try:
  856. run_capture([self._docker_exe, "rm", "-f", container_id], timeout=30)
  857. except (subprocess.TimeoutExpired, OSError) as e:
  858. logger.warning("Failed to remove mismatched container %s: %s", container_id[:12], e)
  859. return False
  860. if state != "running":
  861. err = self._start_container(container_id)
  862. if err is not None:
  863. logger.warning(
  864. "Failed to start existing container %s (state=%s): "
  865. "%s — falling back to a fresh container.",
  866. container_id[:12], state, err)
  867. return False
  868. self._container_id = container_id
  869. logger.info(
  870. "Reusing container %s (task=%s, profile=%s, prior state=%s)",
  871. container_id[:12], task_label, profile_name, state)
  872. return True
  873. def _image_available_locally(self) -> bool:
  874. """True once ``self._image`` is in the local image store, pulling it when it is not. Called BEFORE a
  875. container replacement removes anything: a pull that fails must leave the old sandbox intact."""
  876. try:
  877. probe = run_capture([self._docker_exe, "image", "inspect", self._image, "--format", "{{.Id}}"],
  878. timeout=30)
  879. if probe.returncode == 0:
  880. return True
  881. pull = run_capture([self._docker_exe, "pull", self._image], timeout=900)
  882. return pull.returncode == 0
  883. except (subprocess.SubprocessError, OSError) as e:
  884. logger.warning("Docker: could not pull %s: %s", self._image, e)
  885. return False
  886. def _start_container(self, container_id: str) -> Exception | None:
  887. """``docker start`` a stopped container; returns the failure instead of raising."""
  888. try:
  889. run_capture([self._docker_exe, "start", container_id], timeout=30, check=True)
  890. return None
  891. except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
  892. return e
  893. def _run_command(self, name: str, workdir: str) -> list[str]:
  894. """``docker run -d`` argv for a fresh ``sleep infinity`` container (idle reaper handles
  895. lifetime). s6-overlay images already provide PID 1, so ``--init`` is skipped for them."""
  896. label_args = [arg for k, v in self._labels.items() for arg in ("--label", f"{k}={v}")]
  897. return [
  898. # tini/catatonit as PID 1 reaps zombie children — but s6-overlay images already provide their
  899. # own /init PID 1, so adding --init there creates two competing inits and breaks startup
  900. # (#34628).
  901. self._docker_exe, "run", "-d",
  902. *([] if self._image_uses_s6_init or self._snap_compat else ["--init"]),
  903. "--name", name,
  904. *label_args,
  905. "-w", workdir,
  906. *self._all_run_args,
  907. self._image,
  908. "sleep", "infinity"]
  909. def _docker_run(self, cwd: str) -> str:
  910. """Start a fresh container and return its id. A failed ``docker run`` (exit 125, timeout
  911. mid-pull) can leave a "Created" orphan the exited-only reaper never catches, so it is
  912. removed by name before re-raising."""
  913. container_name = f"hermes-{uuid.uuid4().hex[:8]}"
  914. run_cmd = self._run_command(container_name, cwd)
  915. logger.debug("Starting container: %s", ' '.join(run_cmd))
  916. try:
  917. result = run_capture(
  918. run_cmd, timeout=120, check=True, # image pull may take a while
  919. env=self._docker_client_env(self._run_env_values))
  920. except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
  921. logger.warning("docker run failed for %s, cleaning up orphaned container: %s", container_name, e)
  922. subprocess.run(
  923. [self._docker_exe, "rm", "-f", container_name],
  924. capture_output=True, timeout=10, stdin=subprocess.DEVNULL, check=False)
  925. raise
  926. container_id = result.stdout.strip()
  927. logger.info("Started container %s (%s)", container_name, container_id[:12])
  928. return container_id
  929. # --- Env forwarding ---
  930. def _docker_client_env(self, values: dict[str, str]) -> dict[str, str] | None:
  931. """Env for the docker-client subprocess carrying forwarded values (pairs with name-only
  932. ``-e KEY`` flags to keep secrets out of cmdline); ``None`` = inherit when empty.
  933. Name-only ``-e KEY`` flags make the docker CLI read each value from its own process environment,
  934. keeping secrets out of the client's world-readable ``/proc/<pid>/cmdline`` (issue #96268). Values
  935. live in ``/proc/<pid>/environ`` instead, which is owner/root-only. Returns ``None`` (inherit as
  936. before) when there is nothing to add.
  937. """
  938. return client_env_with(values)
  939. def _build_init_env_args(self) -> list[str]:
  940. """Name-only ``-e`` args for init_session so ``export -p`` captures docker_env
  941. plus the current profile's forwarded values (values travel via the client env).
  942. The VALUES intentionally do not appear in the argv — they are passed via the docker client
  943. subprocess env (see _docker_client_env and issue #96268); the flags here are name-only ``-e KEY``.
  944. """
  945. passthrough_env, unset_names = self._resolve_passthrough_env()
  946. exec_env = {**self._env, **passthrough_env}
  947. for name in unset_names:
  948. exec_env.pop(name, None)
  949. self._init_unset_passthrough_names = tuple(sorted(unset_names))
  950. self._init_env_values = dict(exec_env)
  951. return _name_only_env_args(exec_env)
  952. def _resolve_passthrough_env(self) -> tuple[dict[str, str], set[str]]:
  953. """See ``remote_common.resolve_passthrough_env``; explicit docker_forward_env bypasses the blocklist."""
  954. return resolve_passthrough_env(self._forward_env, hermes_env_loader=_load_hermes_env_vars)
  955. def _build_runtime_env_args_with_unsets(self) -> tuple[list[str], tuple[str, ...], dict[str, str]]:
  956. """Runtime name-only forwarding args, names absent from scope, and the values
  957. to inject into the docker client subprocess env.
  958. See #96268.
  959. """
  960. passthrough_env, unset_names = self._resolve_passthrough_env()
  961. return _name_only_env_args(passthrough_env), tuple(sorted(unset_names)), dict(passthrough_env)
  962. def _run_bash(self, cmd_string: str, *, login: bool = False,
  963. timeout: int = 120,
  964. stdin_data: str | None = None) -> subprocess.Popen:
  965. """Spawn bash inside the container. Init seeds the snapshot; profile-scoped passthrough
  966. values are re-injected on every command because one container can be shared by
  967. multiple routed profiles in a gateway process."""
  968. assert self._container_id, "Container not started"
  969. cmd = [self._docker_exe, "exec"]
  970. if stdin_data is not None:
  971. cmd.append("-i")
  972. # Init seeds the snapshot. Profile-scoped passthrough values are also injected on every later
  973. # command because this container can be shared by multiple routed profiles in one gateway process.
  974. # Env flags are name-only; values travel via the client subprocess env so they never hit
  975. # world-readable /proc/*/cmdline (#96268).
  976. unset_names: tuple[str, ...] = ()
  977. env_values: dict[str, str] = {}
  978. if login:
  979. cmd.extend(self._init_env_args)
  980. env_values = dict(getattr(self, "_init_env_values", {}))
  981. unset_names = getattr(self, "_init_unset_passthrough_names", ())
  982. elif self._profile_scoped_passthrough:
  983. runtime_args, unset_names, env_values = self._build_runtime_env_args_with_unsets()
  984. cmd.extend(runtime_args)
  985. cmd += [self._container_id, *bash_argv(prepend_unset(cmd_string, unset_names), login)]
  986. client_env = self._docker_client_env(env_values)
  987. return _popen_bash(cmd, stdin_data, env=client_env) if client_env is not None else _popen_bash(cmd, stdin_data)
  988. # --- "No such container" recovery ---
  989. _NO_CONTAINER_PATTERNS = ("No such container", "is not running", "no such container")
  990. def _is_container_gone(self, output: str) -> bool:
  991. return any(p in output for p in self._NO_CONTAINER_PATTERNS)
  992. def _recreate_container(self) -> bool:
  993. """Recreate a container removed out-of-band: label-based reuse first (another process
  994. may have recreated it), else a fresh one from the saved image/run-args. False when
  995. recovery fails so the caller surfaces the original error."""
  996. logger.warning("Container %s appears to be gone — attempting recovery", (self._container_id or "")[:12])
  997. self._container_id = None
  998. existing = self._find_reusable_container(
  999. self._labels.get("hermes-task-id", ""),
  1000. self._labels.get("hermes-profile", ""),
  1001. self._labels.get(_EGRESS_LABEL_KEY, "off"))
  1002. if existing is not None:
  1003. cid, state = existing
  1004. if state == "running":
  1005. self._container_id = cid
  1006. logger.info("Recovery: reusing running container %s", cid[:12])
  1007. elif (err := self._start_container(cid)) is None:
  1008. self._container_id = cid
  1009. logger.info("Recovery: restarted container %s", cid[:12])
  1010. else:
  1011. logger.warning("Recovery: failed to start container %s: %s", cid[:12], err)
  1012. if not self._container_id:
  1013. if not self._image:
  1014. logger.error("Recovery: no saved image name, cannot recreate container")
  1015. return False
  1016. try:
  1017. new_name = f"hermes-{uuid.uuid4().hex[:8]}"
  1018. result = run_capture(
  1019. self._run_command(new_name, self.cwd), timeout=120, check=True,
  1020. env=self._docker_client_env(self._run_env_values))
  1021. self._container_id = result.stdout.strip()
  1022. logger.info("Recovery: created fresh container %s (%s)", new_name, self._container_id[:12])
  1023. except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e:
  1024. logger.error("Recovery: failed to create new container: %s", e)
  1025. return False
  1026. try:
  1027. self._snapshot_ready = False
  1028. self.init_session()
  1029. except Exception as e:
  1030. logger.error("Recovery: init_session failed in new container: %s", e)
  1031. return False
  1032. logger.info("Recovery successful — new container %s", (self._container_id or "")[:12])
  1033. self._mark_recreated()
  1034. return True
  1035. def execute(self, command: str, cwd: str = "", **kwargs) -> dict:
  1036. """Execute a command; if the container was removed out-of-band (idle reaper,
  1037. docker prune, OOM, daemon restart) recreate it and retry once."""
  1038. result = super().execute(command, cwd, **kwargs)
  1039. if (
  1040. result.get("returncode", 0) != 0
  1041. and self._is_container_gone(result.get("output", ""))
  1042. and self._persist_across_processes
  1043. and self._recreate_container()):
  1044. result = super().execute(command, cwd, **kwargs)
  1045. return result
  1046. @staticmethod
  1047. def _storage_opt_supported() -> bool:
  1048. """Whether ``--storage-opt size=`` works (only overlay2 on XFS with pquota; ext4 errors out).
  1049. Only definitive answers are cached: a probe that could not run (daemon cold-start,
  1050. hello-world pull timeout) says nothing about pquota support and is retried next spawn."""
  1051. global _storage_opt_ok
  1052. if _storage_opt_ok is not None:
  1053. return _storage_opt_ok
  1054. try:
  1055. docker = find_docker() or "docker"
  1056. result = run_capture([docker, "info", "--format", "{{.Driver}}"], timeout=10)
  1057. if result.returncode != 0:
  1058. return False # daemon unreachable etc. is transient; retry next spawn
  1059. if result.stdout.strip().lower() != "overlay2":
  1060. _storage_opt_ok = False # storage driver is a host property
  1061. return False
  1062. # Probe with a real create — the fastest reliable check.
  1063. probe = run_capture([docker, "create", "--storage-opt", "size=1m", "hello-world"], timeout=15)
  1064. if probe.returncode == 0:
  1065. _storage_opt_ok = True
  1066. if probe.stdout.strip():
  1067. subprocess.run([docker, "rm", probe.stdout.strip()],
  1068. capture_output=True, timeout=5, stdin=subprocess.DEVNULL, check=False)
  1069. elif "storage" in (probe.stderr or "").lower():
  1070. _storage_opt_ok = False # daemon rejected --storage-opt: a host property
  1071. # else: pull/daemon failure unrelated to storage-opt; not cached, retried next spawn
  1072. except Exception:
  1073. return False # TimeoutExpired, missing binary; transient, retried next spawn
  1074. logger.debug("Docker --storage-opt support: %s", _storage_opt_ok)
  1075. return _storage_opt_ok or False
  1076. def _container_image(self, container_id: str) -> Optional[str]:
  1077. """The image reference a container was created from (``Config.Image``: the tag as given
  1078. to ``docker run``, so it compares directly with ``docker_image``), or ``None`` when
  1079. inspection fails (callers then keep the container: a failed probe must not churn)."""
  1080. result = _docker_query(
  1081. [self._docker_exe, "inspect", "--format", "{{.Config.Image}}", container_id], timeout=10,
  1082. fail="docker inspect Image failed: %s", nonzero="docker inspect Image returned %d: %s")
  1083. return (result.stdout.strip() or None) if result is not None else None
  1084. def _container_network_mode(self, container_id: str) -> Optional[str]:
  1085. """``HostConfig.NetworkMode`` of a container, or ``None`` when inspection fails (callers
  1086. treat ``None`` as a mismatch under lockdown, so a failed inspect fails closed)."""
  1087. result = _docker_query(
  1088. [self._docker_exe, "inspect", "--format", "{{.HostConfig.NetworkMode}}", container_id], timeout=10,
  1089. fail="docker inspect NetworkMode failed: %s", nonzero="docker inspect NetworkMode returned %d: %s")
  1090. return (result.stdout.strip() or None) if result is not None else None
  1091. def _find_reusable_container(
  1092. self, task_label: str, profile_label: str, egress_label: str) -> Optional[tuple[str, str]]:
  1093. """``(container_id, state)`` of an existing container labeled for this task/profile/
  1094. egress posture and immutable environment, or ``None`` on miss or any failure.
  1095. Explicit shared keys opt out of the environment filter. The egress posture is a label
  1096. FILTER for every posture, "off" included: a container built with egress on must not be
  1097. reused after ``hermes egress disable`` (baked-in proxy env and CA mounts), and every
  1098. container this class creates carries the label. The ``{{.Label "key"}}`` template
  1099. function is Docker-only — podman ps exits 125 on it — so the probe never uses it (#99213)."""
  1100. filters = [
  1101. "--filter", "label=hermes-agent=1",
  1102. "--filter", f"label=hermes-task-id={task_label}",
  1103. "--filter", f"label=hermes-profile={profile_label}",
  1104. "--filter", f"label={_EGRESS_LABEL_KEY}={egress_label}"]
  1105. if environment_label := self._labels.get(_ENVIRONMENT_LABEL_KEY):
  1106. filters.extend(["--filter", f"label={_ENVIRONMENT_LABEL_KEY}={environment_label}"])
  1107. result = _docker_query(
  1108. [self._docker_exe, "ps", "-a", *filters, "--format", "{{.ID}}\t{{.State}}"], timeout=10,
  1109. fail="docker ps probe failed: %s — will start a fresh container",
  1110. nonzero="docker ps probe returned %d: %s — will start a fresh container")
  1111. if result is None:
  1112. return None
  1113. # Multiple matches can happen after a crash mid-cleanup: prefer a running
  1114. # one, else the first listed; stale duplicates are the orphan reaper's job.
  1115. running = first = None
  1116. for ln in (ln for ln in result.stdout.splitlines() if ln.strip()):
  1117. parts = ln.split("\t", 1)
  1118. if len(parts) != 2:
  1119. continue
  1120. cid, state = parts[0], parts[1].strip().lower()
  1121. if first is None:
  1122. first = (cid, state)
  1123. if state == "running" and running is None:
  1124. running = (cid, state)
  1125. return running or first
  1126. def _remove_bind_dirs(self) -> None:
  1127. for d in (self._workspace_dir, self._home_dir):
  1128. if d:
  1129. shutil.rmtree(d, ignore_errors=True)
  1130. def cleanup(self, *, force_remove: bool = False):
  1131. """Tear down per persist mode. Persist mode (default) leaves the container RUNNING —
  1132. stopping it on every exit would kill background processes and add a ``docker start``
  1133. delay per session; reclamation is ``reap_orphan_containers()`` at next startup.
  1134. ``persist_across_processes=False`` or ``force_remove=True`` (explicit-teardown hook,
  1135. unused so far) does ``docker stop`` + ``docker rm -f`` on a daemon thread that the
  1136. atexit hook joins via ``wait_for_cleanup`` so the work completes before exit.
  1137. Cleanup runs on a daemon thread with bounded ``subprocess.run`` calls (not the racy ``Popen(... &)``
  1138. pattern from before PR #33645). The atexit hook in ``tools/terminal_tool.py`` waits up to 15s for
  1139. the thread to finish before the interpreter exits, so ``docker stop`` / ``docker rm`` actually
  1140. completes when we do trigger it.
  1141. """
  1142. container_id = self._container_id
  1143. if not container_id:
  1144. # Bind-mount dirs are still dropped in non-persistent mode.
  1145. if not self._persistent:
  1146. self._remove_bind_dirs()
  1147. return
  1148. # Capture what the worker needs — the thread can outlive ``self``.
  1149. docker_exe = self._docker_exe
  1150. log_id = container_id[:12]
  1151. sandbox_mounts = list(getattr(self, "_sandbox_mounts", []))
  1152. if not force_remove and self._persist_across_processes:
  1153. # Drop the in-process handle so a fresh __init__ re-probes via
  1154. # labels instead of reusing a stale Python reference.
  1155. self._container_id = None
  1156. # The host user may run what the container left before it is reattached.
  1157. self._start_teardown(lambda: strip_sandbox_privileges(docker_exe, container_id), log_id)
  1158. return
  1159. def _do_cleanup() -> None:
  1160. for argv, fail_msg in ((["stop", "-t", "10"], "docker stop %s timed out / failed: %s"),
  1161. (["rm", "-f"], "docker rm -f %s failed: %s")):
  1162. if argv[0] == "rm" and sandbox_mounts: # stopped: no writer left, volumes still reachable
  1163. hand_back_sandbox_files(docker_exe, container_id, sandbox_mounts)
  1164. try:
  1165. subprocess.run(
  1166. [docker_exe, *argv, container_id],
  1167. capture_output=True, timeout=30, stdin=subprocess.DEVNULL, check=False)
  1168. except (subprocess.TimeoutExpired, OSError) as e:
  1169. logger.warning(fail_msg, log_id, e)
  1170. self._start_teardown(_do_cleanup, log_id)
  1171. self._container_id = None
  1172. # Bind-mount dirs are the container's filesystem state; only drop them
  1173. # once the container itself is removed.
  1174. if not self._persistent:
  1175. self._remove_bind_dirs()
  1176. def stop_idle(self) -> None:
  1177. """Idle reclaim of a per-workspace persistent container (#140665): ``docker stop``, never
  1178. ``rm``. Its writable layer and ``/root`` sandbox stay, and the next use in that workspace
  1179. re-attaches through :meth:`_attach_existing_container`, which ``docker start``s a stopped
  1180. container. Without this, persistent mode left one RUNNING container per workspace a user
  1181. ever opened (each with its own ``container_memory``) until the process exited. Like a
  1182. persist-mode cleanup, the host user may touch what it left before it is reattached, so its
  1183. sandbox privileges are stripped once it has stopped."""
  1184. container_id = self._container_id
  1185. if not container_id:
  1186. return
  1187. docker_exe, log_id = self._docker_exe, container_id[:12]
  1188. def _do_stop() -> None:
  1189. try:
  1190. subprocess.run([docker_exe, "stop", "-t", "10", container_id], capture_output=True,
  1191. timeout=30, stdin=subprocess.DEVNULL, check=False)
  1192. except (subprocess.TimeoutExpired, OSError) as e:
  1193. logger.warning("docker stop %s (idle workspace) failed: %s", log_id, e)
  1194. strip_sandbox_privileges(docker_exe, container_id)
  1195. self._start_teardown(_do_stop, log_id)
  1196. self._container_id = None
  1197. def _start_teardown(self, work, log_id: str) -> None:
  1198. """Run *work* on a daemon thread the atexit drain joins."""
  1199. t = threading.Thread(target=work, daemon=True, name=f"hermes-cleanup-{log_id}")
  1200. with _TEARDOWN_LOCK:
  1201. _TEARDOWN_THREADS.add(t)
  1202. t.start()
  1203. self._cleanup_thread = t
  1204. @staticmethod
  1205. def wait_for_all_teardowns(timeout: float = 15.0) -> bool:
  1206. """Join every in-flight teardown worker, including those of envs the idle reaper already
  1207. popped out of the active registry (their handles are otherwise unreachable at exit, so
  1208. ``docker rm`` died with the interpreter and left a stopped container behind — #86317).
  1209. Re-snapshots each pass: the reaper may start a worker while the drain runs."""
  1210. deadline = time.monotonic() + timeout
  1211. while True:
  1212. with _TEARDOWN_LOCK:
  1213. _TEARDOWN_THREADS.difference_update({t for t in _TEARDOWN_THREADS if not t.is_alive()})
  1214. pending = list(_TEARDOWN_THREADS)
  1215. if not pending:
  1216. return True
  1217. remaining = deadline - time.monotonic()
  1218. if remaining <= 0:
  1219. return False
  1220. pending[0].join(timeout=remaining)
  1221. def wait_for_cleanup(self, timeout: float = 30.0) -> bool:
  1222. """Block up to *timeout* seconds for the cleanup thread (atexit hook). True if it
  1223. finished or none was started, False on timeout."""
  1224. thread = getattr(self, "_cleanup_thread", None)
  1225. if thread is None or not thread.is_alive():
  1226. return True
  1227. thread.join(timeout=timeout)
  1228. return not thread.is_alive()