credential_files.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409
  1. """File passthrough registry for remote terminal backends (Docker, Modal, SSH).
  2. Sandboxes start with no host files; this module tells them which credential files
  3. (skill ``required_credential_files`` + ``terminal.credential_files`` config), skill
  4. dirs, and host cache dirs to mount or sync in, at creation and before each command.
  5. """
  6. from __future__ import annotations
  7. import logging
  8. import os
  9. import posixpath
  10. import stat
  11. import time
  12. from contextvars import ContextVar
  13. from pathlib import Path
  14. from typing import Callable, Dict, Iterator, List, Optional, Tuple
  15. from hermes_cli.config import cfg_get
  16. from hermes_constants import get_hermes_dir, get_hermes_home
  17. from agent.provider_media import GENERATED_SUBDIR, MEDIA_CACHE_MAX_AGE_HOURS
  18. from agent.skill_utils import EXCLUDED_SKILL_DIRS
  19. try: # pragma: no cover - exercised via the fail-closed test below
  20. from agent.file_safety import get_read_block_error
  21. except ImportError:
  22. get_read_block_error = None # type: ignore[assignment]
  23. logger = logging.getLogger(__name__)
  24. # Session-scoped registry; ContextVar prevents cross-session bleed in the gateway.
  25. _registered_files_var: ContextVar[dict[str, str]] = ContextVar("_registered_files")
  26. # Cache for config-based file list, one entry per profile home (tests reset it).
  27. _config_files: dict[str, list[dict[str, str]]] = {}
  28. # Reused across calls so sanitized skill copies don't accumulate.
  29. _safe_skills_tempdir: Path | None = None
  30. def _get_registered() -> dict[str, str]:
  31. val = _registered_files_var.get(None)
  32. if val is None:
  33. _registered_files_var.set(val := {})
  34. return val
  35. def _mount(host_path: Path | str, container_path: str) -> dict[str, str]:
  36. return {"host_path": str(host_path), "container_path": container_path}
  37. def _contained_host_path(rel: str, hermes_home: Path, abs_msg: str, traversal_msg: str) -> Optional[Path]:
  38. """Resolve *rel* under HERMES_HOME, refusing absolute paths and escapes."""
  39. if os.path.isabs(rel):
  40. logger.warning(abs_msg, rel)
  41. return None
  42. host_path = hermes_home / rel
  43. from tools.path_security import validate_within_dir # resolves symlinks and ``..`` before checking
  44. if containment_error := validate_within_dir(host_path, hermes_home):
  45. logger.warning(traversal_msg, rel, containment_error)
  46. return None
  47. return host_path.resolve()
  48. def register_credential_file(relative_path: str, container_base: str = "/root/.hermes") -> bool:
  49. """Register a HERMES_HOME-relative credential file for mounting; True if it exists and was registered.
  50. Rejects absolute paths and traversal out of HERMES_HOME. Containment alone is not
  51. enough: HERMES_HOME holds the MASTER stores (``.env``, ``auth.json``, ``mcp-tokens/``),
  52. which are refused via the canonical read deny-list so the mount surface cannot hand a
  53. skill what the read surface denies. Fails CLOSED (logged) if the guard is unavailable or raises.
  54. """
  55. resolved = _contained_host_path(
  56. relative_path, get_hermes_home(),
  57. "credential_files: rejected absolute path %r (must be relative to HERMES_HOME)",
  58. "credential_files: rejected path traversal %r (%s)")
  59. if resolved is None:
  60. return False
  61. if not resolved.is_file():
  62. logger.debug("credential_files: skipping %s (not found)", resolved)
  63. return False
  64. # Master credential stores are never mountable, even though they sit inside HERMES_HOME and therefore
  65. # pass the containment check above. Fails CLOSED: if the canonical guard can't be consulted we refuse
  66. # the mount rather than risk bind-mounting auth.json into a sandbox. The import lives at module top (no
  67. # circular-import concern — file_safety is stdlib-only); the sentinel + logger.exception keep guard
  68. # failures debuggable instead of silently swallowed (#67665).
  69. if get_read_block_error is None:
  70. logger.error("credential_files: refusing %r — agent.file_safety could not be "
  71. "imported, so the master-store deny-list cannot be consulted", relative_path)
  72. return False
  73. try:
  74. denied = get_read_block_error(str(resolved))
  75. except Exception:
  76. logger.exception("credential_files: refusing %r — read guard raised", relative_path)
  77. return False
  78. if denied:
  79. logger.warning("credential_files: refused %r — it is a credential store the agent "
  80. "is denied from reading; a skill may mount its own service token, "
  81. "not the master key files", relative_path)
  82. return False
  83. container_path = f"{container_base.rstrip('/')}/{relative_path}"
  84. _get_registered()[container_path] = str(resolved)
  85. logger.debug("credential_files: registered %s -> %s", resolved, container_path)
  86. return True
  87. def register_credential_files(entries: list, container_base: str = "/root/.hermes") -> list[str]:
  88. """Register skill-frontmatter entries (str or dict with ``path``); return missing paths."""
  89. missing = []
  90. for entry in entries:
  91. if isinstance(entry, dict):
  92. entry = entry.get("path") or entry.get("name") or ""
  93. elif not isinstance(entry, str):
  94. continue
  95. rel_path = entry.strip()
  96. if rel_path and not register_credential_file(rel_path, container_base):
  97. missing.append(rel_path)
  98. return missing
  99. def _load_config_files() -> list[dict[str, str]]:
  100. """Load ``terminal.credential_files`` from config.yaml (cached per profile home: the
  101. multiplexed gateway must never mount the launch profile's credential files into a
  102. secondary profile's sandbox)."""
  103. from hermes_constants import hermes_home_key
  104. home_key = hermes_home_key()
  105. cached = _config_files.get(home_key)
  106. if cached is not None:
  107. return cached
  108. result: list[dict[str, str]] = []
  109. try:
  110. from hermes_cli.config import read_raw_config
  111. hermes_home = get_hermes_home()
  112. cred_files = cfg_get(read_raw_config(), "terminal", "credential_files")
  113. for item in cred_files if isinstance(cred_files, list) else []:
  114. rel = item.strip() if isinstance(item, str) else ""
  115. if not rel:
  116. continue
  117. resolved_path = _contained_host_path(
  118. rel, hermes_home,
  119. "credential_files: rejected absolute config path %r",
  120. "credential_files: rejected config path traversal %r (%s)")
  121. if resolved_path is not None and resolved_path.is_file():
  122. result.append(_mount(resolved_path, f"/root/.hermes/{rel}"))
  123. except Exception as e:
  124. logger.warning("Could not read terminal.credential_files from config: %s", e)
  125. _config_files[home_key] = result
  126. return result
  127. def get_credential_file_mounts() -> list[dict[str, str]]:
  128. """Skill-registered + config credential files as ``host_path``/``container_path`` dicts (re-checked for existence)."""
  129. mounts = {cp: hp for cp, hp in _get_registered().items() if Path(hp).is_file()}
  130. for entry in _load_config_files():
  131. cp, hp = entry["container_path"], entry["host_path"]
  132. if cp not in mounts and Path(hp).is_file():
  133. mounts[cp] = hp
  134. return [_mount(hp, cp) for cp, hp in mounts.items()]
  135. # --- Skills directory mounts ---
  136. def _skill_dir_roots(container_base: str) -> Iterator[tuple[Path, str]]:
  137. """Yield ``(host_dir, container_root)`` for every existing skills directory.
  138. Local skills mount at ``<base>/skills``, external at ``<base>/external_skills/<i>``, trusted
  139. project-local at ``<base>/project_skills/<i>`` (own namespace so paths stay stable if external_dirs change).
  140. """
  141. base = container_base.rstrip("/")
  142. skills_dir = get_hermes_home() / "skills"
  143. if skills_dir.is_dir():
  144. yield skills_dir, f"{base}/skills"
  145. try:
  146. from agent.skill_utils import get_external_skills_dirs, get_project_skills_dirs
  147. except ImportError:
  148. return
  149. for label, dirs in (("external_skills", get_external_skills_dirs()), ("project_skills", get_project_skills_dirs())):
  150. yield from ((d, f"{base}/{label}/{idx}") for idx, d in enumerate(dirs) if d.is_dir())
  151. def _walk_skill_tree(root: Path) -> Iterator[tuple[Path, list[Path]]]:
  152. """Yield ``(dir, regular_non_symlink_files)`` for every directory a sandbox should receive.
  153. Prunes ``EXCLUDED_SKILL_DIRS`` *before* descending so bookkeeping/dependency trees (``.hub``,
  154. ``.archive``, ``.curator_backups``, ``node_modules``, ``.git``, ...) the remote agent never reads
  155. are never even walked; sync thus agrees with discovery on what is skill content. Deliberately
  156. not ``is_excluded_skill_path()``: that also prunes ``references/``, ``templates/``, ``assets/``,
  157. ``scripts/`` — progressive-disclosure files and bundled scripts the sandbox does execute.
  158. """
  159. for dirpath, dirnames, filenames in os.walk(root):
  160. dirnames[:] = sorted(d for d in dirnames if d not in EXCLUDED_SKILL_DIRS)
  161. base = Path(dirpath)
  162. yield base, [f for f in (base / n for n in filenames) if not f.is_symlink() and f.is_file()]
  163. def get_skills_directory_mount(container_base: str = "/root/.hermes") -> list[dict[str, str]]:
  164. """Directory mount entries for all skill dirs (local + external + project).
  165. Bind mounts follow symlinks, so a dir containing any symlink is replaced by a sanitized
  166. temp copy (regular files only); symlink-free dirs are returned directly, zero overhead.
  167. """
  168. return [_mount(_safe_skills_path(d), cp) for d, cp in _skill_dir_roots(container_base)]
  169. def _safe_skills_path(skills_dir: Path) -> str:
  170. """Return *skills_dir* if symlink-free, else a sanitized temp copy (same exclusions as sync)."""
  171. global _safe_skills_tempdir
  172. symlinks = [p for p in skills_dir.rglob("*") if p.is_symlink()]
  173. if not symlinks:
  174. return str(skills_dir)
  175. for link in symlinks:
  176. logger.warning("credential_files: skipping symlink in skills dir: %s -> %s", link, os.readlink(link))
  177. import atexit
  178. import shutil
  179. import tempfile
  180. if _safe_skills_tempdir and _safe_skills_tempdir.is_dir():
  181. shutil.rmtree(_safe_skills_tempdir, ignore_errors=True)
  182. safe_dir = _safe_skills_tempdir = Path(tempfile.mkdtemp(prefix="hermes-skills-safe-"))
  183. for base, files in _walk_skill_tree(skills_dir):
  184. (safe_dir / base.relative_to(skills_dir)).mkdir(parents=True, exist_ok=True)
  185. for item in files:
  186. shutil.copy2(str(item), str(safe_dir / item.relative_to(skills_dir)))
  187. atexit.register(lambda: safe_dir.is_dir() and shutil.rmtree(safe_dir, ignore_errors=True))
  188. logger.info("credential_files: created symlink-safe skills copy at %s", safe_dir)
  189. return str(safe_dir)
  190. def iter_skills_files(container_base: str = "/root/.hermes") -> list[dict[str, str]]:
  191. """Per-file entries for all skills files (for backends that upload individually)."""
  192. return [_mount(item, f"{container_root}/{item.relative_to(host_dir).as_posix()}")
  193. for host_dir, container_root in _skill_dir_roots(container_base)
  194. for _base, files in _walk_skill_tree(host_dir) for item in files]
  195. # --- Cache directory mounts (documents, images, audio, videos, screenshots) ---
  196. # (new_subpath, old_name) pairs matching hermes_constants.get_hermes_dir().
  197. _GENERATED_CACHE = f"cache/{GENERATED_SUBDIR}"
  198. _CACHE_DIRS: list[tuple[str, str]] = [
  199. ("cache/documents", "document_cache"),
  200. ("cache/images", "image_cache"),
  201. ("cache/audio", "audio_cache"),
  202. ("cache/videos", "video_cache"),
  203. ("cache/screenshots", "browser_screenshots"),
  204. ("cache/web", "web_cache"),
  205. ("cache/delegation", "delegation_cache"),
  206. ("cache/spillover", "cache/spillover"), # oversized tool results; host side is canonical
  207. # Unswept generated image/video deliverables (#126445) need their own mount/sync
  208. # entry or remote backends never see them. No legacy alias exists.
  209. (_GENERATED_CACHE, _GENERATED_CACHE),
  210. # Flat top-level desktop staging dirs (tui_gateway attach RPCs; no legacy alias),
  211. # mounted so vision/file tools in sandboxes reach uploads and dropped files.
  212. # Mount it so vision can reach uploads inside sandbox containers (#69575). No legacy alias exists, so
  213. # both tuple slots are ``images``.
  214. ("images", "images"),
  215. # Mount it so the agent's file tools can read dropped binaries (zip/pdf/...) from inside sandbox
  216. # containers instead of dangling host paths (#76577).
  217. ("attachments", "attachments"),
  218. # Desktop stages a large plain-text paste as a `.txt` under this Hermes-managed dir
  219. # (apps/desktop/electron/composer-paste.ts; `COMPOSER_PASTES_DIRNAME` in
  220. # agent/context_references.py) and attaches it as `@file:`. Without a mount/sync
  221. # entry, remote execution backends (ssh/daytona/vercel_sandbox) never received the
  222. # bytes and `to_agent_visible_cache_path` left the gateway-host path dangling on
  223. # the remote host (#110174). No legacy alias, so both tuple slots match.
  224. ("composer-pastes", "composer-pastes"),
  225. ]
  226. def _cache_dir_roots(container_base: str, *, create_missing: bool) -> Iterator[tuple[Path, str]]:
  227. """Yield ``(host_dir, container_root)`` per cache dir; always maps to the *new* container layout."""
  228. base = container_base.rstrip("/")
  229. for new_subpath, old_name in _CACHE_DIRS:
  230. host_dir = get_hermes_dir(new_subpath, old_name)
  231. if not host_dir.is_dir():
  232. if not create_missing:
  233. continue
  234. # Docker snapshots this list at container CREATION, so a dir appearing later
  235. # would dangle for the container's life: create it now (empty bind mount is free).
  236. # get_hermes_dir already picked new-vs-legacy, so this can't shadow a legacy dir.
  237. try:
  238. # Create missing staging dirs instead of skipping them: Docker snapshots this mount list at
  239. # container CREATION, so a dir that appears later (first desktop attachment, first clipboard
  240. # image) would dangle for the whole life of a persistent container (#76577). An empty
  241. # bind-mounted dir costs nothing; a missing mount costs the feature. get_hermes_dir()
  242. # already resolved new-vs-legacy layout, so creating its answer cannot shadow a populated
  243. # legacy dir.
  244. host_dir.mkdir(parents=True, exist_ok=True)
  245. except OSError:
  246. continue # unwritable home (tests, RO mounts) — skip as before
  247. yield host_dir, f"{base}/{new_subpath}"
  248. def get_cache_directory_mounts(container_base: str = "/root/.hermes") -> list[dict[str, str]]:
  249. """Bind-mount entries for each cache directory (host layout via ``get_hermes_dir``)."""
  250. return [_mount(h, c) for h, c in _cache_dir_roots(container_base, create_missing=True)]
  251. def _remap_cache_path(path: str, container_base: str, src: str, dst: str, join: Callable[[str, Path], str]) -> Optional[str]:
  252. """Translate *path* from the *src* side of a cache mount to its *dst* side; None if unmounted."""
  253. for mount in get_cache_directory_mounts(container_base=container_base):
  254. if Path(path).is_relative_to(mount[src]):
  255. return join(mount[dst], Path(path).relative_to(mount[src]))
  256. return None
  257. def map_cache_path_to_container(host_path: str, container_base: str = "/root/.hermes") -> Optional[str]:
  258. """POSIX container path for a host path under an auto-mounted cache dir, else None.
  259. Also matches through symlinks: ``@file:`` expansion hands over RESOLVED paths while the mount roots keep
  260. HERMES_HOME's configured spelling, so a symlinked home (``~/.hermes`` -> dotfiles, macOS ``/var`` ->
  261. ``/private/var``) left a staged attachment's host path in front of the sandboxed agent (#103147)."""
  262. def join(root: str, rel: Path) -> str:
  263. return posixpath.join(root, rel.as_posix())
  264. mapped = _remap_cache_path(host_path, container_base, "host_path", "container_path", join)
  265. if mapped is not None:
  266. return mapped
  267. try:
  268. real = Path(host_path).resolve()
  269. for mount in get_cache_directory_mounts(container_base=container_base):
  270. root = Path(mount["host_path"]).resolve()
  271. if real.is_relative_to(root):
  272. return join(mount["container_path"], real.relative_to(root))
  273. except (OSError, RuntimeError):
  274. pass
  275. return None
  276. def from_agent_visible_cache_path(container_path: str, container_base: str = "/root/.hermes") -> str:
  277. """Inverse of :func:`to_agent_visible_cache_path`; unchanged unless Docker + cache dir."""
  278. if _terminal_backend() != "docker":
  279. return container_path
  280. mapped = _remap_cache_path(container_path, container_base, "container_path", "host_path", lambda root, rel: str(Path(root) / rel))
  281. return mapped if mapped is not None else container_path
  282. # Backends whose file-sync lands under the remote home: ``~/.hermes`` is
  283. # expanded by the remote shell, so it resolves regardless of the actual home.
  284. _HOME_RELATIVE_BACKENDS = frozenset({"ssh", "daytona", "vercel_sandbox"})
  285. def _terminal_backend() -> str:
  286. """Active ``TERMINAL_ENV`` through the per-turn terminal scope (a routed multiplex profile's
  287. backend, never the launch profile's process env)."""
  288. from tools.terminal_scope import terminal_env
  289. return (terminal_env("TERMINAL_ENV") or "local").strip().lower()
  290. def to_agent_visible_cache_path(host_path: str, container_base: str = "/root/.hermes") -> str:
  291. """Translate a host cache path to where the active backend (TERMINAL_ENV) sees it.
  292. Mirrors ``_agent_cache_base_for_env`` in tools/image_generation_tool.py: docker/modal mount at
  293. ``/root/.hermes``; ssh/daytona/vercel_sandbox under ``~/.hermes``; plugin backends declare
  294. ``cache_path_base`` (None = host paths stay correct, e.g. the singularity plugin); local/unknown
  295. unchanged.
  296. * docker / modal — bind-mounted (docker) or per-file-synced (modal) at ``/root/.hermes`` (the
  297. *container_base* default). * ssh / daytona / vercel_sandbox — file-synced under the remote user's home;
  298. ``~/.hermes`` is shell-expanded by the remote shell, so tool commands resolve it regardless of the
  299. actual remote home. Previously these backends synced the bytes but still rendered the dangling host path
  300. (#76577 gap).
  301. """
  302. backend = _terminal_backend()
  303. if backend in _HOME_RELATIVE_BACKENDS:
  304. container_base = "~/.hermes"
  305. elif backend not in ("docker", "modal"):
  306. try:
  307. from agent.terminal_env_registry import provider_flag
  308. plugin_base = provider_flag(backend, "cache_path_base", None)
  309. except Exception:
  310. plugin_base = None
  311. if not plugin_base:
  312. return host_path
  313. container_base = str(plugin_base)
  314. mapped = map_cache_path_to_container(host_path, container_base=container_base)
  315. return mapped if mapped is not None else host_path
  316. def iter_cache_files(container_base: str = "/root/.hermes") -> list[dict[str, str]]:
  317. """Per-file cache entries (Modal upload/resync); skips symlinks. ``cache/generated`` is
  318. never swept, so only its files from the last ``MEDIA_CACHE_MAX_AGE_HOURS`` are synced —
  319. otherwise every remote sync would re-walk and upload the whole generation history."""
  320. generated_cutoff = time.time() - MEDIA_CACHE_MAX_AGE_HOURS * 3600
  321. gen_root = f"{container_base.rstrip('/')}/{_GENERATED_CACHE}"
  322. entries: list[dict[str, str]] = []
  323. for host_dir, root in _cache_dir_roots(container_base, create_missing=False):
  324. for item in host_dir.rglob("*"):
  325. try:
  326. st = item.lstat()
  327. except OSError:
  328. continue
  329. if stat.S_ISREG(st.st_mode) and (root != gen_root or st.st_mtime >= generated_cutoff):
  330. entries.append(_mount(item, f"{root}/{item.relative_to(host_dir)}"))
  331. return entries
  332. def clear_credential_files() -> None:
  333. """Reset the skill-scoped registry (e.g. on session reset)."""
  334. _get_registered().clear()