| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500 |
- """Shared file sync manager for remote execution backends.
- Tracks local file changes via mtime+size, detects deletions, and syncs to
- remote environments transactionally. Used by SSH, Modal, and Daytona.
- Docker uses bind mounts (live host FS view) and doesn't need this.
- """
- import hashlib
- import logging
- import os
- import posixpath
- import shlex
- import shutil
- import signal
- import tarfile
- import tempfile
- import threading
- import time
- try:
- import fcntl
- except ImportError:
- fcntl = None # Windows — file locking skipped
- from pathlib import Path
- from typing import Callable
- import psutil
- from hermes_constants import get_hermes_home
- from tools.environments.base import _file_mtime_key
- from utils import mtime_is_racy
- logger = logging.getLogger(__name__)
- # Tests patch these module-level aliases instead of ``time.sleep`` /
- # ``time.monotonic``: patching attributes on the shared ``time`` module object
- # leaks into unrelated threads when tests share a process and inflates retry
- # call counts.
- _sleep = time.sleep
- _monotonic = time.monotonic
- _SYNC_INTERVAL_SECONDS = 5.0
- _FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC"
- # Synced-state key no stat result equals (see _plan_sync): forces a content check next sync.
- _RACY_SYNC_KEY = (-1, -1, -1, -1)
- # Transport callbacks provided by each backend
- UploadFn = Callable[[str, str], None] # (host_path, remote_path) -> raises on failure
- BulkUploadFn = Callable[[list[tuple[str, str]]], None] # [(host_path, remote_path), ...] -> raises on failure
- BulkDownloadFn = Callable[[Path], None] # (dest_tar_path) -> writes tar archive, raises on failure
- DeleteFn = Callable[[list[str]], None] # (remote_paths) -> raises on failure
- GetFilesFn = Callable[[], list[tuple[str, str]]] # () -> [(host_path, remote_path), ...]
- _SYNC_BACK_MAX_RETRIES = 3
- _SYNC_BACK_BACKOFF = (2, 4, 8) # seconds between retries
- _SYNC_BACK_MAX_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB — refuse to extract larger tars
- _SYNC_BACK_MAX_BYTES_KEY = "sync_back_max_bytes" # config.yaml terminal.<key>
- _SYNC_BACK_TEMP_PREFIX = "hermes-sync-back-"
- # A sync-back temp entry (the downloaded tar or the extraction staging dir) is only leaked by
- # a hard kill (SIGKILL/OOM/power loss — the ``finally`` never runs). Entry names embed the
- # owning PID, so a dead owner's entry is reclaimed at once; the age cutoff covers the rest
- # (live or recycled PIDs, pre-ownership names, Windows where a PID cannot be probed). The
- # download is bounded by a 120 s subprocess timeout, so a live transfer is minutes old at
- # most; the old 6 h window let a crash loop pile up tens of GB before anything was reclaimed.
- _SYNC_BACK_STALE_SECONDS = 30 * 60
- def _sync_back_max_bytes() -> int:
- """Extraction cap; config.yaml ``terminal.sync_back_max_bytes`` overrides it for trees that
- legitimately exceed 2 GiB (a skipped extraction silently discards the whole download)."""
- from hermes_cli.config import load_config
- raw = ((load_config() or {}).get("terminal") or {}).get(_SYNC_BACK_MAX_BYTES_KEY)
- if raw is not None:
- try:
- return int(raw)
- except (TypeError, ValueError):
- logger.warning("sync_back: ignoring non-integer terminal.%s=%r", _SYNC_BACK_MAX_BYTES_KEY, raw)
- return _SYNC_BACK_MAX_BYTES
- def _sync_back_temp_prefix() -> str:
- """Temp prefix embedding the owning PID so the stale sweep can tell a hard-killed
- process's leftovers from another live gateway's in-flight transfer without guessing
- from mtime (a staging dir's mtime does not move while content streams into it)."""
- return f"{_SYNC_BACK_TEMP_PREFIX}{os.getpid()}-"
- def _temp_entry_owner_alive(name: str) -> bool:
- """Whether the process that created a sync-back temp entry may still be running.
- Names without a PID count as alive: the age cutoff applies."""
- pid_part = name[len(_SYNC_BACK_TEMP_PREFIX):].split("-", 1)[0]
- if not pid_part.isdigit():
- return True
- return psutil.pid_exists(int(pid_part))
- def _cleanup_stale_sync_back_temp(temp_dir: Path | None = None) -> int:
- """Remove sync-back tars and staging dirs left behind by a hard-killed process.
- Only entries carrying this module's prefix are touched: at once when their owner PID
- is dead, otherwise only past ``_SYNC_BACK_STALE_SECONDS``. Returns the number of
- entries removed; a permission error or a race with another sync-back must not prevent
- the current one.
- """
- directory = temp_dir or Path(tempfile.gettempdir())
- cutoff = time.time() - _SYNC_BACK_STALE_SECONDS
- removed = 0
- try:
- candidates = list(directory.glob(f"{_SYNC_BACK_TEMP_PREFIX}*"))
- except OSError:
- logger.debug("sync_back: could not scan temporary directory %s", directory)
- return 0
- for candidate in candidates:
- try:
- if candidate.is_symlink():
- continue
- if _temp_entry_owner_alive(candidate.name) and candidate.lstat().st_mtime >= cutoff:
- continue
- if candidate.is_dir():
- shutil.rmtree(candidate)
- else:
- candidate.unlink()
- removed += 1
- logger.debug("sync_back: removed stale temporary entry %s", candidate)
- except OSError:
- logger.debug("sync_back: could not remove stale temporary entry %s", candidate)
- return removed
- def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]:
- """Enumerate all (host_path, remote_path) pairs to sync to a remote. Credential paths are
- remapped from the hardcoded /root/.hermes to *container_base* (remote home may differ)."""
- # Late import: credential_files pulls in agent modules (circular at module level).
- from tools.credential_files import get_credential_file_mounts, iter_cache_files, iter_skills_files
- files = [
- (entry["host_path"], entry["container_path"].replace("/root/.hermes", container_base, 1))
- for entry in get_credential_file_mounts()]
- files += [
- (entry["host_path"], entry["container_path"])
- for entry in (*iter_skills_files(container_base=container_base),
- *iter_cache_files(container_base=container_base))]
- return files
- def _resolve_host_path_str(host_path: str) -> str:
- """Canonical string form of a host path (``resolve()`` falling back to ``expanduser()``)."""
- try:
- return str(Path(host_path).expanduser().resolve())
- except OSError:
- return str(Path(host_path).expanduser())
- def _credential_host_paths() -> set[str]:
- """Return credential files that are upload-only for remote sandboxes."""
- try:
- from tools.credential_files import get_credential_file_mounts
- mounts = get_credential_file_mounts()
- except Exception:
- return set()
- return {
- _resolve_host_path_str(entry["host_path"])
- for entry in mounts if isinstance(entry, dict) and entry.get("host_path")}
- def quoted_rm_command(remote_paths: list[str]) -> str:
- """Build a shell ``rm -f`` command for a batch of remote paths."""
- return "rm -f " + " ".join(shlex.quote(p) for p in remote_paths)
- def quoted_mkdir_command(dirs: list[str]) -> str:
- """Build a shell ``mkdir -p`` command for a batch of directories."""
- return "mkdir -p " + " ".join(shlex.quote(d) for d in dirs)
- def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:
- """Extract sorted unique parent directories from (host, remote) pairs."""
- return sorted({posixpath.dirname(remote) for _, remote in files})
- def _sha256_file(path: str) -> str:
- """Return hex SHA-256 digest of a file."""
- h = hashlib.sha256()
- with open(path, "rb") as f:
- for chunk in iter(lambda: f.read(65536), b""):
- h.update(chunk)
- return h.hexdigest()
- class FileSyncManager:
- """Tracks local file changes and syncs to a remote environment. Backends supply transport
- callbacks (upload, delete) and a file-source callable; the manager handles mtime-based
- change detection, deletion tracking, rate limiting, and transactional state."""
- def __init__(
- self,
- get_files_fn: GetFilesFn,
- upload_fn: UploadFn,
- delete_fn: DeleteFn,
- sync_interval: float = _SYNC_INTERVAL_SECONDS,
- bulk_upload_fn: BulkUploadFn | None = None,
- bulk_download_fn: BulkDownloadFn | None = None):
- self._get_files_fn = get_files_fn
- self._upload_fn = upload_fn
- self._bulk_upload_fn = bulk_upload_fn
- self._bulk_download_fn = bulk_download_fn
- self._delete_fn = delete_fn
- self._transaction_lock = threading.Lock()
- self._synced_files: dict[str, tuple[int, int, int, int]] = {} # remote_path -> file_signature
- self._pushed_hashes: dict[str, str] = {} # remote_path -> sha256 hex digest
- self._upload_only_host_paths: set[str] = set()
- self._last_sync_time: float = 0.0 # monotonic; 0 ensures first sync runs
- self._sync_interval = sync_interval
- def sync(self, *, force: bool = False) -> None:
- """Run a sync cycle: upload changed files, delete removed files. Rate-limited to once
- per ``sync_interval`` unless *force* or ``HERMES_FORCE_FILE_SYNC=1``. Transactional:
- state is committed only if ALL operations succeed; on failure it rolls back so the
- next cycle retries everything."""
- with self._transaction_lock:
- self._sync_transaction(force=force)
- def _sync_transaction(self, *, force: bool = False) -> None:
- """Execute one sync cycle while holding the per-manager lock."""
- if (
- not force
- and not os.environ.get(_FORCE_SYNC_ENV)
- and _monotonic() - self._last_sync_time < self._sync_interval):
- return
- current_files = self._get_files_fn()
- self._upload_only_host_paths.update(_credential_host_paths())
- prev_files = dict(self._synced_files)
- prev_hashes = dict(self._pushed_hashes)
- try:
- # Planning reads files too (content check against the pushed hash): a file removed
- # or unreadable after its stat fails the cycle here like a staging read, so the
- # caller's command still runs and the next cycle retries.
- to_upload, new_files, to_delete = self._plan_sync(current_files)
- if not to_upload and not to_delete:
- self._synced_files = new_files # content-confirmed keys, nothing to push
- self._last_sync_time = _monotonic()
- return
- # Hash and upload the same bytes: the original may be saved while
- # the transport is reading it or waiting for remote acknowledgement.
- with tempfile.TemporaryDirectory(prefix="hermes-sync-push-") as staging:
- staged_files = []
- pushed_hashes = {}
- for index, (host_path, remote_path) in enumerate(to_upload):
- staged_path = os.path.join(staging, str(index))
- shutil.copy2(host_path, staged_path)
- pushed_hashes[remote_path] = _sha256_file(staged_path)
- staged_files.append((staged_path, remote_path))
- self._push(staged_files, to_delete)
- # Commit (all succeeded).
- self._pushed_hashes.update(pushed_hashes)
- for p in to_delete:
- new_files.pop(p, None)
- self._pushed_hashes.pop(p, None)
- self._synced_files = new_files
- self._last_sync_time = _monotonic()
- except Exception as exc:
- self._synced_files = prev_files
- self._pushed_hashes = prev_hashes
- # Do NOT advance _last_sync_time: bumping the rate-limit clock on failure would
- # suppress the retry for up to _sync_interval, contradicting the retry contract.
- logger.warning("file_sync: sync failed, rolled back state: %s", exc)
- def _plan_sync(
- self, current_files: list[tuple[str, str]]
- ) -> tuple[list[tuple[str, str]], dict[str, tuple[int, int, int, int]], list[str]]:
- """Diff *current_files* against synced state -> ``(to_upload, new_synced_state, to_delete)``."""
- to_upload: list[tuple[str, str]] = []
- new_files = dict(self._synced_files)
- for host_path, remote_path in current_files:
- file_key = _file_mtime_key(host_path)
- if file_key is None or self._synced_files.get(remote_path) == file_key:
- continue
- # A key whose mtime is still inside the racy window is recorded as never-matching: a
- # same-size rewrite in the same coarse tick would keep every stat field (ctime ticks
- # just as coarsely). The next sync then confirms the bytes against the pushed hash.
- recorded_key = _RACY_SYNC_KEY if mtime_is_racy(file_key[0]) else file_key
- new_files[remote_path] = recorded_key
- pushed_hash = self._pushed_hashes.get(remote_path)
- if pushed_hash is not None and _sha256_file(host_path) == pushed_hash:
- continue # the remote already holds these bytes
- to_upload.append((host_path, remote_path))
- current_remote_paths = {remote for _, remote in current_files}
- to_delete = [p for p in self._synced_files if p not in current_remote_paths]
- return to_upload, new_files, to_delete
- def _push(self, to_upload: list[tuple[str, str]], to_delete: list[str]) -> None:
- """Run the transport calls for one cycle (bulk upload when available)."""
- if to_upload:
- logger.debug("file_sync: uploading %d file(s)", len(to_upload))
- if to_delete:
- logger.debug("file_sync: deleting %d stale remote file(s)", len(to_delete))
- if to_upload and self._bulk_upload_fn is not None:
- self._bulk_upload_fn(to_upload)
- logger.debug("file_sync: bulk-uploaded %d file(s)", len(to_upload))
- else:
- for host_path, remote_path in to_upload:
- self._upload_fn(host_path, remote_path)
- logger.debug("file_sync: uploaded %s -> %s", host_path, remote_path)
- if to_delete:
- self._delete_fn(to_delete)
- logger.debug("file_sync: deleted %s", to_delete)
- # --- Sync-back: pull remote changes to host on teardown ---
- def sync_back(self, hermes_home: Path | None = None) -> None:
- """Pull remote changes back to the host: download the remote ``.hermes/`` as a tar and
- apply only files whose SHA-256 differs from what was pushed. SIGINT is deferred until
- complete; concurrent gateway sandboxes are serialized via a file lock."""
- with self._transaction_lock:
- self._sync_back_transaction(hermes_home=hermes_home)
- def _sync_back_transaction(self, hermes_home: Path | None = None) -> None:
- """Execute sync-back (with retries) against a stable snapshot of manager state."""
- if self._bulk_download_fn is None:
- return
- # Nothing was ever committed (initial push failed or never ran): skip
- # to avoid retry storms against an uninitialized remote .hermes/.
- if not self._pushed_hashes and not self._synced_files:
- logger.debug("sync_back: no prior push state — skipping")
- return
- lock_path = (hermes_home or get_hermes_home()) / ".sync.lock"
- lock_path.parent.mkdir(parents=True, exist_ok=True)
- last_exc: Exception | None = None
- for attempt in range(_SYNC_BACK_MAX_RETRIES):
- try:
- self._sync_back_once(lock_path)
- return
- except Exception as exc:
- last_exc = exc
- if attempt < _SYNC_BACK_MAX_RETRIES - 1:
- delay = _SYNC_BACK_BACKOFF[attempt]
- logger.warning("sync_back: attempt %d failed (%s), retrying in %ds", attempt + 1, exc, delay)
- _sleep(delay)
- logger.warning("sync_back: all %d attempts failed: %s", _SYNC_BACK_MAX_RETRIES, last_exc)
- def _sync_back_once(self, lock_path: Path) -> None:
- """Single sync-back attempt with SIGINT protection and file lock."""
- # signal.signal() only works from the main thread; gateway cleanup()
- # may run from a worker thread — skip SIGINT deferral there.
- on_main_thread = threading.current_thread() is threading.main_thread()
- deferred_sigint: list[object] = []
- original_handler = None
- if on_main_thread:
- original_handler = signal.getsignal(signal.SIGINT)
- def _defer_sigint(signum, frame):
- deferred_sigint.append((signum, frame))
- logger.debug("sync_back: SIGINT deferred until sync completes")
- signal.signal(signal.SIGINT, _defer_sigint)
- try:
- self._sync_back_locked(lock_path)
- finally:
- if on_main_thread and original_handler is not None:
- signal.signal(signal.SIGINT, original_handler)
- if deferred_sigint:
- # Re-deliver the deferred Ctrl+C to the restored handler. ``os.kill(os.getpid(),
- # SIGINT)`` is NOT graceful on Windows (routes to TerminateProcess, hard-killing
- # the CLI); ``raise_signal`` invokes the handler everywhere.
- signal.raise_signal(signal.SIGINT)
- def _sync_back_locked(self, lock_path: Path) -> None:
- """Sync-back under file lock (serializes concurrent gateways)."""
- if fcntl is None:
- # Windows: no flock — run without serialization
- self._sync_back_impl()
- return
- lock_fd = open(lock_path, "w", encoding="utf-8")
- try:
- fcntl.flock(lock_fd, fcntl.LOCK_EX)
- self._sync_back_impl()
- finally:
- try:
- fcntl.flock(lock_fd, fcntl.LOCK_UN)
- except OSError:
- pass
- lock_fd.close()
- def _sync_back_impl(self) -> None:
- """Download, diff, and apply remote changes to host."""
- if self._bulk_download_fn is None:
- raise RuntimeError("_sync_back_impl called without bulk_download_fn")
- # Cache file mapping once to avoid O(n*m) from repeated iteration
- try:
- file_mapping = list(self._get_files_fn())
- except Exception:
- file_mapping = []
- # A hard kill bypasses the finally below. Reclaim only old entries carrying our
- # prefix before allocating another full-tree download.
- _cleanup_stale_sync_back_temp()
- # mkstemp + close: NamedTemporaryFile keeps an exclusive handle on Windows, so the
- # backend's open(dest, "wb") / write_bytes on the same path raised PermissionError.
- fd, tar_path = tempfile.mkstemp(prefix=_sync_back_temp_prefix(), suffix=".tar")
- os.close(fd)
- try:
- self._bulk_download_fn(Path(tar_path))
- # A misbehaving sandbox could produce an arbitrarily large tar.
- try:
- tar_size = os.path.getsize(tar_path)
- except OSError:
- tar_size = 0
- max_bytes = _sync_back_max_bytes()
- if tar_size > max_bytes:
- logger.warning(
- "sync_back: remote tar is %d bytes (cap %d, override with terminal.%s) — skipping extraction",
- tar_size, max_bytes, _SYNC_BACK_MAX_BYTES_KEY)
- return
- with tempfile.TemporaryDirectory(prefix=_sync_back_temp_prefix()) as staging:
- with tarfile.open(tar_path) as tar:
- tar.extractall(staging, filter="data")
- upload_only = self._upload_only_host_paths | _credential_host_paths()
- applied = 0
- for dirpath, _dirnames, filenames in os.walk(staging):
- for fname in filenames:
- staged_file = os.path.join(dirpath, fname)
- # Remote keys are POSIX; relpath uses host separators (backslashes on Windows).
- remote_path = "/" + Path(os.path.relpath(staged_file, staging)).as_posix()
- applied += self._apply_staged_file(staged_file, remote_path, file_mapping, upload_only)
- if applied:
- logger.info("sync_back: applied %d changed file(s)", applied)
- else:
- logger.debug("sync_back: no remote changes detected")
- finally:
- try:
- os.unlink(tar_path)
- except OSError:
- pass
- def _apply_staged_file(
- self, staged_file: str, remote_path: str, file_mapping: list[tuple[str, str]], upload_only_host_paths: set[str],
- ) -> int:
- """Copy one extracted remote file onto the host if it changed since push. Returns 1 if
- applied, 0 if skipped (unchanged, unmapped, or an upload-only credential). A host file
- modified since push is overwritten with the remote version (last-write-wins) with a warning."""
- pushed_hash = self._pushed_hashes.get(remote_path)
- if pushed_hash is not None and _sha256_file(staged_file) == pushed_hash:
- return 0 # unchanged from push
- host_path = self._resolve_host_path(remote_path, file_mapping)
- if host_path is None:
- host_path = self._infer_host_path(remote_path, file_mapping, upload_only_host_paths=upload_only_host_paths)
- if host_path is None:
- logger.debug("sync_back: skipping %s (no host mapping)", remote_path)
- return 0
- if self._is_upload_only_host_path(host_path, upload_only_host_paths):
- logger.debug("sync_back: skipping upload-only credential file %s", remote_path)
- return 0
- if pushed_hash is not None and os.path.exists(host_path) and _sha256_file(host_path) != pushed_hash:
- logger.warning(
- "sync_back: conflict on %s — host modified "
- "since push, remote also changed. Applying remote version (last-write-wins).",
- remote_path)
- os.makedirs(os.path.dirname(host_path), exist_ok=True)
- shutil.copy2(staged_file, host_path)
- return 1
- def _resolve_host_path(self, remote_path: str, file_mapping: list[tuple[str, str]] | None = None) -> str | None:
- """Find the host path for a known remote path from the file mapping."""
- return next((host for host, remote in file_mapping or [] if remote == remote_path), None)
- def _infer_host_path(self, remote_path: str, file_mapping: list[tuple[str, str]] | None = None, *,
- upload_only_host_paths: set[str] | None = None) -> str | None:
- """Infer a host path for a new remote file by matching path prefixes: an existing
- remote->host pair whose parent directory prefixes *remote_path* gets the same
- substitution (``/root/.hermes/skills/b.md`` -> ``~/.hermes/skills/b.md``)."""
- upload_only_host_paths = upload_only_host_paths or set()
- for host, remote in file_mapping or []:
- if self._is_upload_only_host_path(host, upload_only_host_paths):
- continue
- remote_dir = posixpath.dirname(remote) # remote paths are POSIX even on a Windows host
- if remote_path.startswith(remote_dir + "/"):
- return str(Path(host).parent / remote_path[len(remote_dir) + 1:])
- return None
- @staticmethod
- def _is_upload_only_host_path(host_path: str, upload_only_host_paths: set[str]) -> bool:
- return _resolve_host_path_str(host_path) in upload_only_host_paths
|