| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134 |
- """Backup and import commands for hermes CLI."""
- import json
- import logging
- import os
- import shutil
- import sqlite3
- import stat
- import sys
- import tempfile
- import threading
- import time
- import zipfile
- import zlib
- from contextlib import contextmanager, suppress
- from datetime import datetime, timezone, UTC
- from pathlib import Path
- from typing import Any, Callable, Dict, List, Optional, Tuple
- from hermes_constants import (
- LOCAL_RUNTIME_ROOT_DIRS, _get_platform_default_hermes_home, get_default_hermes_root, get_hermes_home,
- display_hermes_home,
- )
- from hermes_state_dbfile import RETIRED_GENERATION_DIR_SUFFIX
- from hermes_state_holders import read_only_db_uri
- from agent.provider_media import GENERATED_SUBDIR
- from hermes_cli.archive_safe import normalize_archive_parts
- from hermes_cli.backup_sqlite import _close_quietly, _safe_copy_db
- from hermes_cli.config_backend import require_file_tooling, supports_file_tooling
- from hermes_cli.home_data_layout import PM_RUNTIME_ROOT_DIRS, SECRET_CACHE_FILES, profile_root_entry
- from hermes_cli.sizefmt import format_bytes as _format_size
- from hermes_cli.backup_restore import (
- _count_cron_jobs,
- _count_session_rows,
- _default_new_file_mode,
- _detect_prefix,
- _extract_member_atomically,
- _get_config_path_value,
- _import_db_member,
- _restore_auth_json,
- _restore_destination_epoch,
- _safe_restore_db,
- _set_config_path_value,
- _validate_backup_zip,
- )
- logger = logging.getLogger(__name__)
- def _foreign_db_holder_pids(db_path: Path) -> Optional[list[int]]:
- # Shim to stop the old updater doing work until relaunch. None means unknown,
- # not permission to restore over a database whose holders we did not scan.
- return None
- # --- Exclusion rules ---
- # Where ``hermes backup --quick`` / ``/snapshot`` / the pre-update safety net write state
- # snapshots (see ``create_quick_snapshot``); defined here because the exclusion set needs it.
- _QUICK_SNAPSHOTS_DIR = "state-snapshots"
- def _snapshot_recovery_hint() -> str:
- """How to restore a state snapshot. There is no `hermes snapshot` subcommand — only the /snapshot
- slash command inside a `hermes` session (hermes_cli/commands.py)."""
- return ("To restore a newer snapshot, start `hermes` in a terminal and run `/snapshot list`, then "
- "`/snapshot restore <id>` (CLI only).")
- # Directory names to skip (matched against each path component). ``hermes-agent`` only matches at
- # the root (``_should_exclude``) so skill dirs like ``skills/.../hermes-agent/`` survive. The
- # dependency/cache entries matter: one plugin venv or pip/uv cache under HERMES_HOME walked
- # file-by-file balloons a backup to hundreds of thousands of entries ("backup stuck for days").
- # Mostly mirrors ``agent.skill_utils.EXCLUDED_SKILL_DIRS``; ``.cache`` is backup-only. ``.archive``
- # is deliberately NOT excluded: the curator's ``skills/.archive/`` holds restorable user skills.
- _EXCLUDED_DIRS = {
- "hermes-agent", # the codebase repo — re-clone instead
- "__pycache__", # bytecode caches — regenerated on import
- ".git", # nested git dirs (profiles shouldn't have these, but safety)
- "node_modules", # js deps — reinstalled on demand
- "backups", # prior auto-backups — don't nest backups exponentially
- _QUICK_SNAPSHOTS_DIR, # each holds a full state.db copy — same reason as ``backups``
- "checkpoints", # session-hash-keyed trajectory caches — regenerated, don't port
- # Live CDP browser profiles: Chromium holds their SQLite DBs exclusively locked while running
- # and sqlite3.backup() retries SQLITE_BUSY forever, hanging the backup. Regenerable anyway.
- "browser-profiles",
- # Real-profile browsing snapshot (browser.use_real_profile): copies of the user's Cookies /
- # Login Data — a credential store that must NOT enter an archive. Regenerated on next launch.
- "browser-profile",
- # Python dependency trees (plugin / MCP-server venvs) — regenerated by reinstalling.
- ".venv", "venv", "site-packages",
- # Tool / build caches — all regeneratable.
- ".cache", ".tox", ".nox", ".pytest_cache", ".mypy_cache", ".ruff_cache",
- }
- # Hermes-managed runtime downloads are regenerable. Match only profile roots:
- # a deeper directory of the same name (such as a skill's models/) is user data.
- _EXCLUDED_ROOT_DIRS = LOCAL_RUNTIME_ROOT_DIRS | (PM_RUNTIME_ROOT_DIRS - {"cache"})
- # Browser Use CLI profile dir (browser.backend: browser-use): Chromium user-data with Login Data / Cookies;
- # the secret caches and Photon sidecar token. Root-scoped like models/ — a skill's own browser_profiles/ is
- # user data. Backup-only: do not fold into LOCAL_RUNTIME_ROOT_DIRS (clone-all identity contract).
- _EXCLUDED_BACKUP_ROOT_DIRS = frozenset({"browser_profiles", *SECRET_CACHE_FILES})
- # ``cache/`` at those same roots mixes regenerable state (model/plugin catalogs, stamps, browser
- # profiles with locked SQLite, tool-output spill) with durable artifacts nothing can rebuild: media
- # the gateway delivered to or received from the user (``gateway.platforms.base``'s media-delivery
- # subdirs) and the grounded-citations evidence ledger. Only these subdirs are archived.
- _KEPT_CACHE_SUBDIRS = {
- "images", "audio", "videos", "documents", "screenshots", "citations", GENERATED_SUBDIR}
- def _in_excluded_root_dir(rel_path: Path) -> bool:
- """True when *rel_path* is inside a regenerable tree at a profile-home root."""
- root_entry = profile_root_entry(rel_path.parts)
- if root_entry in _EXCLUDED_ROOT_DIRS or root_entry in _EXCLUDED_BACKUP_ROOT_DIRS:
- return True
- parts = rel_path.parts
- if len(parts) >= 3 and parts[0] == "profiles":
- parts = parts[2:]
- return len(parts) >= 2 and parts[0] == "cache" and parts[1] not in _KEPT_CACHE_SUBDIRS
- # SQLite sidecars are excluded because ``*.db`` is snapshotted via ``sqlite3.backup()``:
- # shipping the live WAL/SHM/journal alongside would pair a fresh snapshot with stale sidecar
- # state and produce a torn restore on next open. They are regenerated on first connection.
- _SQLITE_SIDECAR_SUFFIXES = (".db-wal", ".db-shm", ".db-journal")
- _EXCLUDED_SUFFIXES = (".pyc", ".pyo", *_SQLITE_SIDECAR_SUFFIXES)
- # File names to skip (runtime state that's meaningless on another machine)
- _EXCLUDED_NAMES = {".backup.lock", "gateway.pid", "cron.pid"}
- # The desktop updater's pre-flight drops ``state.db.pre-update-emergency-<ts>.bak`` at the root
- # — a backup artifact like ``backups/``. Prefix-matched because the name carries a timestamp;
- # a plain ``.bak`` suffix rule would drop user files.
- # Retired-WAL capture dirs (``<name>.retired-wal-<ts>-<pid>/``) are excluded whole: a
- # ``sqlite3.backup()`` snapshot of the live db paired with the captured ``-wal`` is exactly the
- # torn-restore hazard the sidecar exclusion below exists to prevent, and the capture is an
- # operator-recovery artifact that must move as a unit (manifest + image + WAL), never partially.
- _EXCLUDED_PREFIXES = (
- "state.db.pre-update-emergency-",
- f"state.db{RETIRED_GENERATION_DIR_SUFFIX}",
- )
- # Files ``hermes import`` must never overwrite, matched by basename so root and named profiles are
- # both covered. They hold runtime state namespaced to the SOURCE machine: ``gateway_state.json``
- # drives the container-boot reconciler (a foreign value leaves the gateway stuck "starting" and
- # disconnected from the Nous portal); PID/lock/registry files reference source PIDs. Mirrors
- # ``container_boot._STALE_RUNTIME_FILES``; import filters too because older backups predate the
- # backup-side exclusions.
- _IMPORT_SKIP_NAMES = {"gateway_state.json", "gateway.pid", "cron.pid", "gateway.lock", "processes.json"}
- try: # zipfile already imports lzma (free); it is absent only from Pythons built without liblzma
- import lzma
- _LZMA_ERRORS: tuple[type[BaseException], ...] = (lzma.LZMAError,)
- except ImportError: # pragma: no cover
- _LZMA_ERRORS = ()
- # What reading a member's data raises when the archive itself is bad (a bzip2 bad stream and a
- # media read error are OSError, caught alongside): bad deflate stream, bad CRC, truncated stream.
- _ZIP_MEMBER_READ_ERRORS: tuple[type[BaseException], ...] = (
- zipfile.BadZipFile, zlib.error, EOFError, *_LZMA_ERRORS)
- # zipfile.open() drops Unix mode bits on extract; restore tightens these to 0600.
- # vault.key / vault.json.enc: the local credential vault (agent/vault_store.py)
- # IS included in backups (user-entered secrets, not regenerable — unlike the
- # excluded browser-profile/ snapshot) but must come back owner-only.
- _SECRET_FILE_NAMES = {".env", "auth.json", "state.db", "vault.key", "vault.json.enc"}
- # Reserved archive subtree for memory-provider state OUTSIDE HERMES_HOME (e.g. ~/.honcho, via
- # MemoryProvider.backup_paths()), stored and restored relative to the user's home; paths not
- # under home are skipped.
- _EXTERNAL_PREFIX = "_external/"
- class BackupInProgressError(RuntimeError):
- """Raised when another process already owns the Hermes backup slot."""
- class _SQLiteSnapshotError(RuntimeError):
- pass
- @contextmanager
- def _backup_operation_lock(hermes_home: Path, timeout_seconds: float = 0.25):
- """Acquire one cross-process backup slot for full and quick snapshots."""
- lock_path = hermes_home / ".backup.lock"
- lock_path.parent.mkdir(parents=True, exist_ok=True)
- handle = lock_path.open("a+b")
- acquired = False
- deadline = time.monotonic() + max(0.0, timeout_seconds)
- try:
- if os.name == "nt":
- import msvcrt
- if lock_path.stat().st_size == 0:
- handle.write(b" ")
- handle.flush()
- def _lock_op(flag: int) -> None:
- handle.seek(0)
- msvcrt.locking(handle.fileno(), flag, 1)
- lock_flag, unlock_flag = msvcrt.LK_NBLCK, msvcrt.LK_UNLCK
- else:
- import fcntl
- def _lock_op(flag: int) -> None:
- fcntl.flock(handle.fileno(), flag)
- lock_flag, unlock_flag = fcntl.LOCK_EX | fcntl.LOCK_NB, fcntl.LOCK_UN
- while not acquired:
- try:
- _lock_op(lock_flag)
- acquired = True
- except OSError:
- if time.monotonic() >= deadline:
- raise BackupInProgressError("another Hermes backup is already running")
- time.sleep(0.05)
- yield
- finally:
- if acquired:
- with suppress(OSError):
- _lock_op(unlock_flag)
- handle.close()
- @contextmanager
- def _atomic_output_path(final_path: Path, publish_path: Optional[Callable[[], Optional[Path]]] = None):
- """Yield a hidden sibling path and publish it only after a clean close.
- ``publish_path`` picks the destination at publish time (default ``final_path``) so a caller
- can divert an incomplete archive elsewhere without ever touching ``final_path``; returning
- ``None`` discards the partial instead of publishing it.
- """
- partial_path = final_path.with_name(f".{final_path.name}.{os.getpid()}-{threading.get_ident()}.partial")
- partial_path.unlink(missing_ok=True)
- try:
- yield partial_path
- destination = publish_path() if publish_path else final_path
- if destination is None:
- partial_path.unlink(missing_ok=True)
- else:
- os.replace(partial_path, destination)
- except BaseException:
- partial_path.unlink(missing_ok=True)
- raise
- def _collect_memory_provider_external_paths() -> list[Path]:
- """Existing paths the active memory provider declares via ``backup_paths()``; ``[]`` on any
- provider failure (backup must never fail because of a flaky plugin)."""
- try:
- from plugins.memory import _get_active_memory_provider, load_memory_provider
- active = _get_active_memory_provider()
- provider = load_memory_provider(active) if active else None
- except Exception:
- return []
- if provider is None:
- return []
- try:
- declared = provider.backup_paths() or []
- except Exception as exc:
- logger.warning("backup_paths() failed for memory provider %r: %s", active, exc)
- return []
- out: dict[Path, Path] = {} # resolved -> first declared spelling
- for raw in declared:
- try:
- p = Path(raw).expanduser()
- except Exception:
- continue
- if not p.exists():
- continue
- try:
- resolved = p.resolve()
- except (OSError, ValueError):
- continue
- out.setdefault(resolved, p)
- return list(out.values())
- def _iter_external_files(base: Path) -> list[Path]:
- """Regular files under *base* (a file or a directory), skipping symlinks, caches, and pyc."""
- if base.is_file() and not base.is_symlink():
- return [base]
- if not base.is_dir():
- return []
- files: list[Path] = []
- for dirpath, dirnames, filenames in os.walk(base, followlinks=False):
- dirnames[:] = [d for d in dirnames if d not in _EXCLUDED_DIRS]
- files.extend(fp for fp in (Path(dirpath) / f for f in filenames)
- if not (_is_non_regular_path(fp) or fp.name in _EXCLUDED_NAMES
- or fp.name.endswith(_EXCLUDED_SUFFIXES)))
- return files
- def _is_non_regular_path(path: Path) -> bool:
- """True for symlinks, sockets, devices, and other non-regular filesystem entries.
- A failed ``lstat`` is not treated as an exclusion: the archive writer must see the path and
- report the read failure instead of silently claiming a complete backup.
- """
- try:
- return not stat.S_ISREG(path.lstat().st_mode)
- except OSError:
- return False
- def _is_link_path(path: Path) -> bool:
- """True for symlinks and Windows junctions/reparse points — the only
- directory entries a strict walk must never descend (os.walk already
- refuses POSIX dir symlinks; this also covers junctions, which it follows)."""
- try:
- info = path.lstat()
- except OSError:
- return False
- if stat.S_ISLNK(info.st_mode):
- return True
- return os.name == "nt" and bool(getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT)
- def _should_exclude(rel_path: Path) -> bool:
- """Return True if *rel_path* (relative to hermes root) should be skipped."""
- parts = rel_path.parts
- if _in_excluded_root_dir(rel_path):
- return True
- # ``hermes-agent`` only matches at the root level; nested same-named dirs are preserved.
- if any(p in _EXCLUDED_DIRS and (p != "hermes-agent" or p == parts[0]) for p in parts):
- return True
- name = rel_path.name
- return name in _EXCLUDED_NAMES or name.startswith(_EXCLUDED_PREFIXES) or name.endswith(_EXCLUDED_SUFFIXES)
- def _iter_backup_files(hermes_root: Path, out_path: Path, skipped_dirs: Optional[set] = None):
- """Yield ``(abs_path, rel_path)`` for every file a full backup should hold.
- The one owner of the walk policy (directory pruning so os.walk never descends a multi-GB
- excluded tree, the root-only ``hermes-agent`` carve-out, root runtime trees, per-file rules),
- shared by ``hermes backup`` and the pre-update path so they can never drift.
- """
- for dirpath, dirnames, filenames in os.walk(hermes_root, followlinks=False):
- rel_dir = Path(dirpath).relative_to(hermes_root)
- is_root = rel_dir == Path(".")
- kept = [
- d for d in dirnames
- if (d not in _EXCLUDED_DIRS or (d == "hermes-agent" and not is_root))
- and not _in_excluded_root_dir(rel_dir / d)]
- if skipped_dirs is not None:
- skipped_dirs.update(str(rel_dir / d) for d in set(dirnames) - set(kept))
- # No walk may follow a junction.
- dirnames[:] = [name for name in kept if not _is_link_path(Path(dirpath) / name)]
- for fname in filenames:
- rel = rel_dir / fname
- fpath = hermes_root / rel
- # zipfile.write() follows file symlinks, so skip links before any archive write can
- # copy data from outside HERMES_HOME; never archive the output zip into itself.
- if _should_exclude(rel):
- continue
- if _is_non_regular_path(fpath):
- continue
- with suppress(OSError, ValueError):
- if fpath.resolve() == out_path.resolve():
- continue
- yield fpath, rel
- # --- SQLite safe copy ---
- def _query_ro_sqlite(path: Path, fn):
- """Run ``fn(conn)`` on a read-only connection to *path*; return ``(value, None)`` or ``(None, exc)``."""
- conn = None
- try:
- conn = sqlite3.connect(read_only_db_uri(path), uri=True, timeout=1.0)
- return fn(conn), None
- except Exception as exc:
- return None, exc
- finally:
- _close_quietly(conn)
- def is_zeroed_sqlite_file(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
- """True when *path* looks like the #68474 zeroed-state.db signature.
- Only regular files qualify: probing a FIFO/device/socket could block indefinitely.
- Signature: no ``SQLite format 3`` header and no data — either empty (size 0, the total-loss case,
- #97568) or first *probe_bytes* all NUL. Used at SessionDB open and for snapshot diagnostics so a silent
- all-zero file becomes a guided recovery instead of a generic failure.
- """
- try:
- if not path.is_file():
- return False
- except OSError:
- return False
- from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
- if not force and has_live_connection(path):
- return False
- head = read_header_bytes_preopen(path, length=max(16, probe_bytes), force=force)
- # Empty or all-NUL header => zeroed; a real header (or unreadable) => not.
- return head is not None and not head.startswith(b"SQLite format 3") and not any(head)
- # --- SQLite integrity verification ---
- _SQLITE_HEADER = b"SQLite format 3\0"
- # Above this size ``PRAGMA integrity_check`` (walks every b-tree page — minutes of pegged CPU on a
- # 30 GB state.db, reading as a hung ``hermes update``) is replaced by the O(1) header+schema probe.
- # Default ceiling above which ``PRAGMA integrity_check`` is skipped in favour of the (O(1)) header +
- # structural probe. Sessions databases in the tens of GB are normal for heavy users, so the size-unbounded
- # check is never an acceptable default on the update path. See #70553.
- DEFAULT_INTEGRITY_CHECK_MAX_BYTES = 2 << 30 # 2 GiB
- def verify_sqlite_integrity(
- path: Path, *, check_header: bool = True, run_pragma: bool = True,
- max_bytes: int = DEFAULT_INTEGRITY_CHECK_MAX_BYTES) -> dict:
- """Verify a SQLite database: existence + minimum size, header magic, then a read-only
- ``PRAGMA integrity_check`` (or a cheap structural probe above ``max_bytes``)."""
- def _done(message: str, valid: bool = False, size: Optional[int] = None) -> dict:
- return {"valid": valid, "message": message, "size": size}
- try:
- st = path.stat()
- except FileNotFoundError:
- return _done(f"not found: {path}")
- except OSError as exc:
- return _done(f"cannot stat: {exc}")
- size = st.st_size
- if size < 100: # SQLite minimum viable size (header + 1 page)
- return _done(f"too small ({size} bytes) to be a valid SQLite database", size=size)
- if check_header:
- # Refused when a live connection exists (close() would cancel this process's POSIX locks
- # — see sqlite_safe_read); verification targets offline snapshots/backup artifacts anyway.
- from hermes_cli.sqlite_safe_read import read_header_bytes_preopen
- head = read_header_bytes_preopen(path, length=len(_SQLITE_HEADER))
- if head is None:
- return _done("cannot read header", size=size)
- if head != _SQLITE_HEADER:
- return _done(f"missing SQLite header magic (got {head[:16].hex()!r})", size=size)
- if max_bytes > 0 and size > max_bytes:
- # O(1) probe: the header check caught the zeroed signature; reading sqlite_master + page
- # geometry catches malformed-schema and truncated-header-page classes without a data walk.
- _, exc = _query_ro_sqlite(path, lambda c: (
- c.execute("PRAGMA schema_version").fetchone(),
- c.execute("SELECT count(*) FROM sqlite_master").fetchone()))
- if exc is not None:
- kind = "failed" if isinstance(exc, sqlite3.DatabaseError) else "error"
- return _done(f"schema probe {kind}: {exc}", size=size)
- return _done(
- f"size {size:,} bytes exceeds max_bytes {max_bytes:,}; "
- "skipped PRAGMA integrity_check (header + schema probe passed)",
- valid=True, size=size)
- if run_pragma:
- rows, exc = _query_ro_sqlite(
- path, lambda c: [str(r[0]) for r in c.execute("PRAGMA integrity_check")])
- if exc is not None:
- kind = "cannot open database" if isinstance(exc, sqlite3.DatabaseError) else "integrity check error"
- return _done(f"{kind}: {exc}", size=size)
- if rows == ["ok"]:
- return _done("integrity check passed", valid=True, size=size)
- return _done(f"integrity check failed: {'; '.join(rows[:5])}", size=size)
- return _done("header check passed", valid=True, size=size)
- def _discard_failed_zip_members(zf: zipfile.ZipFile, filelist_len: int) -> None:
- """Drop the member(s) created by a failed write, both from the central directory and the file.
- ZipFile.write finalizes its destination member while unwinding a source-read
- failure, so the partial bytes can otherwise become a CRC-valid archive member.
- This runs immediately after that failed write, so the dropped bytes are the tail
- of the file: truncate at the first dropped local header and rewind start_dir so
- later members overwrite it. Leaving the bytes (with a valid local header) would
- still expose a ghost member to streaming readers. Rebuilding NameToInfo from the
- surviving file list also restores the previous entry when a duplicate name failed.
- """
- if len(zf.filelist) <= filelist_len:
- return
- offset = zf.filelist[filelist_len].header_offset
- zf.fp.seek(offset)
- zf.fp.truncate()
- zf.start_dir = offset
- del zf.filelist[filelist_len:]
- zf.NameToInfo.clear()
- zf.NameToInfo.update((info.filename, info) for info in zf.filelist)
- def _write_zip_file(zf: zipfile.ZipFile, path: Path, arcname: str) -> None:
- """Write one member while keeping a failed partial write out of the central directory."""
- filelist_len = len(zf.filelist)
- try:
- zf.write(path, arcname=arcname)
- except Exception:
- _discard_failed_zip_members(zf, filelist_len)
- raise
- def _zip_sqlite_snapshot(zf: zipfile.ZipFile, abs_path: Path, rel_path: Path, out_path: Path) -> Optional[int]:
- """Add a WAL-safe snapshot of *abs_path* to *zf*; return its byte size, or None on failure.
- Staged beside the output zip: /tmp may be a small tmpfs that cannot hold large databases.
- """
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False, dir=str(out_path.parent)) as tmp:
- tmp_db = Path(tmp.name)
- try:
- if not _safe_copy_db(abs_path, tmp_db):
- return None
- _write_zip_file(zf, tmp_db, str(rel_path))
- return tmp_db.stat().st_size
- finally:
- tmp_db.unlink(missing_ok=True)
- def _write_zip_entries(
- zf: zipfile.ZipFile, files_to_add: list[tuple[Path, Path]], out_path: Path,
- *, on_db_failure, on_error, on_progress, track_bytes: bool) -> int:
- """Add every ``(abs_path, rel_path)`` to *zf*, WAL-safe for ``*.db``; return bytes archived.
- ``on_db_failure(rel_path)`` runs when a SQLite snapshot fails (may raise to abort);
- ``on_error(rel_path, exc)`` records a read failure; ``on_progress(i)`` fires every 500 files;
- ``track_bytes`` stats plain files for the size total.
- """
- total_bytes = 0
- for i, (abs_path, rel_path) in enumerate(files_to_add, 1):
- try:
- if abs_path.suffix == ".db":
- size = _zip_sqlite_snapshot(zf, abs_path, rel_path, out_path)
- if size is None:
- on_db_failure(rel_path)
- continue
- total_bytes += size
- else:
- _write_zip_file(zf, abs_path, str(rel_path))
- if track_bytes:
- total_bytes += abs_path.stat().st_size
- except (PermissionError, OSError, ValueError) as exc:
- on_error(rel_path, exc)
- continue
- if i % 500 == 0:
- on_progress(i)
- return total_bytes
- def _print_capped(header: str, lines: list[str], indent: str) -> None:
- """Print *header*, then at most 10 of *lines* (each prefixed by *indent*) and a "... and N more" tail."""
- print(header)
- for line in lines[:10]:
- print(f"{indent}{line}")
- if len(lines) > 10:
- print(f"{indent}... and {len(lines) - 10} more")
- # --- Backup ---
- _RUN_BACKUP_PREFIX = "hermes-backup-"
- def _resolve_backup_output_path(output: Optional[str]) -> Path:
- """Turn ``--output`` (file, directory, or None) into a ``.zip`` path whose parent exists;
- an unwritable path exits with a one-line error, not a traceback."""
- out_path = None
- default_name = f"{_RUN_BACKUP_PREFIX}{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
- try:
- if output:
- out_path = Path(output).expanduser().resolve()
- if out_path.is_dir():
- out_path = out_path / default_name
- else:
- out_path = Path.home() / default_name
- if out_path.suffix.lower() != ".zip":
- out_path = out_path.with_suffix(out_path.suffix + ".zip")
- out_path.parent.mkdir(parents=True, exist_ok=True)
- except OSError as exc:
- print(f"Error: cannot write backup to {output or out_path}: {exc}")
- raise SystemExit(1) from exc
- return out_path
- def _collect_external_entries() -> tuple[list[tuple[Path, str]], list[str]]:
- """``([(abs_path, arcname)], [skipped])`` for the memory provider's external state, arc-named
- ``_external/<home-relative>``; paths outside home are skipped (security + portability)."""
- home_dir = Path.home().resolve()
- external_to_add: list[tuple[Path, str]] = []
- skipped_external: list[str] = []
- for base in _collect_memory_provider_external_paths():
- try:
- base.resolve().relative_to(home_dir)
- except (ValueError, OSError):
- skipped_external.append(str(base))
- continue
- for fpath in _iter_external_files(base):
- with suppress(ValueError, OSError):
- rel_to_home = fpath.resolve().relative_to(home_dir)
- external_to_add.append((fpath, _EXTERNAL_PREFIX + rel_to_home.as_posix()))
- return external_to_add, skipped_external
- def run_backup(args) -> bool:
- """Create a zip backup of the Hermes home directory.
- True when every selected file landed in the archive (or there was nothing to back up); False
- when the zip was written but is incomplete — it is kept so the rest can still be restored, and
- the caller turns False into exit status 1 so a cron/systemd timer never publishes a "successful"
- archive that is missing state.db. Hard failures keep raising ``SystemExit``.
- """
- require_file_tooling("`hermes backup`")
- hermes_root = get_default_hermes_root()
- if not hermes_root.is_dir():
- print(f"Error: Hermes home directory not found at {hermes_root}")
- sys.exit(1)
- try:
- with _backup_operation_lock(hermes_root):
- return _run_backup_locked(args, hermes_root)
- except BackupInProgressError as exc:
- print(f"Error: {exc}")
- raise SystemExit(2) from exc
- def _run_backup_locked(args, hermes_root: Path) -> bool:
- """Write a full backup while the cross-process backup slot is held."""
- out_path = _resolve_backup_output_path(args.output)
- scan_started = time.monotonic()
- logger.info("backup phase=scan status=started")
- print(f"Scanning {display_hermes_home()} ...")
- skipped_dirs: set = set()
- files_to_add: list[tuple[Path, Path]] = list(_iter_backup_files(hermes_root, out_path, skipped_dirs))
- external_to_add, skipped_external = _collect_external_entries()
- if not files_to_add and not external_to_add:
- logger.info("backup phase=scan status=empty duration_ms=%.1f", (time.monotonic() - scan_started) * 1000)
- print("No files to back up.")
- return True
- file_count = len(files_to_add) + len(external_to_add)
- logger.info("backup phase=scan status=complete duration_ms=%.1f files=%d",
- (time.monotonic() - scan_started) * 1000, file_count)
- logger.info("backup phase=archive status=started files=%d", file_count)
- print(f"Backing up {file_count} files ...")
- errors = []
- t0 = time.monotonic()
- def _progress(i: int) -> None:
- print(f" {i}/{file_count} files ...")
- logger.info("backup phase=archive status=progress completed=%d total=%d", i, file_count)
- with _atomic_output_path(out_path) as archive_path, zipfile.ZipFile(
- archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
- total_bytes = _write_zip_entries(
- zf, files_to_add, out_path, on_progress=_progress, track_bytes=True,
- on_db_failure=lambda rel: errors.append(f"{rel}: SQLite safe copy failed"),
- on_error=lambda rel, exc: errors.append(f"{rel}: {exc}"))
- # External memory-provider state never includes ``.db`` files in practice, so no
- # SQLite snapshot is needed; _write_zip_file still drops a failed partial member.
- for abs_path, arcname in external_to_add:
- try:
- _write_zip_file(zf, abs_path, arcname)
- total_bytes += abs_path.stat().st_size
- except (PermissionError, OSError, ValueError) as exc:
- errors.append(f"{arcname}: {exc}")
- elapsed = time.monotonic() - t0
- zip_size = out_path.stat().st_size
- logger.info("backup phase=archive status=complete duration_ms=%.1f files=%d errors=%d bytes=%d",
- elapsed * 1000, file_count, len(errors), zip_size)
- print(f"\nBackup {'incomplete' if errors else 'complete'}: {out_path}\n"
- f" Files: {file_count}\n"
- f" Original: {_format_size(total_bytes)}\n"
- f" Compressed: {_format_size(zip_size)}\n"
- f" Time: {elapsed:.1f}s")
- if external_to_add:
- print(f"\n Included {len(external_to_add)} memory-provider file(s) stored outside {display_hermes_home()}.")
- if skipped_external:
- print(f"\n Skipped {len(skipped_external)} memory-provider path(s) outside your home directory "
- "(not portable):\n" + "\n".join(f" {p}" for p in sorted(skipped_external)[:10]))
- if skipped_dirs:
- print("\n Excluded directories:\n" + "\n".join(f" {d}/" for d in sorted(skipped_dirs)))
- if errors:
- _print_capped(f"\n Archive kept, but {len(errors)} file(s) could not be added:", errors, " ")
- else:
- print(f"\nRestore with: hermes import {out_path.name}")
- # Prune only after a complete archive: a timer hitting the same unreadable file every run must
- # not rotate the last good backups out in favour of incomplete ones.
- keep = getattr(args, "keep", 0) # 0 / absent: never prune (non-CLI callers)
- if keep and not errors and out_path.name.startswith(_RUN_BACKUP_PREFIX):
- pruned = _prune_prefixed_zips(out_path.parent, _RUN_BACKUP_PREFIX, keep, "backup")
- if pruned:
- print(f" Pruned {pruned} older {_RUN_BACKUP_PREFIX}*.zip (keeping {keep}).")
- return not errors
- # --- Import ---
- def _find_corrupt_members(zf: zipfile.ZipFile, members: list[str]) -> list[str]:
- """Return ``"<member>: <error>"`` for every member whose data does not decompress or
- fails its CRC, streaming each one in 1 MiB chunks so a multi-GB ``state.db`` is never
- held in memory.
- ``is_zipfile()``/``namelist()`` only read the central directory, so an archive with a
- rotten member passes them and the damage surfaces as ``zlib.error``/``BadZipFile`` in
- the middle of the restore, after earlier members already replaced the user's files
- (#121258). Not ``zf.testzip()``: it lets ``zlib.error`` escape and names at most the
- first bad member.
- """
- bad: list[str] = []
- for member in members:
- try:
- with zf.open(member) as src:
- while src.read(1 << 20): # CRC is checked when the stream hits EOF
- pass
- except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
- bad.append(f"{member}: {exc}")
- return bad
- def _import_skipped(rel: str) -> bool:
- """True for a HERMES_HOME-relative member the import deliberately does not restore: runtime
- state (``_IMPORT_SKIP_NAMES``), PM-local interpreter/dependency roots, or an archived SQLite
- WAL/SHM/journal. A ``.db`` member is page-restored into the live file, and a sidecar from a
- different image would replay a foreign WAL on next open (older archives may ship these)."""
- try:
- parts = tuple(normalize_archive_parts(rel))
- except ValueError:
- return False # A rejected traversal is still reported by the import itself.
- return (parts[-1] in _IMPORT_SKIP_NAMES or
- profile_root_entry(parts) in PM_RUNTIME_ROOT_DIRS or
- rel.endswith(_SQLITE_SIDECAR_SUFFIXES))
- def _import_member_rel(member: str, prefix: str) -> tuple[str, bool]:
- """Classify an archive member exactly as the restore does: return ``(rel, skipped)``.
- ``_external/`` members are home-relative and never skipped; every other member is
- HERMES_HOME-relative after stripping the archive ``prefix``. Shared by the integrity
- pre-flight and ``_import_members`` so the two cannot disagree on what gets restored."""
- if member.startswith(_EXTERNAL_PREFIX):
- return member[len(_EXTERNAL_PREFIX):], False
- rel = member[len(prefix):] if prefix and member.startswith(prefix) else member
- return rel, _import_skipped(rel)
- def run_import(args) -> Optional[int]:
- """Restore a Hermes backup; return 1 on damaged archives or incomplete restores."""
- require_file_tooling("`hermes import`")
- zip_path = Path(args.zipfile).expanduser().resolve()
- if not zip_path.is_file():
- print(f"Error: File not found: {zip_path}")
- sys.exit(1)
- if not zipfile.is_zipfile(zip_path):
- print(f"Error: Not a valid zip file: {zip_path}")
- sys.exit(1)
- # The home this command operates under (the "Target:" display_hermes_home() prints):
- # get_default_hermes_root() would map a profile home back to <root> and restore the live root.
- hermes_root = get_hermes_home()
- with zipfile.ZipFile(zip_path, "r") as zf:
- # Validate
- ok, reason = _validate_backup_zip(zf)
- if not ok:
- print(f"Error: {reason}")
- sys.exit(1)
- prefix = _detect_prefix(zf)
- members = [n for n in zf.namelist() if not n.endswith("/")]
- file_count = len(members)
- print(f"Backup contains {file_count} files")
- print(f"Target: {display_hermes_home()}")
- if prefix:
- print(f"Detected archive prefix: {prefix!r} (will be stripped)")
- # Check for existing installation
- has_config = (hermes_root / "config.yaml").exists() # config-reader: ok — profile marker for the backup scope (file tooling)
- has_env = (hermes_root / ".env").exists()
- if (has_config or has_env) and not args.force:
- print()
- print("Warning: Target directory already has Hermes configuration.")
- print("Importing will overwrite existing files with backup contents.")
- print()
- try:
- answer = input("Continue? [y/N] ").strip().lower()
- except (EOFError, KeyboardInterrupt):
- print("\nAborted.")
- sys.exit(1)
- if answer not in {"y", "yes"}:
- print("Aborted.")
- return
- # Refuse damaged archives before writing any user files. Runtime and PM-local
- # members skipped below cannot block restoration of the portable data.
- print("\nChecking archive integrity ...")
- corrupt = _find_corrupt_members(zf, [m for m in members if not _import_member_rel(m, prefix)[1]])
- if corrupt:
- _print_capped(f"Error: backup archive is damaged ({len(corrupt)} member(s) fail to "
- f"decompress or fail their CRC); nothing was restored:", corrupt, " ")
- return 1
- # Extract
- print(f"\nImporting {file_count} files ...")
- hermes_root.mkdir(parents=True, exist_ok=True)
- t0 = time.monotonic()
- from gateway.runtime_ownership import OwnershipConflict
- try:
- restored, restored_external, errors, skipped_runtime, db_shrunk = _import_members(
- zf, members, prefix, hermes_root, file_count)
- except OwnershipConflict as exc:
- print(f"\nImport refused; no files restored: {exc}")
- sys.exit(1)
- elapsed = time.monotonic() - t0
- # Summary
- print()
- print(f"Import {'incomplete' if errors else 'complete'}: {restored} files restored in {elapsed:.1f}s")
- print(f" Target: {display_hermes_home()}")
- if restored_external:
- print(
- f"\n Restored {restored_external} memory-provider file(s) to "
- f"their original location(s) outside {display_hermes_home()}."
- )
- if errors:
- print(f"\n Warnings ({len(errors)} files skipped):")
- for e in errors[:10]:
- print(e)
- if len(errors) > 10:
- print(f" ... and {len(errors) - 10} more")
- if db_shrunk:
- # The backup predates work that is now overwritten. Say so: the
- # reported incident was twelve sessions disappearing with nothing
- # logged anywhere (issue #100960).
- print("\n ⚠ Session data replaced by older backup contents:")
- for rel, before, after in db_shrunk:
- print(
- f" {rel}: {before[0]} session(s) / {before[1]} message(s)"
- f" -> {after[0]} / {after[1]}"
- )
- print(
- " Anything recorded after the backup was taken is not in it. "
- f"{_snapshot_recovery_hint()}"
- )
- if skipped_runtime:
- print(
- f"\n Preserved {len(skipped_runtime)} runtime state "
- f"file(s) (kept this machine's, not the backup's):"
- )
- for rel in sorted(skipped_runtime)[:10]:
- print(f" {rel}")
- if len(skipped_runtime) > 10:
- print(f" ... and {len(skipped_runtime) - 10} more")
- # Post-import: restore profile wrapper scripts
- profiles_dir = hermes_root / "profiles"
- restored_profiles = []
- if profiles_dir.is_dir():
- try:
- from hermes_cli.profiles import (
- create_wrapper_script, check_alias_collision,
- _is_wrapper_dir_in_path, _get_wrapper_dir,
- )
- for entry in sorted(profiles_dir.iterdir()):
- if not entry.is_dir():
- continue
- profile_name = entry.name
- # Only create wrappers for directories with config
- if not (entry / "config.yaml").exists() and not (entry / ".env").exists(): # config-reader: ok — profile marker for the backup scope (file tooling)
- continue
- collision = check_alias_collision(profile_name)
- if collision:
- print(f" Skipped alias '{profile_name}': {collision}")
- restored_profiles.append((profile_name, False))
- else:
- wrapper = create_wrapper_script(profile_name)
- restored_profiles.append((profile_name, wrapper is not None))
- if restored_profiles:
- created = [n for n, ok in restored_profiles if ok]
- skipped = [n for n, ok in restored_profiles if not ok]
- if created:
- print(f"\n Profile aliases restored: {', '.join(created)}")
- if skipped:
- print(f" Profile aliases skipped: {', '.join(skipped)}")
- if not _is_wrapper_dir_in_path():
- print(f"\n Note: {_get_wrapper_dir()} is not in your PATH.")
- print(' Add to your shell config (~/.bashrc or ~/.zshrc):')
- print(' export PATH="$HOME/.local/bin:$PATH"')
- except ImportError:
- # hermes_cli.profiles might not be available (fresh install)
- if any(profiles_dir.iterdir()):
- print("\n Profiles detected but aliases could not be created.")
- print(" Run: hermes profile list (after installing hermes)")
- # Guidance
- print()
- if not (hermes_root / "hermes-agent").is_dir():
- print("Note: The hermes-agent codebase was not included in the backup.")
- print(" If this is a fresh install, run: hermes update")
- if restored_profiles:
- gw_profiles = [n for n, _ in restored_profiles]
- print("\nTo re-enable gateway services for profiles:")
- for pname in gw_profiles:
- print(f" hermes -p {pname} gateway install")
- _revive_gateway_after_import(hermes_root)
- if errors:
- print(f"Import incomplete: {len(errors)} file(s) were not restored (see Warnings above). "
- "Fix the cause and re-run the import.")
- return 1
- print("Done. Your Hermes configuration has been restored.")
- def _import_members(
- zf: zipfile.ZipFile, members: list[str], prefix: str, hermes_root: Path, file_count: int
- ) -> tuple[int, int, list[str], list[str], list[tuple[str, tuple[int, int], tuple[int, int]]]]:
- """Reserve all affected profiles before publishing even the first config file."""
- from gateway.runtime_ownership import exclusive_maintenance
- root_res = hermes_root.resolve()
- homes = {hermes_root}
- for member in members:
- if member.startswith(_EXTERNAL_PREFIX):
- continue # restored under $HOME, outside every runtime authority
- rel = member[len(prefix):] if prefix and member.startswith(prefix) else member
- parts = Path(rel).parts
- if len(parts) >= 3 and parts[0] == "profiles":
- home = hermes_root / parts[0] / parts[1]
- if home.resolve().is_relative_to(root_res):
- homes.add(home)
- target = hermes_root / rel
- if target.suffix == ".db" and target.resolve().is_relative_to(root_res):
- homes.update([target.absolute().parent, target.resolve().parent])
- with exclusive_maintenance(homes):
- return _import_members_exclusive(zf, members, prefix, hermes_root, file_count)
- def _import_external_member(
- zf: zipfile.ZipFile, member: str, home_dir: Path, new_file_mode: int, errors: list[str]
- ) -> bool:
- """Publish one ``_external/`` member under the user's home; False (with ``errors`` appended)
- when it is a traversal or cannot be written."""
- target = home_dir / member[len(_EXTERNAL_PREFIX):]
- # Security: the resolved target must stay under the home dir.
- try:
- target.resolve().relative_to(home_dir)
- except ValueError:
- errors.append(f" {member}: path traversal blocked")
- return False
- try:
- target.parent.mkdir(parents=True, exist_ok=True)
- _extract_member_atomically(zf, member, target, new_file_mode)
- # External provider configs commonly hold credentials.
- if target.suffix in {".json", ".env", ".conf"} or target.name in _SECRET_FILE_NAMES:
- try:
- os.chmod(target, 0o600)
- except OSError:
- pass
- return True
- except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
- errors.append(f" {member}: {exc}")
- return False
- def _import_members_exclusive(
- zf: zipfile.ZipFile, members: list[str], prefix: str, hermes_root: Path, file_count: int
- ) -> tuple[int, int, list[str], list[str], list[tuple[str, tuple[int, int], tuple[int, int]]]]:
- """Publish every member; return ``(restored, restored_external, errors, skipped_runtime, db_shrunk)``.
- ``db_shrunk`` holds ``(rel, live_counts, imported_counts)`` for every session database the
- import replaced with one holding fewer rows — allowed, but never silently (#100960).
- """
- errors = []
- restored = 0
- restored_external = 0
- skipped_runtime: list[str] = []
- db_shrunk: list[tuple[str, tuple[int, int], tuple[int, int]]] = []
- home_dir = Path.home().resolve()
- # Resolved once: every member is published via a temp file, and mkstemp
- # would otherwise create newly restored files as 0600.
- new_file_mode = _default_new_file_mode()
- for member in members:
- # External memory-provider state captured under the reserved
- # ``_external/`` arc prefix restores to its original home-relative
- # location (e.g. ~/.honcho/config.json), NOT under HERMES_HOME.
- if member.startswith(_EXTERNAL_PREFIX):
- if not member[len(_EXTERNAL_PREFIX):]:
- continue
- if _import_external_member(zf, member, home_dir, new_file_mode, errors):
- restored += 1
- restored_external += 1
- if restored % 500 == 0:
- print(f" {restored}/{file_count} files ...")
- continue
- # Strip prefix if detected
- if prefix and member.startswith(prefix):
- rel = member[len(prefix):]
- else:
- rel = member
- if not rel:
- continue
- try:
- parts = tuple(normalize_archive_parts(rel))
- except ValueError:
- errors.append(f" {rel}: path traversal blocked")
- continue
- # Never overwrite volatile gateway/process runtime state. These are
- # namespaced to the machine/container the backup was taken on;
- # clobbering them (especially gateway_state.json) breaks the gateway
- # reconciler on the target and disconnects hosted instances from the
- # Nous portal. Matched by basename so both the root profile and
- # named profiles (profiles/<name>/gateway_state.json) are covered.
- if parts[-1] in _IMPORT_SKIP_NAMES:
- skipped_runtime.append(rel)
- continue
- # Older archives may contain PM selections pointing at another machine.
- # Match their home-root paths; a plugin's own facts.json is user data.
- if profile_root_entry(parts) in PM_RUNTIME_ROOT_DIRS:
- skipped_runtime.append(rel)
- continue
- # A ``.db`` member is page-restored into the live file below; a
- # WAL/SHM/journal member from the archive describes a different
- # database image, and installing it beside the restored file (over
- # a live sidecar, via os.replace) would replay a foreign WAL on
- # the next open. Current backups never ship these
- # (_EXCLUDED_SUFFIXES); older or hand-built archives might.
- if rel.endswith(_SQLITE_SIDECAR_SUFFIXES):
- skipped_runtime.append(rel)
- continue
- target = hermes_root.joinpath(*parts)
- # Security: reject absolute paths and traversals
- try:
- target.resolve().relative_to(hermes_root.resolve())
- except ValueError:
- errors.append(f" {rel}: path traversal blocked")
- continue
- try:
- target.parent.mkdir(parents=True, exist_ok=True)
- if target.suffix == ".db":
- # Count before the write: afterwards the rows this import
- # drops are gone and there is nothing left to compare.
- before = _count_session_rows(target)
- _import_db_member(zf, member, target, new_file_mode)
- after = _count_session_rows(target)
- if before and after and after[1] < before[1]:
- db_shrunk.append((rel, before, after))
- else:
- _extract_member_atomically(zf, member, target, new_file_mode)
- if target.name in _SECRET_FILE_NAMES:
- os.chmod(target, 0o600)
- restored += 1
- except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
- errors.append(f" {rel}: {exc}")
- if restored % 500 == 0:
- print(f" {restored}/{file_count} files ...")
- return restored, restored_external, errors, skipped_runtime, db_shrunk
- def _revive_gateway_after_import(hermes_root: Path) -> None:
- """Start an installed gateway service after a restore, best-effort and prompt-free.
- Bot tokens and cron jobs are inert without a gateway (a platform-less gateway is supported, so
- this is safe for any backup); failures print a manual fallback, never fail the import. Only
- revived when the restore landed in the default home or no other install exists: a sandbox or
- profile restore must not install a second gateway on the default service name.
- """
- native_default = _get_platform_default_hermes_home()
- if hermes_root != native_default and any(
- (native_default / marker).exists() for marker in ("config.yaml", ".env", "state.db")):
- print("\nRestored into a non-default home; leaving the gateway service alone to avoid clashing "
- f"with the install at {native_default}.\n"
- "To start a gateway for this home, run: hermes gateway run")
- return
- try:
- from hermes_cli.gateway import _is_service_running
- from hermes_cli.gateway_setup_service import ensure_gateway_service
- if not _is_service_running():
- print()
- ensure_gateway_service(context="import")
- except Exception:
- # Import boundary: the restore already succeeded; any service failure prints the
- # manual fallback instead of failing the import.
- logger.debug("gateway revive after import failed", exc_info=True)
- print("\nStart the gateway to activate cron jobs and messaging:\n hermes gateway run")
- # --- Quick state snapshots (used by /snapshot slash command and hermes backup --quick) ---
- # Critical state files (relative to HERMES_HOME) for quick snapshots; everything else is
- # regeneratable or managed separately (skills, repo, sessions/). Entries may be files OR
- # directories (recursive); missing entries are skipped. Pairing data lives in platform JSON blobs
- # outside state.db, so it is listed explicitly — ``hermes update`` snapshots this set (#15733).
- _QUICK_STATE_FILES = (
- "state.db", "config.yaml", ".env", "auth.json", "cron/jobs.json", "cron/executions.db",
- "gateway_state.json", "channel_directory.json", "channel_aliases.json", "processes.json",
- "gateway/discord_message_recovery.db", # Discord reconnect replay ledger
- # Per-profile user stores, destroyed if the update flow replaces the file and the post-update
- # schema-init re-creates an empty one (#52889). Skipped when outside HERMES_HOME.
- "projects.db", # per-profile project store
- "response_store.db", # gateway conversation history / tool payloads
- "memory_store.db", # holographic memory facts/entities
- "verification_evidence.db", # agent verification audit trail
- "kanban.db", # default board (back-compat <root>/kanban.db)
- "kanban/boards", # non-default boards (workspaces/ + attachments/ skipped as regenerable)
- # Pairing stores (generic + per-platform JSONs outside state.db)
- "pairing", # legacy location (gateway/pairing.py)
- "platforms/pairing", # new location (gateway/pairing.py)
- "feishu_comment_pairing.json", # Feishu comment subscription pairings
- )
- _QUICK_DEFAULT_KEEP = 20
- def _quick_snapshot_root(hermes_home: Optional[Path] = None) -> Path:
- home = hermes_home or get_hermes_home()
- return home / _QUICK_SNAPSHOTS_DIR
- def _newest_first(root: Path, keep_entry) -> list[Path]:
- """Entries of *root* passing ``keep_entry``, newest (by name) first; ``[]`` if *root* is missing."""
- if not root.exists():
- return []
- return sorted(filter(keep_entry, root.iterdir()), key=lambda p: p.name, reverse=True)
- # Kept in sync with ``_QUICK_STATE_FILES`` and ``cron/jobs.py``'s ``JOBS_FILE``.
- _CRON_JOBS_REL = "cron/jobs.json"
- def _prune_oldest(newest_first: list[Path], keep: int, remove, what: str) -> int:
- """``remove(path)`` every entry past the first *keep*; return how many succeeded."""
- deleted = 0
- for p in newest_first[keep:]:
- try:
- remove(p)
- deleted += 1
- except OSError as exc:
- logger.warning("Failed to prune %s %s: %s", what, p.name, exc)
- return deleted
- # --- Pre-update / pre-migration auto-backups ---
- _PRE_UPDATE_BACKUPS_DIR = "backups"
- _PRE_UPDATE_PREFIX = "pre-update-"
- _PRE_UPDATE_DEFAULT_KEEP = 5
- _PRE_MIGRATION_PREFIX = "pre-migration-"
- _PRE_MIGRATION_DEFAULT_KEEP = 5
- _INCOMPLETE_ZIP_SUFFIX = ".incomplete.zip"
- def _prune_prefixed_zips(backup_dir: Path, prefix: str, keep: int, what: str) -> int:
- """Remove oldest ``<prefix>*.zip`` in *backup_dir* beyond *keep*; return count deleted.
- Only prefix-matched files are touched, so hand-made zips or other backup kinds survive.
- """
- backups = _newest_first(backup_dir, lambda p: p.is_file() and p.name.startswith(prefix)
- and p.suffix.lower() == ".zip"
- and not p.name.lower().endswith(_INCOMPLETE_ZIP_SUFFIX))
- return _prune_oldest(backups, keep, Path.unlink, what)
- def _prune_incomplete_zips(backup_dir: Path, prefix: str, what: str) -> int:
- """Keep only the newest ``<prefix>*.incomplete.zip`` salvage archive; return count deleted.
- Salvage archives never count toward normal retention, so repeated failing runs would
- otherwise pile up without bound.
- """
- salvage = _newest_first(backup_dir, lambda p: p.is_file() and p.name.startswith(prefix)
- and p.name.lower().endswith(_INCOMPLETE_ZIP_SUFFIX))
- return _prune_oldest(salvage, 1, Path.unlink, f"incomplete {what}")
- def _create_prefixed_full_backup(
- hermes_home: Optional[Path], prefix: str, keep: int, what: str, prune_what: str) -> Optional[Path]:
- """Write ``<HERMES_HOME>/backups/<prefix><timestamp>.zip`` and prune older same-prefix zips.
- Returns the path, or ``None`` if nothing to back up, the write failed, or the archive is
- incomplete (kept as ``<prefix><timestamp>.incomplete.zip``, excluded from retention).
- Never raises."""
- hermes_root = hermes_home or get_default_hermes_root()
- if not hermes_root.is_dir():
- return None
- backup_dir = hermes_root / _PRE_UPDATE_BACKUPS_DIR
- try:
- backup_dir.mkdir(parents=True, exist_ok=True)
- except OSError as exc:
- logger.warning("Could not create %s backup dir %s: %s", what, backup_dir, exc)
- return None
- out_path = backup_dir / f"{prefix}{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
- if _write_full_zip_backup(out_path, hermes_root) is None:
- # Incomplete runs publish straight to ``*.incomplete.zip`` (all-failed runs are discarded);
- # cap those salvages at one without letting them rotate complete backups out.
- _prune_incomplete_zips(backup_dir, prefix, prune_what)
- return None
- _prune_prefixed_zips(backup_dir, prefix, keep, prune_what)
- return out_path
- def create_pre_update_backup(
- hermes_home: Optional[Path] = None, keep: int = _PRE_UPDATE_DEFAULT_KEEP) -> Optional[Path]:
- """Full zip backup to ``backups/pre-update-<timestamp>.zip``, auto-pruned; ``None`` if nothing
- was found, the backup failed, or it was incomplete (salvage kept as ``*.incomplete.zip``).
- Never raises — ``hermes update`` continues anyway."""
- return _create_prefixed_full_backup(hermes_home, _PRE_UPDATE_PREFIX, max(keep, 1), "pre-update", "backup")
- def create_pre_migration_backup(
- hermes_home: Optional[Path] = None, keep: int = _PRE_MIGRATION_DEFAULT_KEEP) -> Optional[Path]:
- """Full zip backup to ``backups/pre-migration-<timestamp>.zip`` before a plugin's agent import
- (``hermes import-agent openclaw``), so its name and signature are API; same dir as update backups
- so ``hermes import`` finds it. ``None`` if nothing was found, the write failed, or it was
- incomplete (salvage kept as ``*.incomplete.zip``). Never raises."""
- return _create_prefixed_full_backup(
- hermes_home, _PRE_MIGRATION_PREFIX, max(keep, 0), "pre-migration", "pre-migration backup")
- # ---------------------------------------------------------------------------
- # Quick state snapshots (used by /snapshot slash command and hermes backup --quick)
- # ---------------------------------------------------------------------------
- def create_quick_snapshot(
- label: Optional[str] = None,
- hermes_home: Optional[Path] = None,
- keep: Optional[int] = None,
- max_file_size: Optional[int] = None,
- ) -> Optional[str]:
- """Create one atomic quick snapshot while holding the shared backup slot."""
- home = hermes_home or get_hermes_home()
- with _backup_operation_lock(home):
- return _create_quick_snapshot_locked(
- label=label,
- hermes_home=home,
- keep=keep,
- max_file_size=max_file_size,
- )
- def _create_quick_snapshot_locked(
- label: Optional[str] = None,
- hermes_home: Optional[Path] = None,
- keep: Optional[int] = None,
- max_file_size: Optional[int] = None,
- ) -> Optional[str]:
- """Create a quick state snapshot of critical files.
- Copies STATE_FILES to a timestamped directory under state-snapshots/.
- Auto-prunes old snapshots beyond the keep limit.
- Args:
- max_file_size: When set, individual files larger than this many bytes
- are skipped (with a printed warning) instead of copied. Used by
- the pre-update safety snapshot so a multi-GB ``state.db`` can
- never stall ``hermes update`` or silently eat disk — the small
- pairing/cron/config files the snapshot exists to protect are
- always captured. ``None`` (default) copies everything, which
- preserves manual ``/snapshot`` and ``hermes backup --quick``
- behavior.
- Returns:
- Snapshot ID (timestamp-based), or None if no files found.
- """
- home = hermes_home or get_hermes_home()
- root = _quick_snapshot_root(home)
- def _too_large(path: Path, rel_name: str) -> bool:
- """True (and warn) when ``path`` exceeds the max_file_size cap."""
- if max_file_size is None:
- return False
- try:
- size = path.stat().st_size
- except OSError:
- return False
- if size <= max_file_size:
- return False
- print(
- f" ⚠ Snapshot: skipping {rel_name} "
- f"({_format_size(size)} exceeds {_format_size(max_file_size)} limit)"
- )
- logger.warning(
- "Quick snapshot skipped %s: %d bytes exceeds %d byte limit",
- rel_name,
- size,
- max_file_size,
- )
- return True
- ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
- base_snap_id = f"{ts}-{label}" if label else ts
- snap_id = base_snap_id
- suffix = 2
- while (root / snap_id).exists():
- snap_id = f"{base_snap_id}-{suffix}"
- suffix += 1
- snap_dir = root / snap_id
- staging_dir = root / f".{snap_id}.{os.getpid()}.partial"
- shutil.rmtree(staging_dir, ignore_errors=True)
- root.mkdir(parents=True, exist_ok=True, mode=0o700)
- if os.name != "nt":
- os.chmod(root, 0o700)
- staging_dir.mkdir(mode=0o700, exist_ok=False)
- logger.info("quick snapshot phase=copy status=started id=%s", snap_id)
- manifest: dict[str, int] = {} # rel_path -> file size
- failed_dbs: list[str] = [] # present *.db that could not be snapshotted
- # #68805: track protected DB files skipped for size — they are snapshot
- # incompleteness just like a failed copy, so pruning must be suppressed
- # to preserve the older complete snapshot that may contain the only
- # recoverable database.
- oversized_skipped: list[str] = []
- for rel in _QUICK_STATE_FILES:
- src = home / rel
- if not src.exists():
- continue
- if src.is_dir():
- # Walk the directory and record each file individually in the
- # manifest so restore can treat them uniformly. Empty dirs are
- # skipped (nothing to snapshot).
- for sub in src.rglob("*"):
- if not sub.is_file():
- continue
- sub_rel = sub.relative_to(home).as_posix()
- # Skip heavy, regenerable per-board subtrees (scratch
- # workspaces and task attachments can be large); we only need
- # the board databases + their metadata to restore a board.
- if "/workspaces/" in f"/{sub_rel}/" or "/attachments/" in f"/{sub_rel}/":
- continue
- if _too_large(sub, sub_rel):
- if sub.suffix == ".db":
- oversized_skipped.append(sub_rel)
- continue
- dst = staging_dir / sub_rel
- dst.parent.mkdir(parents=True, exist_ok=True)
- try:
- # Route SQLite DBs through the WAL-safe backup() path so a
- # board DB with an open WAL (the gateway may hold it at
- # snapshot time) is captured consistently.
- if sub.suffix == ".db":
- if not _safe_copy_db(sub, dst):
- failed_dbs.append(sub_rel)
- print(
- f" ⚠ Snapshot: SQLite safe copy FAILED for {sub_rel} "
- f"— file may be locked or corrupted"
- )
- if is_zeroed_sqlite_file(sub):
- print(
- f" ⚠ Snapshot: {sub_rel} looks ZEROED "
- f"(no SQLite header; {sub.stat().st_size} bytes of NULs?)"
- )
- continue
- else:
- shutil.copy2(sub, dst)
- manifest[sub_rel] = dst.stat().st_size
- except (OSError, PermissionError) as exc:
- logger.warning("Could not snapshot %s: %s", sub_rel, exc)
- continue
- if not src.is_file():
- continue
- if _too_large(src, rel):
- if src.suffix == ".db":
- oversized_skipped.append(rel)
- continue
- dst = staging_dir / rel
- dst.parent.mkdir(parents=True, exist_ok=True)
- try:
- if src.suffix == ".db":
- if not _safe_copy_db(src, dst):
- failed_dbs.append(rel)
- print(
- f" ⚠ Snapshot: SQLite safe copy FAILED for {rel} "
- f"— file may be locked or corrupted"
- )
- if is_zeroed_sqlite_file(src):
- print(
- f" ⚠ Snapshot: {rel} looks ZEROED "
- f"(no SQLite header; {src.stat().st_size} bytes)"
- )
- continue
- else:
- shutil.copy2(src, dst)
- manifest[rel] = dst.stat().st_size
- except (OSError, PermissionError) as exc:
- logger.warning("Could not snapshot %s: %s", rel, exc)
- if failed_dbs:
- # Critical: update path used to log-and-continue with exit 0, so a
- # missing state.db backup looked like a successful pre-update snapshot
- # (#68474). Surface this on stdout where operators actually look.
- print(
- " ⚠ CRITICAL: could not snapshot DB file(s): "
- + ", ".join(failed_dbs)
- )
- print(
- f" ⚠ If sessions disappear after the update, check {root}. {_snapshot_recovery_hint()}"
- )
- logger.error(
- "Quick snapshot failed to capture DB file(s): %s",
- ", ".join(failed_dbs),
- )
- if not manifest:
- shutil.rmtree(staging_dir, ignore_errors=True)
- if failed_dbs:
- # Distinguish "nothing to snapshot" from "state.db present but unreadable"
- print(
- " ⚠ Snapshot aborted: no files captured "
- f"(failed DBs: {', '.join(failed_dbs)})"
- )
- return None
- # Write manifest
- meta = {
- "id": snap_id,
- "timestamp": ts,
- "label": label,
- "file_count": len(manifest),
- "total_size": sum(manifest.values()),
- "files": manifest,
- "failed_dbs": failed_dbs,
- "oversized_skipped": oversized_skipped,
- }
- with open(staging_dir / "manifest.json", "w", encoding="utf-8") as f:
- json.dump(meta, f, indent=2)
- # Make the staged quick snapshot owner-only before it is published. The
- # staging directory is private from creation, so copied source modes can
- # be normalized safely before the final atomic rename exposes the
- # snapshot. Permission failures are intentionally fatal: publishing a
- # readable recovery bundle is worse than reporting a failed snapshot.
- if os.name != "nt":
- os.chmod(root, 0o700)
- os.chmod(staging_dir, 0o700)
- for path in staging_dir.rglob("*"):
- if path.is_dir():
- os.chmod(path, 0o700)
- elif path.is_file():
- os.chmod(path, 0o600)
- os.replace(staging_dir, snap_dir)
- # Auto-prune. Defaults preserve historical manual /snapshot behavior; callers
- # with known high-churn safety snapshots (for example pre-update) can pass a
- # smaller keep value so large state.db copies do not accumulate indefinitely.
- # #68805 review: skip pruning when a present DB failed to capture OR was
- # skipped for size — either way the snapshot is incomplete and the older
- # snapshot may contain the only recoverable database.
- incomplete = failed_dbs or oversized_skipped
- if not incomplete:
- _prune_quick_snapshots(root, keep=_QUICK_DEFAULT_KEEP if keep is None else keep)
- else:
- if oversized_skipped:
- print(
- " ⚠ Skipping snapshot prune: DB file(s) skipped for size: "
- + ", ".join(oversized_skipped)
- )
- logger.warning(
- "Quick snapshot skipped oversized DB file(s): %s",
- ", ".join(oversized_skipped),
- )
- logger.warning(
- "Skipping snapshot prune because %d DB(s) failed to capture "
- "and/or %d were oversized — preserving older snapshots as "
- "recovery source",
- len(failed_dbs), len(oversized_skipped),
- )
- logger.info(
- "quick snapshot phase=copy status=complete id=%s files=%d bytes=%d",
- snap_id,
- len(manifest),
- sum(manifest.values()),
- )
- return snap_id
- def list_quick_snapshots(
- limit: int = 20,
- hermes_home: Optional[Path] = None,
- ) -> list[dict[str, Any]]:
- """List existing quick state snapshots, most recent first."""
- root = _quick_snapshot_root(hermes_home)
- if not root.exists():
- return []
- results = []
- for d in sorted(root.iterdir(), reverse=True):
- if not d.is_dir() or d.name.startswith(".") or d.name.endswith(".partial"):
- continue
- manifest_path = d / "manifest.json"
- if manifest_path.exists():
- try:
- with open(manifest_path, encoding="utf-8-sig") as f:
- results.append(json.load(f))
- except (json.JSONDecodeError, OSError):
- results.append({"id": d.name, "file_count": 0, "total_size": 0})
- if len(results) >= limit:
- break
- return results
- def _is_trusted_root_auth_alias(path: Path, home: Path) -> bool:
- """True only for a file symlink resolving to this restore home's default-root auth store."""
- if not path.is_symlink():
- return False
- try:
- root = get_default_hermes_root(home=home).resolve(strict=False)
- if home.resolve(strict=False) == root:
- return False
- trusted = (root / "auth.json").resolve(strict=False)
- trusted.relative_to(root)
- return path.resolve(strict=False) == trusted
- except (OSError, RuntimeError, ValueError):
- return False
- def restore_quick_snapshot(
- snapshot_id: str,
- hermes_home: Optional[Path] = None,
- ) -> bool:
- """Restore the whole snapshot offline, or refuse before changing any file.
- Overwrites current state files with the snapshot's copies.
- Returns True if at least one file was restored and the listed auth.json
- was not refused or skipped.
- """
- from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
- require_file_tooling("Snapshot restore")
- home = hermes_home or get_hermes_home()
- try:
- with exclusive_maintenance([home]):
- return _restore_quick_snapshot_exclusive(snapshot_id, home)
- except OwnershipConflict as exc:
- logger.error("%s", exc)
- return False
- def _snapshot_epochs_restorable(files, entry_paths) -> bool:
- """False (logged) when any listed ``state.db`` destination cannot prove its runtime epoch."""
- for rel in files:
- entry = entry_paths(rel)
- if entry and entry[0].exists() and "state.db" in (entry[1].name, entry[1].resolve().name):
- try:
- _restore_destination_epoch(entry[1])
- except OSError as exc:
- logger.error("%s", exc)
- return False
- return True
- def _restore_snapshot_entry(rel: str, src: Path, dst: Path) -> bool:
- """Publish one quick-snapshot file over ``dst``; False (logged) when refused or failed."""
- try:
- if dst.suffix == ".db":
- # Restore through SQLite backup API so live connections
- # (gateway, dashboard, another CLI session) see the
- # restored data instead of continuing to serve stale
- # cached pages from a replaced inode (issue #65942).
- if not _safe_restore_db(src, dst):
- # Refused, failed, or source failed its integrity check:
- # dst left as it was. Count as a failure, not a restore.
- logger.error("Failed to restore %s: refused or source integrity check failed (see previous log)", rel)
- return False
- elif rel == "auth.json":
- # Refresh tokens for these OAuth providers rotate on use. A historical
- # snapshot can therefore contain a spent pair even though the current
- # auth.json has the live successor. Restore the historical auth state
- # while retaining that live single-use grant under the auth-store lock.
- if not _restore_auth_json(src, dst):
- logger.error("Failed to restore %s safely", rel)
- return False
- else:
- shutil.copy2(src, dst)
- return True
- except (OSError, PermissionError) as exc:
- logger.error("Failed to restore %s: %s", rel, exc)
- return False
- def quick_snapshot_dir(snapshot_id: str, hermes_home: Optional[Path] = None) -> Optional[Path]:
- """The directory of a restorable quick snapshot, or None for an invalid or missing id."""
- root = _quick_snapshot_root(hermes_home)
- # Security: reject snapshot_id values that contain path separators or
- # traversal sequences so that `root / snapshot_id` stays inside root.
- if not snapshot_id or "/" in snapshot_id or "\\" in snapshot_id or snapshot_id in (".", ".."):
- logger.error("Invalid snapshot_id: %s", snapshot_id)
- return None
- snap_dir = root / snapshot_id
- # Confirm the resolved path is still inside root (handles symlinks etc.)
- try:
- snap_dir.resolve().relative_to(root.resolve())
- except ValueError:
- logger.error("Snapshot path traversal blocked for id: %s", snapshot_id)
- return None
- if not snap_dir.is_dir() or not (snap_dir / "manifest.json").exists():
- return None
- return snap_dir
- def _restore_quick_snapshot_exclusive(snapshot_id: str, home: Path) -> bool:
- """Restore state from a quick snapshot while ``home`` is reserved."""
- from gateway.runtime_ownership import exclusive_maintenance
- snap_dir = quick_snapshot_dir(snapshot_id, home)
- if snap_dir is None:
- return False
- with open(snap_dir / "manifest.json", encoding="utf-8-sig") as f:
- meta = json.load(f)
- snap_res, home_res = snap_dir.resolve(), home.resolve()
- def _entry_paths(rel: str) -> Optional[tuple[Path, Path]]:
- # Security: reject absolute paths and traversals in manifest entries
- src, dst = snap_dir / rel, home / rel
- if not src.resolve().is_relative_to(snap_res):
- return None
- if not dst.resolve().is_relative_to(home_res):
- # Named profiles may deliberately share the machine-root auth store via an
- # auth.json symlink. Let only that exact trusted alias reach _restore_auth_json;
- # every other manifest destination outside the profile remains a traversal.
- if rel != "auth.json" or not _is_trusted_root_auth_alias(dst, home):
- return None
- return src, dst
- homes = {home}
- for rel in meta.get("files", {}):
- entry = _entry_paths(rel)
- if entry and entry[1].suffix == ".db":
- homes.update([entry[1].absolute().parent, entry[1].resolve().parent])
- with exclusive_maintenance(homes):
- # Epoch safety is a whole-profile preflight: copying config first would
- # both partially roll back the profile and report a refused DB as success.
- if not _snapshot_epochs_restorable(meta.get("files", {}), _entry_paths):
- return False
- restored = 0
- auth_restore_failed = False
- for rel in meta.get("files", {}):
- entry = _entry_paths(rel)
- if entry is None:
- logger.error("Manifest path traversal blocked: %s", rel)
- if rel == "auth.json":
- auth_restore_failed = True
- continue
- src, dst = entry
- if not src.exists():
- if rel == "auth.json":
- logger.error("Snapshot auth.json listed in manifest is missing: %s", src)
- auth_restore_failed = True
- continue
- dst.parent.mkdir(parents=True, exist_ok=True)
- if _restore_snapshot_entry(rel, src, dst):
- restored += 1
- elif rel == "auth.json":
- auth_restore_failed = True
- logger.info("Restored %d files from snapshot %s", restored, snapshot_id)
- return restored > 0 and not auth_restore_failed
- def restore_cron_jobs_if_emptied(
- snapshot_id: str,
- hermes_home: Optional[Path] = None,
- ) -> Optional[dict[str, Any]]:
- """Safety net for silent cron-job loss across ``hermes update``.
- Config-version migrations have been observed to leave ``cron/jobs.json``
- valid-but-empty after an update, silently dropping every scheduled job
- (issue #34600). The desktop scheduler can also overwrite the file with its
- own small set of internally-tracked crons, causing partial loss (issue
- #52144).
- This compares the *current* job count against the pre-update snapshot. If
- the live file now has **fewer** jobs than the snapshot, the snapshot copy
- of ``cron/jobs.json`` is restored in place.
- The check is deliberately conservative — it only ever restores when there
- is unambiguous evidence of loss (snapshot had more jobs than live file),
- so a user who genuinely deleted jobs during/after the update is never
- second-guessed, and an unreadable live file (count ``None``) is left
- untouched so real corruption still surfaces.
- Args:
- snapshot_id: The pre-update quick-snapshot id (from
- :func:`create_quick_snapshot`).
- hermes_home: Override for the Hermes home directory (tests).
- Returns:
- ``None`` when no action was taken (the common, healthy path). On a
- successful restore, a dict ``{"restored": True, "job_count": N,
- "snapshot_id": ...}`` so the caller can warn the user.
- """
- if not snapshot_id:
- return None
- home = hermes_home or get_hermes_home()
- live_path = home / _CRON_JOBS_REL
- live_count = _count_cron_jobs(live_path)
- # ``None`` (missing or unparseable) is intentionally left alone — that's a
- # different failure mode the user should see rather than have papered over.
- if live_count is None:
- return None
- snap_path = _quick_snapshot_root(home) / snapshot_id / _CRON_JOBS_REL
- snap_count = _count_cron_jobs(snap_path)
- if not snap_count: # None or 0 — nothing worth restoring
- return None
- # Restore when live has FEWER jobs than the pre-update snapshot.
- # Catches both total loss (0 vs N) and partial loss (1 vs 19) — the
- # desktop scheduler can overwrite jobs.json with its own small set of
- # internally-tracked crons after an update/restart.
- if live_count >= snap_count:
- return None
- try:
- live_path.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(snap_path, live_path)
- except (OSError, PermissionError) as exc:
- logger.error(
- "Cron jobs were emptied during update but auto-restore failed: %s", exc
- )
- return None
- logger.warning(
- "Restored %d cron job(s) from pre-update snapshot %s "
- "(live file had %d job(s), snapshot had %d — jobs were lost during migration)",
- snap_count,
- snapshot_id,
- live_count,
- snap_count,
- )
- return {"restored": True, "job_count": snap_count, "snapshot_id": snapshot_id}
- def _sibling_profile_homes(invoking_home: Path) -> list[tuple[str, Path]]:
- """(name, home) for every OTHER profile on this install. Never raises.
- The update's code swap and gateway fleet restart touch every profile,
- so the pre-update snapshot must too (#66140). The invoking profile is
- excluded — its snapshot is taken by the existing call.
- """
- homes: list[tuple[str, Path]] = []
- try:
- from hermes_cli.profiles import (
- _get_default_hermes_home,
- _get_profiles_root,
- _PROFILE_ID_RE,
- )
- invoking = invoking_home.resolve()
- default_home = _get_default_hermes_home()
- if default_home.is_dir() and default_home.resolve() != invoking:
- homes.append(("default", default_home))
- root = _get_profiles_root()
- if root.is_dir():
- for entry in sorted(root.iterdir()):
- if (
- entry.is_dir()
- and entry.name != "default"
- and _PROFILE_ID_RE.match(entry.name)
- and entry.resolve() != invoking
- ):
- homes.append((entry.name, entry))
- except Exception as exc:
- logger.debug("Sibling profile enumeration failed: %s", exc)
- return homes
- def create_pre_update_snapshots_all_profiles(
- invoking_home: Optional[Path] = None,
- keep: Optional[int] = None,
- max_file_size: Optional[int] = None,
- ) -> dict[str, str]:
- """Pre-update quick snapshots for every SIBLING profile (#66140).
- Same snapshot set, same per-file size cap, same keep policy as the
- invoking profile's snapshot — identical semantics per profile, no
- partial-tier coherence class. Each sibling's snapshot lands under its
- OWN ``<home>/state-snapshots/`` so per-profile restore tooling finds
- it where it expects. Returns ``{profile_name: snapshot_id}`` for the
- siblings that snapshotted successfully. Never raises.
- """
- results: dict[str, str] = {}
- home = invoking_home or get_hermes_home()
- for name, profile_home in _sibling_profile_homes(home):
- try:
- snap_id = create_quick_snapshot(
- label="pre-update",
- hermes_home=profile_home,
- keep=keep,
- max_file_size=max_file_size,
- )
- if snap_id:
- results[name] = snap_id
- except Exception as exc:
- logger.debug("Pre-update snapshot for profile %s failed: %s", name, exc)
- return results
- # Config paths that the update flow must never change (#64160): the model
- # routing keys and the Mixture-of-Agents section are consumed machine-wide
- # (gateway, cron, desktop), so an update/repair cycle that rewrites them
- # silently redirects paid inference. Each entry is a dotted path into the raw
- # config.yaml document; a single-element tuple protects the whole section.
- _PROTECTED_CONFIG_PATHS: tuple[tuple[str, ...], ...] = (
- ("model", "provider"),
- ("model", "default"),
- ("model", "base_url"),
- ("model", "api_key"),
- ("moa",),
- )
- def _read_raw_yaml_dict(path: Path) -> Optional[dict[str, Any]]:
- """Parse ``path`` as a YAML mapping. ``None`` = missing/unreadable/non-dict."""
- if not path.is_file():
- return None
- try:
- import hermes_yaml as yaml
- with open(path, "r", encoding="utf-8-sig") as f:
- data = yaml.safe_load(f)
- except Exception:
- return None
- return data if isinstance(data, dict) else None
- def restore_config_model_settings_if_rewritten(
- snapshot_id: str,
- hermes_home: Optional[Path] = None,
- ) -> Optional[dict[str, Any]]:
- """Safety net for silent config.yaml model/MoA loss across ``hermes update``.
- Desktop update/repair cycles have been observed to rewrite user-set
- ``model.provider``/``model.default`` and drop the ``moa:`` section
- entirely (issue #64160; the macOS repair/relaunch variant rewrote a
- pinned ``model.default`` to a transient composer pick). These keys are
- consumed by the gateway and unattended cron jobs too, so a rewrite
- silently changes paid inference behavior machine-wide.
- Mirrors :func:`restore_cron_jobs_if_emptied`: compare the *current*
- config against the pre-update snapshot taken minutes earlier by this
- same update run, and restore only the protected keys — never the whole
- file — when a value the user had set was changed or dropped. Everything
- the update legitimately wrote (version stamps, new sections) is left in
- place.
- Args:
- snapshot_id: The pre-update quick-snapshot id (from
- :func:`create_quick_snapshot`).
- hermes_home: Override for the Hermes home directory (tests/siblings).
- Returns:
- ``None`` when no action was taken (the common, healthy path). On a
- successful restore, ``{"restored": True, "keys": [...],
- "snapshot_id": ...}`` so the caller can warn the user.
- """
- if not snapshot_id:
- return None
- if not supports_file_tooling(): # Remote Config: never write a LOCAL snapshot's values to the plane
- return None
- home = hermes_home or get_hermes_home()
- live_path = home / "config.yaml"
- snap_path = _quick_snapshot_root(home) / snapshot_id / "config.yaml"
- snap = _read_raw_yaml_dict(snap_path)
- if not snap:
- return None # no snapshot copy — nothing to compare against
- live = _read_raw_yaml_dict(live_path)
- if live is None:
- # Missing or unparseable live config is a different failure mode the
- # user should see rather than have papered over (matches the cron net).
- return None
- restored_keys: list[str] = []
- for dotted in _PROTECTED_CONFIG_PATHS:
- snap_val = _get_config_path_value(snap, dotted)
- if snap_val in (None, "", {}, []):
- continue # user never set it — nothing to protect
- live_val = _get_config_path_value(live, dotted)
- if live_val == snap_val:
- continue
- _set_config_path_value(live, dotted, snap_val)
- restored_keys.append(".".join(dotted))
- if not restored_keys:
- return None
- try:
- from hermes_cli.config import atomic_config_write
- atomic_config_write(live_path, live)
- except (OSError, PermissionError) as exc:
- logger.error(
- "config.yaml model settings were rewritten during update but "
- "auto-restore failed: %s",
- exc,
- )
- return None
- logger.warning(
- "Restored user config value(s) %s from pre-update snapshot %s — "
- "the update flow rewrote them (#64160)",
- ", ".join(restored_keys),
- snapshot_id,
- )
- return {"restored": True, "keys": restored_keys, "snapshot_id": snapshot_id}
- def restore_config_model_settings_all_profiles(
- profile_snapshots: dict[str, str],
- invoking_home: Optional[Path] = None,
- ) -> list[dict[str, Any]]:
- """Run the config model-settings safety net for every sibling profile.
- Same contract as :func:`restore_cron_jobs_all_profiles`: each profile's
- live ``config.yaml`` is compared against ITS OWN same-generation
- pre-update snapshot. Returns one result dict per restored profile, each
- with a ``profile`` key added. Never raises.
- """
- restored: list[dict[str, Any]] = []
- if not profile_snapshots:
- return restored
- home = invoking_home or get_hermes_home()
- by_name = dict(_sibling_profile_homes(home))
- for name, snap_id in profile_snapshots.items():
- profile_home = by_name.get(name)
- if profile_home is None:
- continue
- try:
- result = restore_config_model_settings_if_rewritten(
- snap_id, hermes_home=profile_home
- )
- except Exception as exc:
- logger.debug(
- "Config model-settings restore check for profile %s failed: %s",
- name,
- exc,
- )
- continue
- if result:
- result["profile"] = name
- restored.append(result)
- return restored
- def restore_cron_jobs_all_profiles(
- profile_snapshots: dict[str, str],
- invoking_home: Optional[Path] = None,
- ) -> list[dict[str, Any]]:
- """Run the cron-jobs safety net for every sibling profile (#66140).
- ``profile_snapshots`` is the map returned by
- :func:`create_pre_update_snapshots_all_profiles`. Each profile's live
- ``cron/jobs.json`` is compared against ITS OWN snapshot — restores are
- same-generation by construction (the snapshot was taken minutes ago by
- this update run). Returns one result dict per restored profile, each
- with a ``profile`` key added. Never raises.
- """
- restored: list[dict[str, Any]] = []
- if not profile_snapshots:
- return restored
- home = invoking_home or get_hermes_home()
- by_name = dict(_sibling_profile_homes(home))
- for name, snap_id in profile_snapshots.items():
- profile_home = by_name.get(name)
- if profile_home is None:
- continue
- try:
- result = restore_cron_jobs_if_emptied(snap_id, hermes_home=profile_home)
- except Exception as exc:
- logger.debug("Cron restore check for profile %s failed: %s", name, exc)
- continue
- if result:
- result["profile"] = name
- restored.append(result)
- return restored
- def _prune_quick_snapshots(root: Path, keep: int = _QUICK_DEFAULT_KEEP) -> int:
- """Remove oldest quick snapshots beyond the keep limit. Returns count deleted."""
- if not root.exists():
- return 0
- dirs = sorted(
- (
- d
- for d in root.iterdir()
- if d.is_dir() and not d.name.startswith(".") and not d.name.endswith(".partial")
- ),
- key=lambda d: d.name,
- reverse=True,
- )
- deleted = 0
- for d in dirs[keep:]:
- try:
- shutil.rmtree(d)
- deleted += 1
- except OSError as exc:
- logger.warning("Failed to prune snapshot %s: %s", d.name, exc)
- return deleted
- def prune_quick_snapshots(
- keep: int = _QUICK_DEFAULT_KEEP,
- hermes_home: Optional[Path] = None,
- ) -> int:
- """Manually prune quick snapshots. Returns count deleted."""
- return _prune_quick_snapshots(_quick_snapshot_root(hermes_home), keep=keep)
- def run_quick_backup(args) -> None:
- """CLI entry point for hermes backup --quick."""
- label = getattr(args, "label", None)
- snap_id = create_quick_snapshot(label=label)
- if snap_id:
- print(f"State snapshot created: {snap_id}")
- snaps = list_quick_snapshots()
- print(f" {len(snaps)} snapshot(s) stored in {display_hermes_home()}/state-snapshots/")
- print(f" Restore with: /snapshot restore {snap_id}")
- else:
- print("No state files found to snapshot.")
- # ---------------------------------------------------------------------------
- # Shared full-zip backup helper
- # ---------------------------------------------------------------------------
- def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
- """Single-flight wrapper for automatic full zip backups."""
- try:
- with _backup_operation_lock(hermes_root):
- return _write_full_zip_backup_locked(out_path, hermes_root)
- except BackupInProgressError as exc:
- logger.warning("Full-zip backup skipped: %s", exc)
- return None
- def _write_full_zip_backup_locked(out_path: Path, hermes_root: Path) -> Optional[Path]:
- scan_started = time.monotonic()
- logger.info("automatic backup phase=scan status=started")
- try:
- files_to_add = list(_iter_backup_files(hermes_root, out_path))
- except OSError as exc:
- logger.warning("Full-zip backup: walk failed: %s", exc)
- return None
- if not files_to_add:
- return None
- logger.info("automatic backup phase=scan status=complete duration_ms=%.1f files=%d",
- (time.monotonic() - scan_started) * 1000, len(files_to_add))
- def _db_failure(rel_path: Path) -> None:
- logger.warning("Full-zip backup aborted: SQLite snapshot failed for %s", rel_path)
- raise _SQLiteSnapshotError(str(rel_path))
- errors: list[str] = []
- def _capped_errors() -> str:
- # Cap the logged list: a broken tree can fail thousands of entries in one run.
- shown = "; ".join(errors[:10])
- return f"{shown} (+{len(errors) - 10} more)" if len(errors) > 10 else shown
- # Salvage name keeps an incomplete archive out of normal retention (otherwise the next
- # complete run would prune the last complete backups by count) and, because the partial is
- # published straight there, never clobbers a previous good backup at ``out_path``. A run where
- # every entry failed salvages nothing, so its empty archive is discarded rather than kept.
- salvage_path = out_path.with_name(out_path.stem + _INCOMPLETE_ZIP_SUFFIX)
- published: Optional[Path] = None
- def _publish_path() -> Optional[Path]:
- # Decide clean/salvage/discard once; the post-publish stat and return reuse it.
- nonlocal published
- if not errors:
- published = out_path
- elif len(errors) < len(files_to_add):
- published = salvage_path
- return published
- archive_started = time.monotonic()
- try:
- with _atomic_output_path(out_path, _publish_path) as archive_path, zipfile.ZipFile(
- archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
- _write_zip_entries(
- zf, files_to_add, out_path, on_db_failure=_db_failure, track_bytes=False,
- on_error=lambda rel, exc: errors.append(f"{rel}: {exc}"),
- on_progress=lambda i: logger.info(
- "automatic backup phase=archive status=progress completed=%d total=%d", i, len(files_to_add)))
- except (OSError, _SQLiteSnapshotError) as exc:
- # The hidden partial is already gone; ``out_path`` may be a previous valid backup: keep it.
- logger.warning("Full-zip backup: zip write failed: %s", exc)
- return None
- if published is None:
- logger.warning("Full-zip backup: every entry failed, nothing salvaged: %s", _capped_errors())
- return None
- zip_size = published.stat().st_size
- if published != out_path:
- logger.warning(
- "automatic backup phase=archive status=incomplete duration_ms=%.1f files=%d errors=%d "
- "bytes=%d salvage=%s skipped=%s",
- (time.monotonic() - archive_started) * 1000, len(files_to_add), len(errors), zip_size,
- salvage_path, _capped_errors())
- return None
- logger.info("automatic backup phase=archive status=complete duration_ms=%.1f files=%d bytes=%d",
- (time.monotonic() - archive_started) * 1000, len(files_to_add), zip_size)
- return out_path
- # ---------------------------------------------------------------------------
|