backup.py 92 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134
  1. """Backup and import commands for hermes CLI."""
  2. import json
  3. import logging
  4. import os
  5. import shutil
  6. import sqlite3
  7. import stat
  8. import sys
  9. import tempfile
  10. import threading
  11. import time
  12. import zipfile
  13. import zlib
  14. from contextlib import contextmanager, suppress
  15. from datetime import datetime, timezone, UTC
  16. from pathlib import Path
  17. from typing import Any, Callable, Dict, List, Optional, Tuple
  18. from hermes_constants import (
  19. LOCAL_RUNTIME_ROOT_DIRS, _get_platform_default_hermes_home, get_default_hermes_root, get_hermes_home,
  20. display_hermes_home,
  21. )
  22. from hermes_state_dbfile import RETIRED_GENERATION_DIR_SUFFIX
  23. from hermes_state_holders import read_only_db_uri
  24. from agent.provider_media import GENERATED_SUBDIR
  25. from hermes_cli.archive_safe import normalize_archive_parts
  26. from hermes_cli.backup_sqlite import _close_quietly, _safe_copy_db
  27. from hermes_cli.config_backend import require_file_tooling, supports_file_tooling
  28. from hermes_cli.home_data_layout import PM_RUNTIME_ROOT_DIRS, SECRET_CACHE_FILES, profile_root_entry
  29. from hermes_cli.sizefmt import format_bytes as _format_size
  30. from hermes_cli.backup_restore import (
  31. _count_cron_jobs,
  32. _count_session_rows,
  33. _default_new_file_mode,
  34. _detect_prefix,
  35. _extract_member_atomically,
  36. _get_config_path_value,
  37. _import_db_member,
  38. _restore_auth_json,
  39. _restore_destination_epoch,
  40. _safe_restore_db,
  41. _set_config_path_value,
  42. _validate_backup_zip,
  43. )
  44. logger = logging.getLogger(__name__)
  45. def _foreign_db_holder_pids(db_path: Path) -> Optional[list[int]]:
  46. # Shim to stop the old updater doing work until relaunch. None means unknown,
  47. # not permission to restore over a database whose holders we did not scan.
  48. return None
  49. # --- Exclusion rules ---
  50. # Where ``hermes backup --quick`` / ``/snapshot`` / the pre-update safety net write state
  51. # snapshots (see ``create_quick_snapshot``); defined here because the exclusion set needs it.
  52. _QUICK_SNAPSHOTS_DIR = "state-snapshots"
  53. def _snapshot_recovery_hint() -> str:
  54. """How to restore a state snapshot. There is no `hermes snapshot` subcommand — only the /snapshot
  55. slash command inside a `hermes` session (hermes_cli/commands.py)."""
  56. return ("To restore a newer snapshot, start `hermes` in a terminal and run `/snapshot list`, then "
  57. "`/snapshot restore <id>` (CLI only).")
  58. # Directory names to skip (matched against each path component). ``hermes-agent`` only matches at
  59. # the root (``_should_exclude``) so skill dirs like ``skills/.../hermes-agent/`` survive. The
  60. # dependency/cache entries matter: one plugin venv or pip/uv cache under HERMES_HOME walked
  61. # file-by-file balloons a backup to hundreds of thousands of entries ("backup stuck for days").
  62. # Mostly mirrors ``agent.skill_utils.EXCLUDED_SKILL_DIRS``; ``.cache`` is backup-only. ``.archive``
  63. # is deliberately NOT excluded: the curator's ``skills/.archive/`` holds restorable user skills.
  64. _EXCLUDED_DIRS = {
  65. "hermes-agent", # the codebase repo — re-clone instead
  66. "__pycache__", # bytecode caches — regenerated on import
  67. ".git", # nested git dirs (profiles shouldn't have these, but safety)
  68. "node_modules", # js deps — reinstalled on demand
  69. "backups", # prior auto-backups — don't nest backups exponentially
  70. _QUICK_SNAPSHOTS_DIR, # each holds a full state.db copy — same reason as ``backups``
  71. "checkpoints", # session-hash-keyed trajectory caches — regenerated, don't port
  72. # Live CDP browser profiles: Chromium holds their SQLite DBs exclusively locked while running
  73. # and sqlite3.backup() retries SQLITE_BUSY forever, hanging the backup. Regenerable anyway.
  74. "browser-profiles",
  75. # Real-profile browsing snapshot (browser.use_real_profile): copies of the user's Cookies /
  76. # Login Data — a credential store that must NOT enter an archive. Regenerated on next launch.
  77. "browser-profile",
  78. # Python dependency trees (plugin / MCP-server venvs) — regenerated by reinstalling.
  79. ".venv", "venv", "site-packages",
  80. # Tool / build caches — all regeneratable.
  81. ".cache", ".tox", ".nox", ".pytest_cache", ".mypy_cache", ".ruff_cache",
  82. }
  83. # Hermes-managed runtime downloads are regenerable. Match only profile roots:
  84. # a deeper directory of the same name (such as a skill's models/) is user data.
  85. _EXCLUDED_ROOT_DIRS = LOCAL_RUNTIME_ROOT_DIRS | (PM_RUNTIME_ROOT_DIRS - {"cache"})
  86. # Browser Use CLI profile dir (browser.backend: browser-use): Chromium user-data with Login Data / Cookies;
  87. # the secret caches and Photon sidecar token. Root-scoped like models/ — a skill's own browser_profiles/ is
  88. # user data. Backup-only: do not fold into LOCAL_RUNTIME_ROOT_DIRS (clone-all identity contract).
  89. _EXCLUDED_BACKUP_ROOT_DIRS = frozenset({"browser_profiles", *SECRET_CACHE_FILES})
  90. # ``cache/`` at those same roots mixes regenerable state (model/plugin catalogs, stamps, browser
  91. # profiles with locked SQLite, tool-output spill) with durable artifacts nothing can rebuild: media
  92. # the gateway delivered to or received from the user (``gateway.platforms.base``'s media-delivery
  93. # subdirs) and the grounded-citations evidence ledger. Only these subdirs are archived.
  94. _KEPT_CACHE_SUBDIRS = {
  95. "images", "audio", "videos", "documents", "screenshots", "citations", GENERATED_SUBDIR}
  96. def _in_excluded_root_dir(rel_path: Path) -> bool:
  97. """True when *rel_path* is inside a regenerable tree at a profile-home root."""
  98. root_entry = profile_root_entry(rel_path.parts)
  99. if root_entry in _EXCLUDED_ROOT_DIRS or root_entry in _EXCLUDED_BACKUP_ROOT_DIRS:
  100. return True
  101. parts = rel_path.parts
  102. if len(parts) >= 3 and parts[0] == "profiles":
  103. parts = parts[2:]
  104. return len(parts) >= 2 and parts[0] == "cache" and parts[1] not in _KEPT_CACHE_SUBDIRS
  105. # SQLite sidecars are excluded because ``*.db`` is snapshotted via ``sqlite3.backup()``:
  106. # shipping the live WAL/SHM/journal alongside would pair a fresh snapshot with stale sidecar
  107. # state and produce a torn restore on next open. They are regenerated on first connection.
  108. _SQLITE_SIDECAR_SUFFIXES = (".db-wal", ".db-shm", ".db-journal")
  109. _EXCLUDED_SUFFIXES = (".pyc", ".pyo", *_SQLITE_SIDECAR_SUFFIXES)
  110. # File names to skip (runtime state that's meaningless on another machine)
  111. _EXCLUDED_NAMES = {".backup.lock", "gateway.pid", "cron.pid"}
  112. # The desktop updater's pre-flight drops ``state.db.pre-update-emergency-<ts>.bak`` at the root
  113. # — a backup artifact like ``backups/``. Prefix-matched because the name carries a timestamp;
  114. # a plain ``.bak`` suffix rule would drop user files.
  115. # Retired-WAL capture dirs (``<name>.retired-wal-<ts>-<pid>/``) are excluded whole: a
  116. # ``sqlite3.backup()`` snapshot of the live db paired with the captured ``-wal`` is exactly the
  117. # torn-restore hazard the sidecar exclusion below exists to prevent, and the capture is an
  118. # operator-recovery artifact that must move as a unit (manifest + image + WAL), never partially.
  119. _EXCLUDED_PREFIXES = (
  120. "state.db.pre-update-emergency-",
  121. f"state.db{RETIRED_GENERATION_DIR_SUFFIX}",
  122. )
  123. # Files ``hermes import`` must never overwrite, matched by basename so root and named profiles are
  124. # both covered. They hold runtime state namespaced to the SOURCE machine: ``gateway_state.json``
  125. # drives the container-boot reconciler (a foreign value leaves the gateway stuck "starting" and
  126. # disconnected from the Nous portal); PID/lock/registry files reference source PIDs. Mirrors
  127. # ``container_boot._STALE_RUNTIME_FILES``; import filters too because older backups predate the
  128. # backup-side exclusions.
  129. _IMPORT_SKIP_NAMES = {"gateway_state.json", "gateway.pid", "cron.pid", "gateway.lock", "processes.json"}
  130. try: # zipfile already imports lzma (free); it is absent only from Pythons built without liblzma
  131. import lzma
  132. _LZMA_ERRORS: tuple[type[BaseException], ...] = (lzma.LZMAError,)
  133. except ImportError: # pragma: no cover
  134. _LZMA_ERRORS = ()
  135. # What reading a member's data raises when the archive itself is bad (a bzip2 bad stream and a
  136. # media read error are OSError, caught alongside): bad deflate stream, bad CRC, truncated stream.
  137. _ZIP_MEMBER_READ_ERRORS: tuple[type[BaseException], ...] = (
  138. zipfile.BadZipFile, zlib.error, EOFError, *_LZMA_ERRORS)
  139. # zipfile.open() drops Unix mode bits on extract; restore tightens these to 0600.
  140. # vault.key / vault.json.enc: the local credential vault (agent/vault_store.py)
  141. # IS included in backups (user-entered secrets, not regenerable — unlike the
  142. # excluded browser-profile/ snapshot) but must come back owner-only.
  143. _SECRET_FILE_NAMES = {".env", "auth.json", "state.db", "vault.key", "vault.json.enc"}
  144. # Reserved archive subtree for memory-provider state OUTSIDE HERMES_HOME (e.g. ~/.honcho, via
  145. # MemoryProvider.backup_paths()), stored and restored relative to the user's home; paths not
  146. # under home are skipped.
  147. _EXTERNAL_PREFIX = "_external/"
  148. class BackupInProgressError(RuntimeError):
  149. """Raised when another process already owns the Hermes backup slot."""
  150. class _SQLiteSnapshotError(RuntimeError):
  151. pass
  152. @contextmanager
  153. def _backup_operation_lock(hermes_home: Path, timeout_seconds: float = 0.25):
  154. """Acquire one cross-process backup slot for full and quick snapshots."""
  155. lock_path = hermes_home / ".backup.lock"
  156. lock_path.parent.mkdir(parents=True, exist_ok=True)
  157. handle = lock_path.open("a+b")
  158. acquired = False
  159. deadline = time.monotonic() + max(0.0, timeout_seconds)
  160. try:
  161. if os.name == "nt":
  162. import msvcrt
  163. if lock_path.stat().st_size == 0:
  164. handle.write(b" ")
  165. handle.flush()
  166. def _lock_op(flag: int) -> None:
  167. handle.seek(0)
  168. msvcrt.locking(handle.fileno(), flag, 1)
  169. lock_flag, unlock_flag = msvcrt.LK_NBLCK, msvcrt.LK_UNLCK
  170. else:
  171. import fcntl
  172. def _lock_op(flag: int) -> None:
  173. fcntl.flock(handle.fileno(), flag)
  174. lock_flag, unlock_flag = fcntl.LOCK_EX | fcntl.LOCK_NB, fcntl.LOCK_UN
  175. while not acquired:
  176. try:
  177. _lock_op(lock_flag)
  178. acquired = True
  179. except OSError:
  180. if time.monotonic() >= deadline:
  181. raise BackupInProgressError("another Hermes backup is already running")
  182. time.sleep(0.05)
  183. yield
  184. finally:
  185. if acquired:
  186. with suppress(OSError):
  187. _lock_op(unlock_flag)
  188. handle.close()
  189. @contextmanager
  190. def _atomic_output_path(final_path: Path, publish_path: Optional[Callable[[], Optional[Path]]] = None):
  191. """Yield a hidden sibling path and publish it only after a clean close.
  192. ``publish_path`` picks the destination at publish time (default ``final_path``) so a caller
  193. can divert an incomplete archive elsewhere without ever touching ``final_path``; returning
  194. ``None`` discards the partial instead of publishing it.
  195. """
  196. partial_path = final_path.with_name(f".{final_path.name}.{os.getpid()}-{threading.get_ident()}.partial")
  197. partial_path.unlink(missing_ok=True)
  198. try:
  199. yield partial_path
  200. destination = publish_path() if publish_path else final_path
  201. if destination is None:
  202. partial_path.unlink(missing_ok=True)
  203. else:
  204. os.replace(partial_path, destination)
  205. except BaseException:
  206. partial_path.unlink(missing_ok=True)
  207. raise
  208. def _collect_memory_provider_external_paths() -> list[Path]:
  209. """Existing paths the active memory provider declares via ``backup_paths()``; ``[]`` on any
  210. provider failure (backup must never fail because of a flaky plugin)."""
  211. try:
  212. from plugins.memory import _get_active_memory_provider, load_memory_provider
  213. active = _get_active_memory_provider()
  214. provider = load_memory_provider(active) if active else None
  215. except Exception:
  216. return []
  217. if provider is None:
  218. return []
  219. try:
  220. declared = provider.backup_paths() or []
  221. except Exception as exc:
  222. logger.warning("backup_paths() failed for memory provider %r: %s", active, exc)
  223. return []
  224. out: dict[Path, Path] = {} # resolved -> first declared spelling
  225. for raw in declared:
  226. try:
  227. p = Path(raw).expanduser()
  228. except Exception:
  229. continue
  230. if not p.exists():
  231. continue
  232. try:
  233. resolved = p.resolve()
  234. except (OSError, ValueError):
  235. continue
  236. out.setdefault(resolved, p)
  237. return list(out.values())
  238. def _iter_external_files(base: Path) -> list[Path]:
  239. """Regular files under *base* (a file or a directory), skipping symlinks, caches, and pyc."""
  240. if base.is_file() and not base.is_symlink():
  241. return [base]
  242. if not base.is_dir():
  243. return []
  244. files: list[Path] = []
  245. for dirpath, dirnames, filenames in os.walk(base, followlinks=False):
  246. dirnames[:] = [d for d in dirnames if d not in _EXCLUDED_DIRS]
  247. files.extend(fp for fp in (Path(dirpath) / f for f in filenames)
  248. if not (_is_non_regular_path(fp) or fp.name in _EXCLUDED_NAMES
  249. or fp.name.endswith(_EXCLUDED_SUFFIXES)))
  250. return files
  251. def _is_non_regular_path(path: Path) -> bool:
  252. """True for symlinks, sockets, devices, and other non-regular filesystem entries.
  253. A failed ``lstat`` is not treated as an exclusion: the archive writer must see the path and
  254. report the read failure instead of silently claiming a complete backup.
  255. """
  256. try:
  257. return not stat.S_ISREG(path.lstat().st_mode)
  258. except OSError:
  259. return False
  260. def _is_link_path(path: Path) -> bool:
  261. """True for symlinks and Windows junctions/reparse points — the only
  262. directory entries a strict walk must never descend (os.walk already
  263. refuses POSIX dir symlinks; this also covers junctions, which it follows)."""
  264. try:
  265. info = path.lstat()
  266. except OSError:
  267. return False
  268. if stat.S_ISLNK(info.st_mode):
  269. return True
  270. return os.name == "nt" and bool(getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT)
  271. def _should_exclude(rel_path: Path) -> bool:
  272. """Return True if *rel_path* (relative to hermes root) should be skipped."""
  273. parts = rel_path.parts
  274. if _in_excluded_root_dir(rel_path):
  275. return True
  276. # ``hermes-agent`` only matches at the root level; nested same-named dirs are preserved.
  277. if any(p in _EXCLUDED_DIRS and (p != "hermes-agent" or p == parts[0]) for p in parts):
  278. return True
  279. name = rel_path.name
  280. return name in _EXCLUDED_NAMES or name.startswith(_EXCLUDED_PREFIXES) or name.endswith(_EXCLUDED_SUFFIXES)
  281. def _iter_backup_files(hermes_root: Path, out_path: Path, skipped_dirs: Optional[set] = None):
  282. """Yield ``(abs_path, rel_path)`` for every file a full backup should hold.
  283. The one owner of the walk policy (directory pruning so os.walk never descends a multi-GB
  284. excluded tree, the root-only ``hermes-agent`` carve-out, root runtime trees, per-file rules),
  285. shared by ``hermes backup`` and the pre-update path so they can never drift.
  286. """
  287. for dirpath, dirnames, filenames in os.walk(hermes_root, followlinks=False):
  288. rel_dir = Path(dirpath).relative_to(hermes_root)
  289. is_root = rel_dir == Path(".")
  290. kept = [
  291. d for d in dirnames
  292. if (d not in _EXCLUDED_DIRS or (d == "hermes-agent" and not is_root))
  293. and not _in_excluded_root_dir(rel_dir / d)]
  294. if skipped_dirs is not None:
  295. skipped_dirs.update(str(rel_dir / d) for d in set(dirnames) - set(kept))
  296. # No walk may follow a junction.
  297. dirnames[:] = [name for name in kept if not _is_link_path(Path(dirpath) / name)]
  298. for fname in filenames:
  299. rel = rel_dir / fname
  300. fpath = hermes_root / rel
  301. # zipfile.write() follows file symlinks, so skip links before any archive write can
  302. # copy data from outside HERMES_HOME; never archive the output zip into itself.
  303. if _should_exclude(rel):
  304. continue
  305. if _is_non_regular_path(fpath):
  306. continue
  307. with suppress(OSError, ValueError):
  308. if fpath.resolve() == out_path.resolve():
  309. continue
  310. yield fpath, rel
  311. # --- SQLite safe copy ---
  312. def _query_ro_sqlite(path: Path, fn):
  313. """Run ``fn(conn)`` on a read-only connection to *path*; return ``(value, None)`` or ``(None, exc)``."""
  314. conn = None
  315. try:
  316. conn = sqlite3.connect(read_only_db_uri(path), uri=True, timeout=1.0)
  317. return fn(conn), None
  318. except Exception as exc:
  319. return None, exc
  320. finally:
  321. _close_quietly(conn)
  322. def is_zeroed_sqlite_file(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
  323. """True when *path* looks like the #68474 zeroed-state.db signature.
  324. Only regular files qualify: probing a FIFO/device/socket could block indefinitely.
  325. Signature: no ``SQLite format 3`` header and no data — either empty (size 0, the total-loss case,
  326. #97568) or first *probe_bytes* all NUL. Used at SessionDB open and for snapshot diagnostics so a silent
  327. all-zero file becomes a guided recovery instead of a generic failure.
  328. """
  329. try:
  330. if not path.is_file():
  331. return False
  332. except OSError:
  333. return False
  334. from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
  335. if not force and has_live_connection(path):
  336. return False
  337. head = read_header_bytes_preopen(path, length=max(16, probe_bytes), force=force)
  338. # Empty or all-NUL header => zeroed; a real header (or unreadable) => not.
  339. return head is not None and not head.startswith(b"SQLite format 3") and not any(head)
  340. # --- SQLite integrity verification ---
  341. _SQLITE_HEADER = b"SQLite format 3\0"
  342. # Above this size ``PRAGMA integrity_check`` (walks every b-tree page — minutes of pegged CPU on a
  343. # 30 GB state.db, reading as a hung ``hermes update``) is replaced by the O(1) header+schema probe.
  344. # Default ceiling above which ``PRAGMA integrity_check`` is skipped in favour of the (O(1)) header +
  345. # structural probe. Sessions databases in the tens of GB are normal for heavy users, so the size-unbounded
  346. # check is never an acceptable default on the update path. See #70553.
  347. DEFAULT_INTEGRITY_CHECK_MAX_BYTES = 2 << 30 # 2 GiB
  348. def verify_sqlite_integrity(
  349. path: Path, *, check_header: bool = True, run_pragma: bool = True,
  350. max_bytes: int = DEFAULT_INTEGRITY_CHECK_MAX_BYTES) -> dict:
  351. """Verify a SQLite database: existence + minimum size, header magic, then a read-only
  352. ``PRAGMA integrity_check`` (or a cheap structural probe above ``max_bytes``)."""
  353. def _done(message: str, valid: bool = False, size: Optional[int] = None) -> dict:
  354. return {"valid": valid, "message": message, "size": size}
  355. try:
  356. st = path.stat()
  357. except FileNotFoundError:
  358. return _done(f"not found: {path}")
  359. except OSError as exc:
  360. return _done(f"cannot stat: {exc}")
  361. size = st.st_size
  362. if size < 100: # SQLite minimum viable size (header + 1 page)
  363. return _done(f"too small ({size} bytes) to be a valid SQLite database", size=size)
  364. if check_header:
  365. # Refused when a live connection exists (close() would cancel this process's POSIX locks
  366. # — see sqlite_safe_read); verification targets offline snapshots/backup artifacts anyway.
  367. from hermes_cli.sqlite_safe_read import read_header_bytes_preopen
  368. head = read_header_bytes_preopen(path, length=len(_SQLITE_HEADER))
  369. if head is None:
  370. return _done("cannot read header", size=size)
  371. if head != _SQLITE_HEADER:
  372. return _done(f"missing SQLite header magic (got {head[:16].hex()!r})", size=size)
  373. if max_bytes > 0 and size > max_bytes:
  374. # O(1) probe: the header check caught the zeroed signature; reading sqlite_master + page
  375. # geometry catches malformed-schema and truncated-header-page classes without a data walk.
  376. _, exc = _query_ro_sqlite(path, lambda c: (
  377. c.execute("PRAGMA schema_version").fetchone(),
  378. c.execute("SELECT count(*) FROM sqlite_master").fetchone()))
  379. if exc is not None:
  380. kind = "failed" if isinstance(exc, sqlite3.DatabaseError) else "error"
  381. return _done(f"schema probe {kind}: {exc}", size=size)
  382. return _done(
  383. f"size {size:,} bytes exceeds max_bytes {max_bytes:,}; "
  384. "skipped PRAGMA integrity_check (header + schema probe passed)",
  385. valid=True, size=size)
  386. if run_pragma:
  387. rows, exc = _query_ro_sqlite(
  388. path, lambda c: [str(r[0]) for r in c.execute("PRAGMA integrity_check")])
  389. if exc is not None:
  390. kind = "cannot open database" if isinstance(exc, sqlite3.DatabaseError) else "integrity check error"
  391. return _done(f"{kind}: {exc}", size=size)
  392. if rows == ["ok"]:
  393. return _done("integrity check passed", valid=True, size=size)
  394. return _done(f"integrity check failed: {'; '.join(rows[:5])}", size=size)
  395. return _done("header check passed", valid=True, size=size)
  396. def _discard_failed_zip_members(zf: zipfile.ZipFile, filelist_len: int) -> None:
  397. """Drop the member(s) created by a failed write, both from the central directory and the file.
  398. ZipFile.write finalizes its destination member while unwinding a source-read
  399. failure, so the partial bytes can otherwise become a CRC-valid archive member.
  400. This runs immediately after that failed write, so the dropped bytes are the tail
  401. of the file: truncate at the first dropped local header and rewind start_dir so
  402. later members overwrite it. Leaving the bytes (with a valid local header) would
  403. still expose a ghost member to streaming readers. Rebuilding NameToInfo from the
  404. surviving file list also restores the previous entry when a duplicate name failed.
  405. """
  406. if len(zf.filelist) <= filelist_len:
  407. return
  408. offset = zf.filelist[filelist_len].header_offset
  409. zf.fp.seek(offset)
  410. zf.fp.truncate()
  411. zf.start_dir = offset
  412. del zf.filelist[filelist_len:]
  413. zf.NameToInfo.clear()
  414. zf.NameToInfo.update((info.filename, info) for info in zf.filelist)
  415. def _write_zip_file(zf: zipfile.ZipFile, path: Path, arcname: str) -> None:
  416. """Write one member while keeping a failed partial write out of the central directory."""
  417. filelist_len = len(zf.filelist)
  418. try:
  419. zf.write(path, arcname=arcname)
  420. except Exception:
  421. _discard_failed_zip_members(zf, filelist_len)
  422. raise
  423. def _zip_sqlite_snapshot(zf: zipfile.ZipFile, abs_path: Path, rel_path: Path, out_path: Path) -> Optional[int]:
  424. """Add a WAL-safe snapshot of *abs_path* to *zf*; return its byte size, or None on failure.
  425. Staged beside the output zip: /tmp may be a small tmpfs that cannot hold large databases.
  426. """
  427. with tempfile.NamedTemporaryFile(suffix=".db", delete=False, dir=str(out_path.parent)) as tmp:
  428. tmp_db = Path(tmp.name)
  429. try:
  430. if not _safe_copy_db(abs_path, tmp_db):
  431. return None
  432. _write_zip_file(zf, tmp_db, str(rel_path))
  433. return tmp_db.stat().st_size
  434. finally:
  435. tmp_db.unlink(missing_ok=True)
  436. def _write_zip_entries(
  437. zf: zipfile.ZipFile, files_to_add: list[tuple[Path, Path]], out_path: Path,
  438. *, on_db_failure, on_error, on_progress, track_bytes: bool) -> int:
  439. """Add every ``(abs_path, rel_path)`` to *zf*, WAL-safe for ``*.db``; return bytes archived.
  440. ``on_db_failure(rel_path)`` runs when a SQLite snapshot fails (may raise to abort);
  441. ``on_error(rel_path, exc)`` records a read failure; ``on_progress(i)`` fires every 500 files;
  442. ``track_bytes`` stats plain files for the size total.
  443. """
  444. total_bytes = 0
  445. for i, (abs_path, rel_path) in enumerate(files_to_add, 1):
  446. try:
  447. if abs_path.suffix == ".db":
  448. size = _zip_sqlite_snapshot(zf, abs_path, rel_path, out_path)
  449. if size is None:
  450. on_db_failure(rel_path)
  451. continue
  452. total_bytes += size
  453. else:
  454. _write_zip_file(zf, abs_path, str(rel_path))
  455. if track_bytes:
  456. total_bytes += abs_path.stat().st_size
  457. except (PermissionError, OSError, ValueError) as exc:
  458. on_error(rel_path, exc)
  459. continue
  460. if i % 500 == 0:
  461. on_progress(i)
  462. return total_bytes
  463. def _print_capped(header: str, lines: list[str], indent: str) -> None:
  464. """Print *header*, then at most 10 of *lines* (each prefixed by *indent*) and a "... and N more" tail."""
  465. print(header)
  466. for line in lines[:10]:
  467. print(f"{indent}{line}")
  468. if len(lines) > 10:
  469. print(f"{indent}... and {len(lines) - 10} more")
  470. # --- Backup ---
  471. _RUN_BACKUP_PREFIX = "hermes-backup-"
  472. def _resolve_backup_output_path(output: Optional[str]) -> Path:
  473. """Turn ``--output`` (file, directory, or None) into a ``.zip`` path whose parent exists;
  474. an unwritable path exits with a one-line error, not a traceback."""
  475. out_path = None
  476. default_name = f"{_RUN_BACKUP_PREFIX}{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
  477. try:
  478. if output:
  479. out_path = Path(output).expanduser().resolve()
  480. if out_path.is_dir():
  481. out_path = out_path / default_name
  482. else:
  483. out_path = Path.home() / default_name
  484. if out_path.suffix.lower() != ".zip":
  485. out_path = out_path.with_suffix(out_path.suffix + ".zip")
  486. out_path.parent.mkdir(parents=True, exist_ok=True)
  487. except OSError as exc:
  488. print(f"Error: cannot write backup to {output or out_path}: {exc}")
  489. raise SystemExit(1) from exc
  490. return out_path
  491. def _collect_external_entries() -> tuple[list[tuple[Path, str]], list[str]]:
  492. """``([(abs_path, arcname)], [skipped])`` for the memory provider's external state, arc-named
  493. ``_external/<home-relative>``; paths outside home are skipped (security + portability)."""
  494. home_dir = Path.home().resolve()
  495. external_to_add: list[tuple[Path, str]] = []
  496. skipped_external: list[str] = []
  497. for base in _collect_memory_provider_external_paths():
  498. try:
  499. base.resolve().relative_to(home_dir)
  500. except (ValueError, OSError):
  501. skipped_external.append(str(base))
  502. continue
  503. for fpath in _iter_external_files(base):
  504. with suppress(ValueError, OSError):
  505. rel_to_home = fpath.resolve().relative_to(home_dir)
  506. external_to_add.append((fpath, _EXTERNAL_PREFIX + rel_to_home.as_posix()))
  507. return external_to_add, skipped_external
  508. def run_backup(args) -> bool:
  509. """Create a zip backup of the Hermes home directory.
  510. True when every selected file landed in the archive (or there was nothing to back up); False
  511. when the zip was written but is incomplete — it is kept so the rest can still be restored, and
  512. the caller turns False into exit status 1 so a cron/systemd timer never publishes a "successful"
  513. archive that is missing state.db. Hard failures keep raising ``SystemExit``.
  514. """
  515. require_file_tooling("`hermes backup`")
  516. hermes_root = get_default_hermes_root()
  517. if not hermes_root.is_dir():
  518. print(f"Error: Hermes home directory not found at {hermes_root}")
  519. sys.exit(1)
  520. try:
  521. with _backup_operation_lock(hermes_root):
  522. return _run_backup_locked(args, hermes_root)
  523. except BackupInProgressError as exc:
  524. print(f"Error: {exc}")
  525. raise SystemExit(2) from exc
  526. def _run_backup_locked(args, hermes_root: Path) -> bool:
  527. """Write a full backup while the cross-process backup slot is held."""
  528. out_path = _resolve_backup_output_path(args.output)
  529. scan_started = time.monotonic()
  530. logger.info("backup phase=scan status=started")
  531. print(f"Scanning {display_hermes_home()} ...")
  532. skipped_dirs: set = set()
  533. files_to_add: list[tuple[Path, Path]] = list(_iter_backup_files(hermes_root, out_path, skipped_dirs))
  534. external_to_add, skipped_external = _collect_external_entries()
  535. if not files_to_add and not external_to_add:
  536. logger.info("backup phase=scan status=empty duration_ms=%.1f", (time.monotonic() - scan_started) * 1000)
  537. print("No files to back up.")
  538. return True
  539. file_count = len(files_to_add) + len(external_to_add)
  540. logger.info("backup phase=scan status=complete duration_ms=%.1f files=%d",
  541. (time.monotonic() - scan_started) * 1000, file_count)
  542. logger.info("backup phase=archive status=started files=%d", file_count)
  543. print(f"Backing up {file_count} files ...")
  544. errors = []
  545. t0 = time.monotonic()
  546. def _progress(i: int) -> None:
  547. print(f" {i}/{file_count} files ...")
  548. logger.info("backup phase=archive status=progress completed=%d total=%d", i, file_count)
  549. with _atomic_output_path(out_path) as archive_path, zipfile.ZipFile(
  550. archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
  551. total_bytes = _write_zip_entries(
  552. zf, files_to_add, out_path, on_progress=_progress, track_bytes=True,
  553. on_db_failure=lambda rel: errors.append(f"{rel}: SQLite safe copy failed"),
  554. on_error=lambda rel, exc: errors.append(f"{rel}: {exc}"))
  555. # External memory-provider state never includes ``.db`` files in practice, so no
  556. # SQLite snapshot is needed; _write_zip_file still drops a failed partial member.
  557. for abs_path, arcname in external_to_add:
  558. try:
  559. _write_zip_file(zf, abs_path, arcname)
  560. total_bytes += abs_path.stat().st_size
  561. except (PermissionError, OSError, ValueError) as exc:
  562. errors.append(f"{arcname}: {exc}")
  563. elapsed = time.monotonic() - t0
  564. zip_size = out_path.stat().st_size
  565. logger.info("backup phase=archive status=complete duration_ms=%.1f files=%d errors=%d bytes=%d",
  566. elapsed * 1000, file_count, len(errors), zip_size)
  567. print(f"\nBackup {'incomplete' if errors else 'complete'}: {out_path}\n"
  568. f" Files: {file_count}\n"
  569. f" Original: {_format_size(total_bytes)}\n"
  570. f" Compressed: {_format_size(zip_size)}\n"
  571. f" Time: {elapsed:.1f}s")
  572. if external_to_add:
  573. print(f"\n Included {len(external_to_add)} memory-provider file(s) stored outside {display_hermes_home()}.")
  574. if skipped_external:
  575. print(f"\n Skipped {len(skipped_external)} memory-provider path(s) outside your home directory "
  576. "(not portable):\n" + "\n".join(f" {p}" for p in sorted(skipped_external)[:10]))
  577. if skipped_dirs:
  578. print("\n Excluded directories:\n" + "\n".join(f" {d}/" for d in sorted(skipped_dirs)))
  579. if errors:
  580. _print_capped(f"\n Archive kept, but {len(errors)} file(s) could not be added:", errors, " ")
  581. else:
  582. print(f"\nRestore with: hermes import {out_path.name}")
  583. # Prune only after a complete archive: a timer hitting the same unreadable file every run must
  584. # not rotate the last good backups out in favour of incomplete ones.
  585. keep = getattr(args, "keep", 0) # 0 / absent: never prune (non-CLI callers)
  586. if keep and not errors and out_path.name.startswith(_RUN_BACKUP_PREFIX):
  587. pruned = _prune_prefixed_zips(out_path.parent, _RUN_BACKUP_PREFIX, keep, "backup")
  588. if pruned:
  589. print(f" Pruned {pruned} older {_RUN_BACKUP_PREFIX}*.zip (keeping {keep}).")
  590. return not errors
  591. # --- Import ---
  592. def _find_corrupt_members(zf: zipfile.ZipFile, members: list[str]) -> list[str]:
  593. """Return ``"<member>: <error>"`` for every member whose data does not decompress or
  594. fails its CRC, streaming each one in 1 MiB chunks so a multi-GB ``state.db`` is never
  595. held in memory.
  596. ``is_zipfile()``/``namelist()`` only read the central directory, so an archive with a
  597. rotten member passes them and the damage surfaces as ``zlib.error``/``BadZipFile`` in
  598. the middle of the restore, after earlier members already replaced the user's files
  599. (#121258). Not ``zf.testzip()``: it lets ``zlib.error`` escape and names at most the
  600. first bad member.
  601. """
  602. bad: list[str] = []
  603. for member in members:
  604. try:
  605. with zf.open(member) as src:
  606. while src.read(1 << 20): # CRC is checked when the stream hits EOF
  607. pass
  608. except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
  609. bad.append(f"{member}: {exc}")
  610. return bad
  611. def _import_skipped(rel: str) -> bool:
  612. """True for a HERMES_HOME-relative member the import deliberately does not restore: runtime
  613. state (``_IMPORT_SKIP_NAMES``), PM-local interpreter/dependency roots, or an archived SQLite
  614. WAL/SHM/journal. A ``.db`` member is page-restored into the live file, and a sidecar from a
  615. different image would replay a foreign WAL on next open (older archives may ship these)."""
  616. try:
  617. parts = tuple(normalize_archive_parts(rel))
  618. except ValueError:
  619. return False # A rejected traversal is still reported by the import itself.
  620. return (parts[-1] in _IMPORT_SKIP_NAMES or
  621. profile_root_entry(parts) in PM_RUNTIME_ROOT_DIRS or
  622. rel.endswith(_SQLITE_SIDECAR_SUFFIXES))
  623. def _import_member_rel(member: str, prefix: str) -> tuple[str, bool]:
  624. """Classify an archive member exactly as the restore does: return ``(rel, skipped)``.
  625. ``_external/`` members are home-relative and never skipped; every other member is
  626. HERMES_HOME-relative after stripping the archive ``prefix``. Shared by the integrity
  627. pre-flight and ``_import_members`` so the two cannot disagree on what gets restored."""
  628. if member.startswith(_EXTERNAL_PREFIX):
  629. return member[len(_EXTERNAL_PREFIX):], False
  630. rel = member[len(prefix):] if prefix and member.startswith(prefix) else member
  631. return rel, _import_skipped(rel)
  632. def run_import(args) -> Optional[int]:
  633. """Restore a Hermes backup; return 1 on damaged archives or incomplete restores."""
  634. require_file_tooling("`hermes import`")
  635. zip_path = Path(args.zipfile).expanduser().resolve()
  636. if not zip_path.is_file():
  637. print(f"Error: File not found: {zip_path}")
  638. sys.exit(1)
  639. if not zipfile.is_zipfile(zip_path):
  640. print(f"Error: Not a valid zip file: {zip_path}")
  641. sys.exit(1)
  642. # The home this command operates under (the "Target:" display_hermes_home() prints):
  643. # get_default_hermes_root() would map a profile home back to <root> and restore the live root.
  644. hermes_root = get_hermes_home()
  645. with zipfile.ZipFile(zip_path, "r") as zf:
  646. # Validate
  647. ok, reason = _validate_backup_zip(zf)
  648. if not ok:
  649. print(f"Error: {reason}")
  650. sys.exit(1)
  651. prefix = _detect_prefix(zf)
  652. members = [n for n in zf.namelist() if not n.endswith("/")]
  653. file_count = len(members)
  654. print(f"Backup contains {file_count} files")
  655. print(f"Target: {display_hermes_home()}")
  656. if prefix:
  657. print(f"Detected archive prefix: {prefix!r} (will be stripped)")
  658. # Check for existing installation
  659. has_config = (hermes_root / "config.yaml").exists() # config-reader: ok — profile marker for the backup scope (file tooling)
  660. has_env = (hermes_root / ".env").exists()
  661. if (has_config or has_env) and not args.force:
  662. print()
  663. print("Warning: Target directory already has Hermes configuration.")
  664. print("Importing will overwrite existing files with backup contents.")
  665. print()
  666. try:
  667. answer = input("Continue? [y/N] ").strip().lower()
  668. except (EOFError, KeyboardInterrupt):
  669. print("\nAborted.")
  670. sys.exit(1)
  671. if answer not in {"y", "yes"}:
  672. print("Aborted.")
  673. return
  674. # Refuse damaged archives before writing any user files. Runtime and PM-local
  675. # members skipped below cannot block restoration of the portable data.
  676. print("\nChecking archive integrity ...")
  677. corrupt = _find_corrupt_members(zf, [m for m in members if not _import_member_rel(m, prefix)[1]])
  678. if corrupt:
  679. _print_capped(f"Error: backup archive is damaged ({len(corrupt)} member(s) fail to "
  680. f"decompress or fail their CRC); nothing was restored:", corrupt, " ")
  681. return 1
  682. # Extract
  683. print(f"\nImporting {file_count} files ...")
  684. hermes_root.mkdir(parents=True, exist_ok=True)
  685. t0 = time.monotonic()
  686. from gateway.runtime_ownership import OwnershipConflict
  687. try:
  688. restored, restored_external, errors, skipped_runtime, db_shrunk = _import_members(
  689. zf, members, prefix, hermes_root, file_count)
  690. except OwnershipConflict as exc:
  691. print(f"\nImport refused; no files restored: {exc}")
  692. sys.exit(1)
  693. elapsed = time.monotonic() - t0
  694. # Summary
  695. print()
  696. print(f"Import {'incomplete' if errors else 'complete'}: {restored} files restored in {elapsed:.1f}s")
  697. print(f" Target: {display_hermes_home()}")
  698. if restored_external:
  699. print(
  700. f"\n Restored {restored_external} memory-provider file(s) to "
  701. f"their original location(s) outside {display_hermes_home()}."
  702. )
  703. if errors:
  704. print(f"\n Warnings ({len(errors)} files skipped):")
  705. for e in errors[:10]:
  706. print(e)
  707. if len(errors) > 10:
  708. print(f" ... and {len(errors) - 10} more")
  709. if db_shrunk:
  710. # The backup predates work that is now overwritten. Say so: the
  711. # reported incident was twelve sessions disappearing with nothing
  712. # logged anywhere (issue #100960).
  713. print("\n ⚠ Session data replaced by older backup contents:")
  714. for rel, before, after in db_shrunk:
  715. print(
  716. f" {rel}: {before[0]} session(s) / {before[1]} message(s)"
  717. f" -> {after[0]} / {after[1]}"
  718. )
  719. print(
  720. " Anything recorded after the backup was taken is not in it. "
  721. f"{_snapshot_recovery_hint()}"
  722. )
  723. if skipped_runtime:
  724. print(
  725. f"\n Preserved {len(skipped_runtime)} runtime state "
  726. f"file(s) (kept this machine's, not the backup's):"
  727. )
  728. for rel in sorted(skipped_runtime)[:10]:
  729. print(f" {rel}")
  730. if len(skipped_runtime) > 10:
  731. print(f" ... and {len(skipped_runtime) - 10} more")
  732. # Post-import: restore profile wrapper scripts
  733. profiles_dir = hermes_root / "profiles"
  734. restored_profiles = []
  735. if profiles_dir.is_dir():
  736. try:
  737. from hermes_cli.profiles import (
  738. create_wrapper_script, check_alias_collision,
  739. _is_wrapper_dir_in_path, _get_wrapper_dir,
  740. )
  741. for entry in sorted(profiles_dir.iterdir()):
  742. if not entry.is_dir():
  743. continue
  744. profile_name = entry.name
  745. # Only create wrappers for directories with config
  746. if not (entry / "config.yaml").exists() and not (entry / ".env").exists(): # config-reader: ok — profile marker for the backup scope (file tooling)
  747. continue
  748. collision = check_alias_collision(profile_name)
  749. if collision:
  750. print(f" Skipped alias '{profile_name}': {collision}")
  751. restored_profiles.append((profile_name, False))
  752. else:
  753. wrapper = create_wrapper_script(profile_name)
  754. restored_profiles.append((profile_name, wrapper is not None))
  755. if restored_profiles:
  756. created = [n for n, ok in restored_profiles if ok]
  757. skipped = [n for n, ok in restored_profiles if not ok]
  758. if created:
  759. print(f"\n Profile aliases restored: {', '.join(created)}")
  760. if skipped:
  761. print(f" Profile aliases skipped: {', '.join(skipped)}")
  762. if not _is_wrapper_dir_in_path():
  763. print(f"\n Note: {_get_wrapper_dir()} is not in your PATH.")
  764. print(' Add to your shell config (~/.bashrc or ~/.zshrc):')
  765. print(' export PATH="$HOME/.local/bin:$PATH"')
  766. except ImportError:
  767. # hermes_cli.profiles might not be available (fresh install)
  768. if any(profiles_dir.iterdir()):
  769. print("\n Profiles detected but aliases could not be created.")
  770. print(" Run: hermes profile list (after installing hermes)")
  771. # Guidance
  772. print()
  773. if not (hermes_root / "hermes-agent").is_dir():
  774. print("Note: The hermes-agent codebase was not included in the backup.")
  775. print(" If this is a fresh install, run: hermes update")
  776. if restored_profiles:
  777. gw_profiles = [n for n, _ in restored_profiles]
  778. print("\nTo re-enable gateway services for profiles:")
  779. for pname in gw_profiles:
  780. print(f" hermes -p {pname} gateway install")
  781. _revive_gateway_after_import(hermes_root)
  782. if errors:
  783. print(f"Import incomplete: {len(errors)} file(s) were not restored (see Warnings above). "
  784. "Fix the cause and re-run the import.")
  785. return 1
  786. print("Done. Your Hermes configuration has been restored.")
  787. def _import_members(
  788. zf: zipfile.ZipFile, members: list[str], prefix: str, hermes_root: Path, file_count: int
  789. ) -> tuple[int, int, list[str], list[str], list[tuple[str, tuple[int, int], tuple[int, int]]]]:
  790. """Reserve all affected profiles before publishing even the first config file."""
  791. from gateway.runtime_ownership import exclusive_maintenance
  792. root_res = hermes_root.resolve()
  793. homes = {hermes_root}
  794. for member in members:
  795. if member.startswith(_EXTERNAL_PREFIX):
  796. continue # restored under $HOME, outside every runtime authority
  797. rel = member[len(prefix):] if prefix and member.startswith(prefix) else member
  798. parts = Path(rel).parts
  799. if len(parts) >= 3 and parts[0] == "profiles":
  800. home = hermes_root / parts[0] / parts[1]
  801. if home.resolve().is_relative_to(root_res):
  802. homes.add(home)
  803. target = hermes_root / rel
  804. if target.suffix == ".db" and target.resolve().is_relative_to(root_res):
  805. homes.update([target.absolute().parent, target.resolve().parent])
  806. with exclusive_maintenance(homes):
  807. return _import_members_exclusive(zf, members, prefix, hermes_root, file_count)
  808. def _import_external_member(
  809. zf: zipfile.ZipFile, member: str, home_dir: Path, new_file_mode: int, errors: list[str]
  810. ) -> bool:
  811. """Publish one ``_external/`` member under the user's home; False (with ``errors`` appended)
  812. when it is a traversal or cannot be written."""
  813. target = home_dir / member[len(_EXTERNAL_PREFIX):]
  814. # Security: the resolved target must stay under the home dir.
  815. try:
  816. target.resolve().relative_to(home_dir)
  817. except ValueError:
  818. errors.append(f" {member}: path traversal blocked")
  819. return False
  820. try:
  821. target.parent.mkdir(parents=True, exist_ok=True)
  822. _extract_member_atomically(zf, member, target, new_file_mode)
  823. # External provider configs commonly hold credentials.
  824. if target.suffix in {".json", ".env", ".conf"} or target.name in _SECRET_FILE_NAMES:
  825. try:
  826. os.chmod(target, 0o600)
  827. except OSError:
  828. pass
  829. return True
  830. except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
  831. errors.append(f" {member}: {exc}")
  832. return False
  833. def _import_members_exclusive(
  834. zf: zipfile.ZipFile, members: list[str], prefix: str, hermes_root: Path, file_count: int
  835. ) -> tuple[int, int, list[str], list[str], list[tuple[str, tuple[int, int], tuple[int, int]]]]:
  836. """Publish every member; return ``(restored, restored_external, errors, skipped_runtime, db_shrunk)``.
  837. ``db_shrunk`` holds ``(rel, live_counts, imported_counts)`` for every session database the
  838. import replaced with one holding fewer rows — allowed, but never silently (#100960).
  839. """
  840. errors = []
  841. restored = 0
  842. restored_external = 0
  843. skipped_runtime: list[str] = []
  844. db_shrunk: list[tuple[str, tuple[int, int], tuple[int, int]]] = []
  845. home_dir = Path.home().resolve()
  846. # Resolved once: every member is published via a temp file, and mkstemp
  847. # would otherwise create newly restored files as 0600.
  848. new_file_mode = _default_new_file_mode()
  849. for member in members:
  850. # External memory-provider state captured under the reserved
  851. # ``_external/`` arc prefix restores to its original home-relative
  852. # location (e.g. ~/.honcho/config.json), NOT under HERMES_HOME.
  853. if member.startswith(_EXTERNAL_PREFIX):
  854. if not member[len(_EXTERNAL_PREFIX):]:
  855. continue
  856. if _import_external_member(zf, member, home_dir, new_file_mode, errors):
  857. restored += 1
  858. restored_external += 1
  859. if restored % 500 == 0:
  860. print(f" {restored}/{file_count} files ...")
  861. continue
  862. # Strip prefix if detected
  863. if prefix and member.startswith(prefix):
  864. rel = member[len(prefix):]
  865. else:
  866. rel = member
  867. if not rel:
  868. continue
  869. try:
  870. parts = tuple(normalize_archive_parts(rel))
  871. except ValueError:
  872. errors.append(f" {rel}: path traversal blocked")
  873. continue
  874. # Never overwrite volatile gateway/process runtime state. These are
  875. # namespaced to the machine/container the backup was taken on;
  876. # clobbering them (especially gateway_state.json) breaks the gateway
  877. # reconciler on the target and disconnects hosted instances from the
  878. # Nous portal. Matched by basename so both the root profile and
  879. # named profiles (profiles/<name>/gateway_state.json) are covered.
  880. if parts[-1] in _IMPORT_SKIP_NAMES:
  881. skipped_runtime.append(rel)
  882. continue
  883. # Older archives may contain PM selections pointing at another machine.
  884. # Match their home-root paths; a plugin's own facts.json is user data.
  885. if profile_root_entry(parts) in PM_RUNTIME_ROOT_DIRS:
  886. skipped_runtime.append(rel)
  887. continue
  888. # A ``.db`` member is page-restored into the live file below; a
  889. # WAL/SHM/journal member from the archive describes a different
  890. # database image, and installing it beside the restored file (over
  891. # a live sidecar, via os.replace) would replay a foreign WAL on
  892. # the next open. Current backups never ship these
  893. # (_EXCLUDED_SUFFIXES); older or hand-built archives might.
  894. if rel.endswith(_SQLITE_SIDECAR_SUFFIXES):
  895. skipped_runtime.append(rel)
  896. continue
  897. target = hermes_root.joinpath(*parts)
  898. # Security: reject absolute paths and traversals
  899. try:
  900. target.resolve().relative_to(hermes_root.resolve())
  901. except ValueError:
  902. errors.append(f" {rel}: path traversal blocked")
  903. continue
  904. try:
  905. target.parent.mkdir(parents=True, exist_ok=True)
  906. if target.suffix == ".db":
  907. # Count before the write: afterwards the rows this import
  908. # drops are gone and there is nothing left to compare.
  909. before = _count_session_rows(target)
  910. _import_db_member(zf, member, target, new_file_mode)
  911. after = _count_session_rows(target)
  912. if before and after and after[1] < before[1]:
  913. db_shrunk.append((rel, before, after))
  914. else:
  915. _extract_member_atomically(zf, member, target, new_file_mode)
  916. if target.name in _SECRET_FILE_NAMES:
  917. os.chmod(target, 0o600)
  918. restored += 1
  919. except (OSError, *_ZIP_MEMBER_READ_ERRORS) as exc:
  920. errors.append(f" {rel}: {exc}")
  921. if restored % 500 == 0:
  922. print(f" {restored}/{file_count} files ...")
  923. return restored, restored_external, errors, skipped_runtime, db_shrunk
  924. def _revive_gateway_after_import(hermes_root: Path) -> None:
  925. """Start an installed gateway service after a restore, best-effort and prompt-free.
  926. Bot tokens and cron jobs are inert without a gateway (a platform-less gateway is supported, so
  927. this is safe for any backup); failures print a manual fallback, never fail the import. Only
  928. revived when the restore landed in the default home or no other install exists: a sandbox or
  929. profile restore must not install a second gateway on the default service name.
  930. """
  931. native_default = _get_platform_default_hermes_home()
  932. if hermes_root != native_default and any(
  933. (native_default / marker).exists() for marker in ("config.yaml", ".env", "state.db")):
  934. print("\nRestored into a non-default home; leaving the gateway service alone to avoid clashing "
  935. f"with the install at {native_default}.\n"
  936. "To start a gateway for this home, run: hermes gateway run")
  937. return
  938. try:
  939. from hermes_cli.gateway import _is_service_running
  940. from hermes_cli.gateway_setup_service import ensure_gateway_service
  941. if not _is_service_running():
  942. print()
  943. ensure_gateway_service(context="import")
  944. except Exception:
  945. # Import boundary: the restore already succeeded; any service failure prints the
  946. # manual fallback instead of failing the import.
  947. logger.debug("gateway revive after import failed", exc_info=True)
  948. print("\nStart the gateway to activate cron jobs and messaging:\n hermes gateway run")
  949. # --- Quick state snapshots (used by /snapshot slash command and hermes backup --quick) ---
  950. # Critical state files (relative to HERMES_HOME) for quick snapshots; everything else is
  951. # regeneratable or managed separately (skills, repo, sessions/). Entries may be files OR
  952. # directories (recursive); missing entries are skipped. Pairing data lives in platform JSON blobs
  953. # outside state.db, so it is listed explicitly — ``hermes update`` snapshots this set (#15733).
  954. _QUICK_STATE_FILES = (
  955. "state.db", "config.yaml", ".env", "auth.json", "cron/jobs.json", "cron/executions.db",
  956. "gateway_state.json", "channel_directory.json", "channel_aliases.json", "processes.json",
  957. "gateway/discord_message_recovery.db", # Discord reconnect replay ledger
  958. # Per-profile user stores, destroyed if the update flow replaces the file and the post-update
  959. # schema-init re-creates an empty one (#52889). Skipped when outside HERMES_HOME.
  960. "projects.db", # per-profile project store
  961. "response_store.db", # gateway conversation history / tool payloads
  962. "memory_store.db", # holographic memory facts/entities
  963. "verification_evidence.db", # agent verification audit trail
  964. "kanban.db", # default board (back-compat <root>/kanban.db)
  965. "kanban/boards", # non-default boards (workspaces/ + attachments/ skipped as regenerable)
  966. # Pairing stores (generic + per-platform JSONs outside state.db)
  967. "pairing", # legacy location (gateway/pairing.py)
  968. "platforms/pairing", # new location (gateway/pairing.py)
  969. "feishu_comment_pairing.json", # Feishu comment subscription pairings
  970. )
  971. _QUICK_DEFAULT_KEEP = 20
  972. def _quick_snapshot_root(hermes_home: Optional[Path] = None) -> Path:
  973. home = hermes_home or get_hermes_home()
  974. return home / _QUICK_SNAPSHOTS_DIR
  975. def _newest_first(root: Path, keep_entry) -> list[Path]:
  976. """Entries of *root* passing ``keep_entry``, newest (by name) first; ``[]`` if *root* is missing."""
  977. if not root.exists():
  978. return []
  979. return sorted(filter(keep_entry, root.iterdir()), key=lambda p: p.name, reverse=True)
  980. # Kept in sync with ``_QUICK_STATE_FILES`` and ``cron/jobs.py``'s ``JOBS_FILE``.
  981. _CRON_JOBS_REL = "cron/jobs.json"
  982. def _prune_oldest(newest_first: list[Path], keep: int, remove, what: str) -> int:
  983. """``remove(path)`` every entry past the first *keep*; return how many succeeded."""
  984. deleted = 0
  985. for p in newest_first[keep:]:
  986. try:
  987. remove(p)
  988. deleted += 1
  989. except OSError as exc:
  990. logger.warning("Failed to prune %s %s: %s", what, p.name, exc)
  991. return deleted
  992. # --- Pre-update / pre-migration auto-backups ---
  993. _PRE_UPDATE_BACKUPS_DIR = "backups"
  994. _PRE_UPDATE_PREFIX = "pre-update-"
  995. _PRE_UPDATE_DEFAULT_KEEP = 5
  996. _PRE_MIGRATION_PREFIX = "pre-migration-"
  997. _PRE_MIGRATION_DEFAULT_KEEP = 5
  998. _INCOMPLETE_ZIP_SUFFIX = ".incomplete.zip"
  999. def _prune_prefixed_zips(backup_dir: Path, prefix: str, keep: int, what: str) -> int:
  1000. """Remove oldest ``<prefix>*.zip`` in *backup_dir* beyond *keep*; return count deleted.
  1001. Only prefix-matched files are touched, so hand-made zips or other backup kinds survive.
  1002. """
  1003. backups = _newest_first(backup_dir, lambda p: p.is_file() and p.name.startswith(prefix)
  1004. and p.suffix.lower() == ".zip"
  1005. and not p.name.lower().endswith(_INCOMPLETE_ZIP_SUFFIX))
  1006. return _prune_oldest(backups, keep, Path.unlink, what)
  1007. def _prune_incomplete_zips(backup_dir: Path, prefix: str, what: str) -> int:
  1008. """Keep only the newest ``<prefix>*.incomplete.zip`` salvage archive; return count deleted.
  1009. Salvage archives never count toward normal retention, so repeated failing runs would
  1010. otherwise pile up without bound.
  1011. """
  1012. salvage = _newest_first(backup_dir, lambda p: p.is_file() and p.name.startswith(prefix)
  1013. and p.name.lower().endswith(_INCOMPLETE_ZIP_SUFFIX))
  1014. return _prune_oldest(salvage, 1, Path.unlink, f"incomplete {what}")
  1015. def _create_prefixed_full_backup(
  1016. hermes_home: Optional[Path], prefix: str, keep: int, what: str, prune_what: str) -> Optional[Path]:
  1017. """Write ``<HERMES_HOME>/backups/<prefix><timestamp>.zip`` and prune older same-prefix zips.
  1018. Returns the path, or ``None`` if nothing to back up, the write failed, or the archive is
  1019. incomplete (kept as ``<prefix><timestamp>.incomplete.zip``, excluded from retention).
  1020. Never raises."""
  1021. hermes_root = hermes_home or get_default_hermes_root()
  1022. if not hermes_root.is_dir():
  1023. return None
  1024. backup_dir = hermes_root / _PRE_UPDATE_BACKUPS_DIR
  1025. try:
  1026. backup_dir.mkdir(parents=True, exist_ok=True)
  1027. except OSError as exc:
  1028. logger.warning("Could not create %s backup dir %s: %s", what, backup_dir, exc)
  1029. return None
  1030. out_path = backup_dir / f"{prefix}{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
  1031. if _write_full_zip_backup(out_path, hermes_root) is None:
  1032. # Incomplete runs publish straight to ``*.incomplete.zip`` (all-failed runs are discarded);
  1033. # cap those salvages at one without letting them rotate complete backups out.
  1034. _prune_incomplete_zips(backup_dir, prefix, prune_what)
  1035. return None
  1036. _prune_prefixed_zips(backup_dir, prefix, keep, prune_what)
  1037. return out_path
  1038. def create_pre_update_backup(
  1039. hermes_home: Optional[Path] = None, keep: int = _PRE_UPDATE_DEFAULT_KEEP) -> Optional[Path]:
  1040. """Full zip backup to ``backups/pre-update-<timestamp>.zip``, auto-pruned; ``None`` if nothing
  1041. was found, the backup failed, or it was incomplete (salvage kept as ``*.incomplete.zip``).
  1042. Never raises — ``hermes update`` continues anyway."""
  1043. return _create_prefixed_full_backup(hermes_home, _PRE_UPDATE_PREFIX, max(keep, 1), "pre-update", "backup")
  1044. def create_pre_migration_backup(
  1045. hermes_home: Optional[Path] = None, keep: int = _PRE_MIGRATION_DEFAULT_KEEP) -> Optional[Path]:
  1046. """Full zip backup to ``backups/pre-migration-<timestamp>.zip`` before a plugin's agent import
  1047. (``hermes import-agent openclaw``), so its name and signature are API; same dir as update backups
  1048. so ``hermes import`` finds it. ``None`` if nothing was found, the write failed, or it was
  1049. incomplete (salvage kept as ``*.incomplete.zip``). Never raises."""
  1050. return _create_prefixed_full_backup(
  1051. hermes_home, _PRE_MIGRATION_PREFIX, max(keep, 0), "pre-migration", "pre-migration backup")
  1052. # ---------------------------------------------------------------------------
  1053. # Quick state snapshots (used by /snapshot slash command and hermes backup --quick)
  1054. # ---------------------------------------------------------------------------
  1055. def create_quick_snapshot(
  1056. label: Optional[str] = None,
  1057. hermes_home: Optional[Path] = None,
  1058. keep: Optional[int] = None,
  1059. max_file_size: Optional[int] = None,
  1060. ) -> Optional[str]:
  1061. """Create one atomic quick snapshot while holding the shared backup slot."""
  1062. home = hermes_home or get_hermes_home()
  1063. with _backup_operation_lock(home):
  1064. return _create_quick_snapshot_locked(
  1065. label=label,
  1066. hermes_home=home,
  1067. keep=keep,
  1068. max_file_size=max_file_size,
  1069. )
  1070. def _create_quick_snapshot_locked(
  1071. label: Optional[str] = None,
  1072. hermes_home: Optional[Path] = None,
  1073. keep: Optional[int] = None,
  1074. max_file_size: Optional[int] = None,
  1075. ) -> Optional[str]:
  1076. """Create a quick state snapshot of critical files.
  1077. Copies STATE_FILES to a timestamped directory under state-snapshots/.
  1078. Auto-prunes old snapshots beyond the keep limit.
  1079. Args:
  1080. max_file_size: When set, individual files larger than this many bytes
  1081. are skipped (with a printed warning) instead of copied. Used by
  1082. the pre-update safety snapshot so a multi-GB ``state.db`` can
  1083. never stall ``hermes update`` or silently eat disk — the small
  1084. pairing/cron/config files the snapshot exists to protect are
  1085. always captured. ``None`` (default) copies everything, which
  1086. preserves manual ``/snapshot`` and ``hermes backup --quick``
  1087. behavior.
  1088. Returns:
  1089. Snapshot ID (timestamp-based), or None if no files found.
  1090. """
  1091. home = hermes_home or get_hermes_home()
  1092. root = _quick_snapshot_root(home)
  1093. def _too_large(path: Path, rel_name: str) -> bool:
  1094. """True (and warn) when ``path`` exceeds the max_file_size cap."""
  1095. if max_file_size is None:
  1096. return False
  1097. try:
  1098. size = path.stat().st_size
  1099. except OSError:
  1100. return False
  1101. if size <= max_file_size:
  1102. return False
  1103. print(
  1104. f" ⚠ Snapshot: skipping {rel_name} "
  1105. f"({_format_size(size)} exceeds {_format_size(max_file_size)} limit)"
  1106. )
  1107. logger.warning(
  1108. "Quick snapshot skipped %s: %d bytes exceeds %d byte limit",
  1109. rel_name,
  1110. size,
  1111. max_file_size,
  1112. )
  1113. return True
  1114. ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
  1115. base_snap_id = f"{ts}-{label}" if label else ts
  1116. snap_id = base_snap_id
  1117. suffix = 2
  1118. while (root / snap_id).exists():
  1119. snap_id = f"{base_snap_id}-{suffix}"
  1120. suffix += 1
  1121. snap_dir = root / snap_id
  1122. staging_dir = root / f".{snap_id}.{os.getpid()}.partial"
  1123. shutil.rmtree(staging_dir, ignore_errors=True)
  1124. root.mkdir(parents=True, exist_ok=True, mode=0o700)
  1125. if os.name != "nt":
  1126. os.chmod(root, 0o700)
  1127. staging_dir.mkdir(mode=0o700, exist_ok=False)
  1128. logger.info("quick snapshot phase=copy status=started id=%s", snap_id)
  1129. manifest: dict[str, int] = {} # rel_path -> file size
  1130. failed_dbs: list[str] = [] # present *.db that could not be snapshotted
  1131. # #68805: track protected DB files skipped for size — they are snapshot
  1132. # incompleteness just like a failed copy, so pruning must be suppressed
  1133. # to preserve the older complete snapshot that may contain the only
  1134. # recoverable database.
  1135. oversized_skipped: list[str] = []
  1136. for rel in _QUICK_STATE_FILES:
  1137. src = home / rel
  1138. if not src.exists():
  1139. continue
  1140. if src.is_dir():
  1141. # Walk the directory and record each file individually in the
  1142. # manifest so restore can treat them uniformly. Empty dirs are
  1143. # skipped (nothing to snapshot).
  1144. for sub in src.rglob("*"):
  1145. if not sub.is_file():
  1146. continue
  1147. sub_rel = sub.relative_to(home).as_posix()
  1148. # Skip heavy, regenerable per-board subtrees (scratch
  1149. # workspaces and task attachments can be large); we only need
  1150. # the board databases + their metadata to restore a board.
  1151. if "/workspaces/" in f"/{sub_rel}/" or "/attachments/" in f"/{sub_rel}/":
  1152. continue
  1153. if _too_large(sub, sub_rel):
  1154. if sub.suffix == ".db":
  1155. oversized_skipped.append(sub_rel)
  1156. continue
  1157. dst = staging_dir / sub_rel
  1158. dst.parent.mkdir(parents=True, exist_ok=True)
  1159. try:
  1160. # Route SQLite DBs through the WAL-safe backup() path so a
  1161. # board DB with an open WAL (the gateway may hold it at
  1162. # snapshot time) is captured consistently.
  1163. if sub.suffix == ".db":
  1164. if not _safe_copy_db(sub, dst):
  1165. failed_dbs.append(sub_rel)
  1166. print(
  1167. f" ⚠ Snapshot: SQLite safe copy FAILED for {sub_rel} "
  1168. f"— file may be locked or corrupted"
  1169. )
  1170. if is_zeroed_sqlite_file(sub):
  1171. print(
  1172. f" ⚠ Snapshot: {sub_rel} looks ZEROED "
  1173. f"(no SQLite header; {sub.stat().st_size} bytes of NULs?)"
  1174. )
  1175. continue
  1176. else:
  1177. shutil.copy2(sub, dst)
  1178. manifest[sub_rel] = dst.stat().st_size
  1179. except (OSError, PermissionError) as exc:
  1180. logger.warning("Could not snapshot %s: %s", sub_rel, exc)
  1181. continue
  1182. if not src.is_file():
  1183. continue
  1184. if _too_large(src, rel):
  1185. if src.suffix == ".db":
  1186. oversized_skipped.append(rel)
  1187. continue
  1188. dst = staging_dir / rel
  1189. dst.parent.mkdir(parents=True, exist_ok=True)
  1190. try:
  1191. if src.suffix == ".db":
  1192. if not _safe_copy_db(src, dst):
  1193. failed_dbs.append(rel)
  1194. print(
  1195. f" ⚠ Snapshot: SQLite safe copy FAILED for {rel} "
  1196. f"— file may be locked or corrupted"
  1197. )
  1198. if is_zeroed_sqlite_file(src):
  1199. print(
  1200. f" ⚠ Snapshot: {rel} looks ZEROED "
  1201. f"(no SQLite header; {src.stat().st_size} bytes)"
  1202. )
  1203. continue
  1204. else:
  1205. shutil.copy2(src, dst)
  1206. manifest[rel] = dst.stat().st_size
  1207. except (OSError, PermissionError) as exc:
  1208. logger.warning("Could not snapshot %s: %s", rel, exc)
  1209. if failed_dbs:
  1210. # Critical: update path used to log-and-continue with exit 0, so a
  1211. # missing state.db backup looked like a successful pre-update snapshot
  1212. # (#68474). Surface this on stdout where operators actually look.
  1213. print(
  1214. " ⚠ CRITICAL: could not snapshot DB file(s): "
  1215. + ", ".join(failed_dbs)
  1216. )
  1217. print(
  1218. f" ⚠ If sessions disappear after the update, check {root}. {_snapshot_recovery_hint()}"
  1219. )
  1220. logger.error(
  1221. "Quick snapshot failed to capture DB file(s): %s",
  1222. ", ".join(failed_dbs),
  1223. )
  1224. if not manifest:
  1225. shutil.rmtree(staging_dir, ignore_errors=True)
  1226. if failed_dbs:
  1227. # Distinguish "nothing to snapshot" from "state.db present but unreadable"
  1228. print(
  1229. " ⚠ Snapshot aborted: no files captured "
  1230. f"(failed DBs: {', '.join(failed_dbs)})"
  1231. )
  1232. return None
  1233. # Write manifest
  1234. meta = {
  1235. "id": snap_id,
  1236. "timestamp": ts,
  1237. "label": label,
  1238. "file_count": len(manifest),
  1239. "total_size": sum(manifest.values()),
  1240. "files": manifest,
  1241. "failed_dbs": failed_dbs,
  1242. "oversized_skipped": oversized_skipped,
  1243. }
  1244. with open(staging_dir / "manifest.json", "w", encoding="utf-8") as f:
  1245. json.dump(meta, f, indent=2)
  1246. # Make the staged quick snapshot owner-only before it is published. The
  1247. # staging directory is private from creation, so copied source modes can
  1248. # be normalized safely before the final atomic rename exposes the
  1249. # snapshot. Permission failures are intentionally fatal: publishing a
  1250. # readable recovery bundle is worse than reporting a failed snapshot.
  1251. if os.name != "nt":
  1252. os.chmod(root, 0o700)
  1253. os.chmod(staging_dir, 0o700)
  1254. for path in staging_dir.rglob("*"):
  1255. if path.is_dir():
  1256. os.chmod(path, 0o700)
  1257. elif path.is_file():
  1258. os.chmod(path, 0o600)
  1259. os.replace(staging_dir, snap_dir)
  1260. # Auto-prune. Defaults preserve historical manual /snapshot behavior; callers
  1261. # with known high-churn safety snapshots (for example pre-update) can pass a
  1262. # smaller keep value so large state.db copies do not accumulate indefinitely.
  1263. # #68805 review: skip pruning when a present DB failed to capture OR was
  1264. # skipped for size — either way the snapshot is incomplete and the older
  1265. # snapshot may contain the only recoverable database.
  1266. incomplete = failed_dbs or oversized_skipped
  1267. if not incomplete:
  1268. _prune_quick_snapshots(root, keep=_QUICK_DEFAULT_KEEP if keep is None else keep)
  1269. else:
  1270. if oversized_skipped:
  1271. print(
  1272. " ⚠ Skipping snapshot prune: DB file(s) skipped for size: "
  1273. + ", ".join(oversized_skipped)
  1274. )
  1275. logger.warning(
  1276. "Quick snapshot skipped oversized DB file(s): %s",
  1277. ", ".join(oversized_skipped),
  1278. )
  1279. logger.warning(
  1280. "Skipping snapshot prune because %d DB(s) failed to capture "
  1281. "and/or %d were oversized — preserving older snapshots as "
  1282. "recovery source",
  1283. len(failed_dbs), len(oversized_skipped),
  1284. )
  1285. logger.info(
  1286. "quick snapshot phase=copy status=complete id=%s files=%d bytes=%d",
  1287. snap_id,
  1288. len(manifest),
  1289. sum(manifest.values()),
  1290. )
  1291. return snap_id
  1292. def list_quick_snapshots(
  1293. limit: int = 20,
  1294. hermes_home: Optional[Path] = None,
  1295. ) -> list[dict[str, Any]]:
  1296. """List existing quick state snapshots, most recent first."""
  1297. root = _quick_snapshot_root(hermes_home)
  1298. if not root.exists():
  1299. return []
  1300. results = []
  1301. for d in sorted(root.iterdir(), reverse=True):
  1302. if not d.is_dir() or d.name.startswith(".") or d.name.endswith(".partial"):
  1303. continue
  1304. manifest_path = d / "manifest.json"
  1305. if manifest_path.exists():
  1306. try:
  1307. with open(manifest_path, encoding="utf-8-sig") as f:
  1308. results.append(json.load(f))
  1309. except (json.JSONDecodeError, OSError):
  1310. results.append({"id": d.name, "file_count": 0, "total_size": 0})
  1311. if len(results) >= limit:
  1312. break
  1313. return results
  1314. def _is_trusted_root_auth_alias(path: Path, home: Path) -> bool:
  1315. """True only for a file symlink resolving to this restore home's default-root auth store."""
  1316. if not path.is_symlink():
  1317. return False
  1318. try:
  1319. root = get_default_hermes_root(home=home).resolve(strict=False)
  1320. if home.resolve(strict=False) == root:
  1321. return False
  1322. trusted = (root / "auth.json").resolve(strict=False)
  1323. trusted.relative_to(root)
  1324. return path.resolve(strict=False) == trusted
  1325. except (OSError, RuntimeError, ValueError):
  1326. return False
  1327. def restore_quick_snapshot(
  1328. snapshot_id: str,
  1329. hermes_home: Optional[Path] = None,
  1330. ) -> bool:
  1331. """Restore the whole snapshot offline, or refuse before changing any file.
  1332. Overwrites current state files with the snapshot's copies.
  1333. Returns True if at least one file was restored and the listed auth.json
  1334. was not refused or skipped.
  1335. """
  1336. from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
  1337. require_file_tooling("Snapshot restore")
  1338. home = hermes_home or get_hermes_home()
  1339. try:
  1340. with exclusive_maintenance([home]):
  1341. return _restore_quick_snapshot_exclusive(snapshot_id, home)
  1342. except OwnershipConflict as exc:
  1343. logger.error("%s", exc)
  1344. return False
  1345. def _snapshot_epochs_restorable(files, entry_paths) -> bool:
  1346. """False (logged) when any listed ``state.db`` destination cannot prove its runtime epoch."""
  1347. for rel in files:
  1348. entry = entry_paths(rel)
  1349. if entry and entry[0].exists() and "state.db" in (entry[1].name, entry[1].resolve().name):
  1350. try:
  1351. _restore_destination_epoch(entry[1])
  1352. except OSError as exc:
  1353. logger.error("%s", exc)
  1354. return False
  1355. return True
  1356. def _restore_snapshot_entry(rel: str, src: Path, dst: Path) -> bool:
  1357. """Publish one quick-snapshot file over ``dst``; False (logged) when refused or failed."""
  1358. try:
  1359. if dst.suffix == ".db":
  1360. # Restore through SQLite backup API so live connections
  1361. # (gateway, dashboard, another CLI session) see the
  1362. # restored data instead of continuing to serve stale
  1363. # cached pages from a replaced inode (issue #65942).
  1364. if not _safe_restore_db(src, dst):
  1365. # Refused, failed, or source failed its integrity check:
  1366. # dst left as it was. Count as a failure, not a restore.
  1367. logger.error("Failed to restore %s: refused or source integrity check failed (see previous log)", rel)
  1368. return False
  1369. elif rel == "auth.json":
  1370. # Refresh tokens for these OAuth providers rotate on use. A historical
  1371. # snapshot can therefore contain a spent pair even though the current
  1372. # auth.json has the live successor. Restore the historical auth state
  1373. # while retaining that live single-use grant under the auth-store lock.
  1374. if not _restore_auth_json(src, dst):
  1375. logger.error("Failed to restore %s safely", rel)
  1376. return False
  1377. else:
  1378. shutil.copy2(src, dst)
  1379. return True
  1380. except (OSError, PermissionError) as exc:
  1381. logger.error("Failed to restore %s: %s", rel, exc)
  1382. return False
  1383. def quick_snapshot_dir(snapshot_id: str, hermes_home: Optional[Path] = None) -> Optional[Path]:
  1384. """The directory of a restorable quick snapshot, or None for an invalid or missing id."""
  1385. root = _quick_snapshot_root(hermes_home)
  1386. # Security: reject snapshot_id values that contain path separators or
  1387. # traversal sequences so that `root / snapshot_id` stays inside root.
  1388. if not snapshot_id or "/" in snapshot_id or "\\" in snapshot_id or snapshot_id in (".", ".."):
  1389. logger.error("Invalid snapshot_id: %s", snapshot_id)
  1390. return None
  1391. snap_dir = root / snapshot_id
  1392. # Confirm the resolved path is still inside root (handles symlinks etc.)
  1393. try:
  1394. snap_dir.resolve().relative_to(root.resolve())
  1395. except ValueError:
  1396. logger.error("Snapshot path traversal blocked for id: %s", snapshot_id)
  1397. return None
  1398. if not snap_dir.is_dir() or not (snap_dir / "manifest.json").exists():
  1399. return None
  1400. return snap_dir
  1401. def _restore_quick_snapshot_exclusive(snapshot_id: str, home: Path) -> bool:
  1402. """Restore state from a quick snapshot while ``home`` is reserved."""
  1403. from gateway.runtime_ownership import exclusive_maintenance
  1404. snap_dir = quick_snapshot_dir(snapshot_id, home)
  1405. if snap_dir is None:
  1406. return False
  1407. with open(snap_dir / "manifest.json", encoding="utf-8-sig") as f:
  1408. meta = json.load(f)
  1409. snap_res, home_res = snap_dir.resolve(), home.resolve()
  1410. def _entry_paths(rel: str) -> Optional[tuple[Path, Path]]:
  1411. # Security: reject absolute paths and traversals in manifest entries
  1412. src, dst = snap_dir / rel, home / rel
  1413. if not src.resolve().is_relative_to(snap_res):
  1414. return None
  1415. if not dst.resolve().is_relative_to(home_res):
  1416. # Named profiles may deliberately share the machine-root auth store via an
  1417. # auth.json symlink. Let only that exact trusted alias reach _restore_auth_json;
  1418. # every other manifest destination outside the profile remains a traversal.
  1419. if rel != "auth.json" or not _is_trusted_root_auth_alias(dst, home):
  1420. return None
  1421. return src, dst
  1422. homes = {home}
  1423. for rel in meta.get("files", {}):
  1424. entry = _entry_paths(rel)
  1425. if entry and entry[1].suffix == ".db":
  1426. homes.update([entry[1].absolute().parent, entry[1].resolve().parent])
  1427. with exclusive_maintenance(homes):
  1428. # Epoch safety is a whole-profile preflight: copying config first would
  1429. # both partially roll back the profile and report a refused DB as success.
  1430. if not _snapshot_epochs_restorable(meta.get("files", {}), _entry_paths):
  1431. return False
  1432. restored = 0
  1433. auth_restore_failed = False
  1434. for rel in meta.get("files", {}):
  1435. entry = _entry_paths(rel)
  1436. if entry is None:
  1437. logger.error("Manifest path traversal blocked: %s", rel)
  1438. if rel == "auth.json":
  1439. auth_restore_failed = True
  1440. continue
  1441. src, dst = entry
  1442. if not src.exists():
  1443. if rel == "auth.json":
  1444. logger.error("Snapshot auth.json listed in manifest is missing: %s", src)
  1445. auth_restore_failed = True
  1446. continue
  1447. dst.parent.mkdir(parents=True, exist_ok=True)
  1448. if _restore_snapshot_entry(rel, src, dst):
  1449. restored += 1
  1450. elif rel == "auth.json":
  1451. auth_restore_failed = True
  1452. logger.info("Restored %d files from snapshot %s", restored, snapshot_id)
  1453. return restored > 0 and not auth_restore_failed
  1454. def restore_cron_jobs_if_emptied(
  1455. snapshot_id: str,
  1456. hermes_home: Optional[Path] = None,
  1457. ) -> Optional[dict[str, Any]]:
  1458. """Safety net for silent cron-job loss across ``hermes update``.
  1459. Config-version migrations have been observed to leave ``cron/jobs.json``
  1460. valid-but-empty after an update, silently dropping every scheduled job
  1461. (issue #34600). The desktop scheduler can also overwrite the file with its
  1462. own small set of internally-tracked crons, causing partial loss (issue
  1463. #52144).
  1464. This compares the *current* job count against the pre-update snapshot. If
  1465. the live file now has **fewer** jobs than the snapshot, the snapshot copy
  1466. of ``cron/jobs.json`` is restored in place.
  1467. The check is deliberately conservative — it only ever restores when there
  1468. is unambiguous evidence of loss (snapshot had more jobs than live file),
  1469. so a user who genuinely deleted jobs during/after the update is never
  1470. second-guessed, and an unreadable live file (count ``None``) is left
  1471. untouched so real corruption still surfaces.
  1472. Args:
  1473. snapshot_id: The pre-update quick-snapshot id (from
  1474. :func:`create_quick_snapshot`).
  1475. hermes_home: Override for the Hermes home directory (tests).
  1476. Returns:
  1477. ``None`` when no action was taken (the common, healthy path). On a
  1478. successful restore, a dict ``{"restored": True, "job_count": N,
  1479. "snapshot_id": ...}`` so the caller can warn the user.
  1480. """
  1481. if not snapshot_id:
  1482. return None
  1483. home = hermes_home or get_hermes_home()
  1484. live_path = home / _CRON_JOBS_REL
  1485. live_count = _count_cron_jobs(live_path)
  1486. # ``None`` (missing or unparseable) is intentionally left alone — that's a
  1487. # different failure mode the user should see rather than have papered over.
  1488. if live_count is None:
  1489. return None
  1490. snap_path = _quick_snapshot_root(home) / snapshot_id / _CRON_JOBS_REL
  1491. snap_count = _count_cron_jobs(snap_path)
  1492. if not snap_count: # None or 0 — nothing worth restoring
  1493. return None
  1494. # Restore when live has FEWER jobs than the pre-update snapshot.
  1495. # Catches both total loss (0 vs N) and partial loss (1 vs 19) — the
  1496. # desktop scheduler can overwrite jobs.json with its own small set of
  1497. # internally-tracked crons after an update/restart.
  1498. if live_count >= snap_count:
  1499. return None
  1500. try:
  1501. live_path.parent.mkdir(parents=True, exist_ok=True)
  1502. shutil.copy2(snap_path, live_path)
  1503. except (OSError, PermissionError) as exc:
  1504. logger.error(
  1505. "Cron jobs were emptied during update but auto-restore failed: %s", exc
  1506. )
  1507. return None
  1508. logger.warning(
  1509. "Restored %d cron job(s) from pre-update snapshot %s "
  1510. "(live file had %d job(s), snapshot had %d — jobs were lost during migration)",
  1511. snap_count,
  1512. snapshot_id,
  1513. live_count,
  1514. snap_count,
  1515. )
  1516. return {"restored": True, "job_count": snap_count, "snapshot_id": snapshot_id}
  1517. def _sibling_profile_homes(invoking_home: Path) -> list[tuple[str, Path]]:
  1518. """(name, home) for every OTHER profile on this install. Never raises.
  1519. The update's code swap and gateway fleet restart touch every profile,
  1520. so the pre-update snapshot must too (#66140). The invoking profile is
  1521. excluded — its snapshot is taken by the existing call.
  1522. """
  1523. homes: list[tuple[str, Path]] = []
  1524. try:
  1525. from hermes_cli.profiles import (
  1526. _get_default_hermes_home,
  1527. _get_profiles_root,
  1528. _PROFILE_ID_RE,
  1529. )
  1530. invoking = invoking_home.resolve()
  1531. default_home = _get_default_hermes_home()
  1532. if default_home.is_dir() and default_home.resolve() != invoking:
  1533. homes.append(("default", default_home))
  1534. root = _get_profiles_root()
  1535. if root.is_dir():
  1536. for entry in sorted(root.iterdir()):
  1537. if (
  1538. entry.is_dir()
  1539. and entry.name != "default"
  1540. and _PROFILE_ID_RE.match(entry.name)
  1541. and entry.resolve() != invoking
  1542. ):
  1543. homes.append((entry.name, entry))
  1544. except Exception as exc:
  1545. logger.debug("Sibling profile enumeration failed: %s", exc)
  1546. return homes
  1547. def create_pre_update_snapshots_all_profiles(
  1548. invoking_home: Optional[Path] = None,
  1549. keep: Optional[int] = None,
  1550. max_file_size: Optional[int] = None,
  1551. ) -> dict[str, str]:
  1552. """Pre-update quick snapshots for every SIBLING profile (#66140).
  1553. Same snapshot set, same per-file size cap, same keep policy as the
  1554. invoking profile's snapshot — identical semantics per profile, no
  1555. partial-tier coherence class. Each sibling's snapshot lands under its
  1556. OWN ``<home>/state-snapshots/`` so per-profile restore tooling finds
  1557. it where it expects. Returns ``{profile_name: snapshot_id}`` for the
  1558. siblings that snapshotted successfully. Never raises.
  1559. """
  1560. results: dict[str, str] = {}
  1561. home = invoking_home or get_hermes_home()
  1562. for name, profile_home in _sibling_profile_homes(home):
  1563. try:
  1564. snap_id = create_quick_snapshot(
  1565. label="pre-update",
  1566. hermes_home=profile_home,
  1567. keep=keep,
  1568. max_file_size=max_file_size,
  1569. )
  1570. if snap_id:
  1571. results[name] = snap_id
  1572. except Exception as exc:
  1573. logger.debug("Pre-update snapshot for profile %s failed: %s", name, exc)
  1574. return results
  1575. # Config paths that the update flow must never change (#64160): the model
  1576. # routing keys and the Mixture-of-Agents section are consumed machine-wide
  1577. # (gateway, cron, desktop), so an update/repair cycle that rewrites them
  1578. # silently redirects paid inference. Each entry is a dotted path into the raw
  1579. # config.yaml document; a single-element tuple protects the whole section.
  1580. _PROTECTED_CONFIG_PATHS: tuple[tuple[str, ...], ...] = (
  1581. ("model", "provider"),
  1582. ("model", "default"),
  1583. ("model", "base_url"),
  1584. ("model", "api_key"),
  1585. ("moa",),
  1586. )
  1587. def _read_raw_yaml_dict(path: Path) -> Optional[dict[str, Any]]:
  1588. """Parse ``path`` as a YAML mapping. ``None`` = missing/unreadable/non-dict."""
  1589. if not path.is_file():
  1590. return None
  1591. try:
  1592. import hermes_yaml as yaml
  1593. with open(path, "r", encoding="utf-8-sig") as f:
  1594. data = yaml.safe_load(f)
  1595. except Exception:
  1596. return None
  1597. return data if isinstance(data, dict) else None
  1598. def restore_config_model_settings_if_rewritten(
  1599. snapshot_id: str,
  1600. hermes_home: Optional[Path] = None,
  1601. ) -> Optional[dict[str, Any]]:
  1602. """Safety net for silent config.yaml model/MoA loss across ``hermes update``.
  1603. Desktop update/repair cycles have been observed to rewrite user-set
  1604. ``model.provider``/``model.default`` and drop the ``moa:`` section
  1605. entirely (issue #64160; the macOS repair/relaunch variant rewrote a
  1606. pinned ``model.default`` to a transient composer pick). These keys are
  1607. consumed by the gateway and unattended cron jobs too, so a rewrite
  1608. silently changes paid inference behavior machine-wide.
  1609. Mirrors :func:`restore_cron_jobs_if_emptied`: compare the *current*
  1610. config against the pre-update snapshot taken minutes earlier by this
  1611. same update run, and restore only the protected keys — never the whole
  1612. file — when a value the user had set was changed or dropped. Everything
  1613. the update legitimately wrote (version stamps, new sections) is left in
  1614. place.
  1615. Args:
  1616. snapshot_id: The pre-update quick-snapshot id (from
  1617. :func:`create_quick_snapshot`).
  1618. hermes_home: Override for the Hermes home directory (tests/siblings).
  1619. Returns:
  1620. ``None`` when no action was taken (the common, healthy path). On a
  1621. successful restore, ``{"restored": True, "keys": [...],
  1622. "snapshot_id": ...}`` so the caller can warn the user.
  1623. """
  1624. if not snapshot_id:
  1625. return None
  1626. if not supports_file_tooling(): # Remote Config: never write a LOCAL snapshot's values to the plane
  1627. return None
  1628. home = hermes_home or get_hermes_home()
  1629. live_path = home / "config.yaml"
  1630. snap_path = _quick_snapshot_root(home) / snapshot_id / "config.yaml"
  1631. snap = _read_raw_yaml_dict(snap_path)
  1632. if not snap:
  1633. return None # no snapshot copy — nothing to compare against
  1634. live = _read_raw_yaml_dict(live_path)
  1635. if live is None:
  1636. # Missing or unparseable live config is a different failure mode the
  1637. # user should see rather than have papered over (matches the cron net).
  1638. return None
  1639. restored_keys: list[str] = []
  1640. for dotted in _PROTECTED_CONFIG_PATHS:
  1641. snap_val = _get_config_path_value(snap, dotted)
  1642. if snap_val in (None, "", {}, []):
  1643. continue # user never set it — nothing to protect
  1644. live_val = _get_config_path_value(live, dotted)
  1645. if live_val == snap_val:
  1646. continue
  1647. _set_config_path_value(live, dotted, snap_val)
  1648. restored_keys.append(".".join(dotted))
  1649. if not restored_keys:
  1650. return None
  1651. try:
  1652. from hermes_cli.config import atomic_config_write
  1653. atomic_config_write(live_path, live)
  1654. except (OSError, PermissionError) as exc:
  1655. logger.error(
  1656. "config.yaml model settings were rewritten during update but "
  1657. "auto-restore failed: %s",
  1658. exc,
  1659. )
  1660. return None
  1661. logger.warning(
  1662. "Restored user config value(s) %s from pre-update snapshot %s — "
  1663. "the update flow rewrote them (#64160)",
  1664. ", ".join(restored_keys),
  1665. snapshot_id,
  1666. )
  1667. return {"restored": True, "keys": restored_keys, "snapshot_id": snapshot_id}
  1668. def restore_config_model_settings_all_profiles(
  1669. profile_snapshots: dict[str, str],
  1670. invoking_home: Optional[Path] = None,
  1671. ) -> list[dict[str, Any]]:
  1672. """Run the config model-settings safety net for every sibling profile.
  1673. Same contract as :func:`restore_cron_jobs_all_profiles`: each profile's
  1674. live ``config.yaml`` is compared against ITS OWN same-generation
  1675. pre-update snapshot. Returns one result dict per restored profile, each
  1676. with a ``profile`` key added. Never raises.
  1677. """
  1678. restored: list[dict[str, Any]] = []
  1679. if not profile_snapshots:
  1680. return restored
  1681. home = invoking_home or get_hermes_home()
  1682. by_name = dict(_sibling_profile_homes(home))
  1683. for name, snap_id in profile_snapshots.items():
  1684. profile_home = by_name.get(name)
  1685. if profile_home is None:
  1686. continue
  1687. try:
  1688. result = restore_config_model_settings_if_rewritten(
  1689. snap_id, hermes_home=profile_home
  1690. )
  1691. except Exception as exc:
  1692. logger.debug(
  1693. "Config model-settings restore check for profile %s failed: %s",
  1694. name,
  1695. exc,
  1696. )
  1697. continue
  1698. if result:
  1699. result["profile"] = name
  1700. restored.append(result)
  1701. return restored
  1702. def restore_cron_jobs_all_profiles(
  1703. profile_snapshots: dict[str, str],
  1704. invoking_home: Optional[Path] = None,
  1705. ) -> list[dict[str, Any]]:
  1706. """Run the cron-jobs safety net for every sibling profile (#66140).
  1707. ``profile_snapshots`` is the map returned by
  1708. :func:`create_pre_update_snapshots_all_profiles`. Each profile's live
  1709. ``cron/jobs.json`` is compared against ITS OWN snapshot — restores are
  1710. same-generation by construction (the snapshot was taken minutes ago by
  1711. this update run). Returns one result dict per restored profile, each
  1712. with a ``profile`` key added. Never raises.
  1713. """
  1714. restored: list[dict[str, Any]] = []
  1715. if not profile_snapshots:
  1716. return restored
  1717. home = invoking_home or get_hermes_home()
  1718. by_name = dict(_sibling_profile_homes(home))
  1719. for name, snap_id in profile_snapshots.items():
  1720. profile_home = by_name.get(name)
  1721. if profile_home is None:
  1722. continue
  1723. try:
  1724. result = restore_cron_jobs_if_emptied(snap_id, hermes_home=profile_home)
  1725. except Exception as exc:
  1726. logger.debug("Cron restore check for profile %s failed: %s", name, exc)
  1727. continue
  1728. if result:
  1729. result["profile"] = name
  1730. restored.append(result)
  1731. return restored
  1732. def _prune_quick_snapshots(root: Path, keep: int = _QUICK_DEFAULT_KEEP) -> int:
  1733. """Remove oldest quick snapshots beyond the keep limit. Returns count deleted."""
  1734. if not root.exists():
  1735. return 0
  1736. dirs = sorted(
  1737. (
  1738. d
  1739. for d in root.iterdir()
  1740. if d.is_dir() and not d.name.startswith(".") and not d.name.endswith(".partial")
  1741. ),
  1742. key=lambda d: d.name,
  1743. reverse=True,
  1744. )
  1745. deleted = 0
  1746. for d in dirs[keep:]:
  1747. try:
  1748. shutil.rmtree(d)
  1749. deleted += 1
  1750. except OSError as exc:
  1751. logger.warning("Failed to prune snapshot %s: %s", d.name, exc)
  1752. return deleted
  1753. def prune_quick_snapshots(
  1754. keep: int = _QUICK_DEFAULT_KEEP,
  1755. hermes_home: Optional[Path] = None,
  1756. ) -> int:
  1757. """Manually prune quick snapshots. Returns count deleted."""
  1758. return _prune_quick_snapshots(_quick_snapshot_root(hermes_home), keep=keep)
  1759. def run_quick_backup(args) -> None:
  1760. """CLI entry point for hermes backup --quick."""
  1761. label = getattr(args, "label", None)
  1762. snap_id = create_quick_snapshot(label=label)
  1763. if snap_id:
  1764. print(f"State snapshot created: {snap_id}")
  1765. snaps = list_quick_snapshots()
  1766. print(f" {len(snaps)} snapshot(s) stored in {display_hermes_home()}/state-snapshots/")
  1767. print(f" Restore with: /snapshot restore {snap_id}")
  1768. else:
  1769. print("No state files found to snapshot.")
  1770. # ---------------------------------------------------------------------------
  1771. # Shared full-zip backup helper
  1772. # ---------------------------------------------------------------------------
  1773. def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
  1774. """Single-flight wrapper for automatic full zip backups."""
  1775. try:
  1776. with _backup_operation_lock(hermes_root):
  1777. return _write_full_zip_backup_locked(out_path, hermes_root)
  1778. except BackupInProgressError as exc:
  1779. logger.warning("Full-zip backup skipped: %s", exc)
  1780. return None
  1781. def _write_full_zip_backup_locked(out_path: Path, hermes_root: Path) -> Optional[Path]:
  1782. scan_started = time.monotonic()
  1783. logger.info("automatic backup phase=scan status=started")
  1784. try:
  1785. files_to_add = list(_iter_backup_files(hermes_root, out_path))
  1786. except OSError as exc:
  1787. logger.warning("Full-zip backup: walk failed: %s", exc)
  1788. return None
  1789. if not files_to_add:
  1790. return None
  1791. logger.info("automatic backup phase=scan status=complete duration_ms=%.1f files=%d",
  1792. (time.monotonic() - scan_started) * 1000, len(files_to_add))
  1793. def _db_failure(rel_path: Path) -> None:
  1794. logger.warning("Full-zip backup aborted: SQLite snapshot failed for %s", rel_path)
  1795. raise _SQLiteSnapshotError(str(rel_path))
  1796. errors: list[str] = []
  1797. def _capped_errors() -> str:
  1798. # Cap the logged list: a broken tree can fail thousands of entries in one run.
  1799. shown = "; ".join(errors[:10])
  1800. return f"{shown} (+{len(errors) - 10} more)" if len(errors) > 10 else shown
  1801. # Salvage name keeps an incomplete archive out of normal retention (otherwise the next
  1802. # complete run would prune the last complete backups by count) and, because the partial is
  1803. # published straight there, never clobbers a previous good backup at ``out_path``. A run where
  1804. # every entry failed salvages nothing, so its empty archive is discarded rather than kept.
  1805. salvage_path = out_path.with_name(out_path.stem + _INCOMPLETE_ZIP_SUFFIX)
  1806. published: Optional[Path] = None
  1807. def _publish_path() -> Optional[Path]:
  1808. # Decide clean/salvage/discard once; the post-publish stat and return reuse it.
  1809. nonlocal published
  1810. if not errors:
  1811. published = out_path
  1812. elif len(errors) < len(files_to_add):
  1813. published = salvage_path
  1814. return published
  1815. archive_started = time.monotonic()
  1816. try:
  1817. with _atomic_output_path(out_path, _publish_path) as archive_path, zipfile.ZipFile(
  1818. archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
  1819. _write_zip_entries(
  1820. zf, files_to_add, out_path, on_db_failure=_db_failure, track_bytes=False,
  1821. on_error=lambda rel, exc: errors.append(f"{rel}: {exc}"),
  1822. on_progress=lambda i: logger.info(
  1823. "automatic backup phase=archive status=progress completed=%d total=%d", i, len(files_to_add)))
  1824. except (OSError, _SQLiteSnapshotError) as exc:
  1825. # The hidden partial is already gone; ``out_path`` may be a previous valid backup: keep it.
  1826. logger.warning("Full-zip backup: zip write failed: %s", exc)
  1827. return None
  1828. if published is None:
  1829. logger.warning("Full-zip backup: every entry failed, nothing salvaged: %s", _capped_errors())
  1830. return None
  1831. zip_size = published.stat().st_size
  1832. if published != out_path:
  1833. logger.warning(
  1834. "automatic backup phase=archive status=incomplete duration_ms=%.1f files=%d errors=%d "
  1835. "bytes=%d salvage=%s skipped=%s",
  1836. (time.monotonic() - archive_started) * 1000, len(files_to_add), len(errors), zip_size,
  1837. salvage_path, _capped_errors())
  1838. return None
  1839. logger.info("automatic backup phase=archive status=complete duration_ms=%.1f files=%d bytes=%d",
  1840. (time.monotonic() - archive_started) * 1000, len(files_to_add), zip_size)
  1841. return out_path
  1842. # ---------------------------------------------------------------------------