backup_restore.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709
  1. """SQLite-safe restore and archive-member publish plumbing for backups.
  2. Owns the restore side of ``hermes_cli.backup``: the page-copy SQLite restore
  3. (``_safe_restore_db`` plus the foreign-holder scan) and the zip-member publish
  4. helpers used by ``hermes import`` and ``/snapshot restore``. Backup
  5. *creation* (``run_backup``, full-zip writing) stays in ``hermes_cli.backup``,
  6. which composes these helpers.
  7. """
  8. import json
  9. import logging
  10. import os
  11. import shutil
  12. import sqlite3
  13. import stat
  14. import sys
  15. import tempfile
  16. import zipfile
  17. from contextlib import closing, contextmanager
  18. from pathlib import Path
  19. from typing import Any, List, Optional, Tuple
  20. from hermes_state_holders import read_only_db_uri
  21. from hermes_cli.backup_sqlite import _safe_copy_db
  22. from utils import (
  23. _preserve_file_mode, _preserve_file_owner, _restore_file_mode, _restore_file_owner, atomic_replace,
  24. mkstemp_beside,
  25. )
  26. logger = logging.getLogger(__name__)
  27. def _foreign_db_holder_pids(db_path: Path) -> Optional[list[int]]:
  28. """PIDs of OTHER processes holding *db_path* or its WAL/SHM open.
  29. Linux-only ``/proc/<pid>/fd`` scan (no psutil dependency), preserving the
  30. kernel's ``(deleted)`` suffix so an already-unlinked sidecar generation —
  31. the #90950 split-brain fingerprint — still counts as held. Returns
  32. ``None`` when the scan is unavailable (non-Linux, or /proc unreadable);
  33. callers must treat ``None`` as "unknown", not as "no holders".
  34. """
  35. if not sys.platform.startswith("linux"):
  36. return None
  37. def _canonical(path: str) -> str:
  38. return os.path.normcase(
  39. os.path.abspath(path.removesuffix(" (deleted)"))
  40. )
  41. canonical_db = _canonical(os.fspath(db_path))
  42. watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
  43. pids: list[int] = []
  44. try:
  45. own_pid = os.getpid()
  46. for pid_str in os.listdir("/proc"):
  47. if not pid_str.isdigit():
  48. continue
  49. pid = int(pid_str)
  50. if pid == own_pid:
  51. continue
  52. fd_dir = f"/proc/{pid}/fd"
  53. try:
  54. fds = os.listdir(fd_dir)
  55. except OSError:
  56. continue
  57. for fd in fds:
  58. try:
  59. target = os.readlink(f"{fd_dir}/{fd}")
  60. except OSError:
  61. continue
  62. if _canonical(target) in watched:
  63. pids.append(pid)
  64. break
  65. except OSError:
  66. return None
  67. return pids
  68. def _auth_restore_target(dst: Path) -> Optional[Path]:
  69. """Writable auth-store path whose lock and publish name the same underlying file.
  70. A file symlink is resolved so refresh writers and restore take the same auth.lock. A hard-linked
  71. auth.json cannot be atomically replaced without splitting a deliberately shared store into two
  72. inodes, so that topology fails closed instead of silently breaking credential sharing.
  73. """
  74. try:
  75. if dst.exists():
  76. links = dst.stat().st_nlink
  77. if links > 1:
  78. logger.error(
  79. "Refusing auth.json restore to %s: it has %d hard links; atomic replacement "
  80. "would split a shared auth store",
  81. dst,
  82. links,
  83. )
  84. return None
  85. return dst.resolve(strict=False) if dst.is_symlink() else dst
  86. except (OSError, RuntimeError) as exc:
  87. logger.error("Refusing auth.json restore to %s: cannot resolve store identity: %s", dst, exc)
  88. return None
  89. def _restore_auth_json(src: Path, dst: Path) -> bool:
  90. """Restore auth.json without rolling back a live single-use OAuth generation.
  91. The live read and final write share the canonical auth-store lock, so a concurrent refresh
  92. cannot land between preservation and publish.
  93. """
  94. try:
  95. snapshot_store = json.loads(src.read_text(encoding="utf-8-sig"))
  96. except (OSError, ValueError, UnicodeError) as exc:
  97. logger.error("Refusing auth.json restore from %s: %s", src, exc)
  98. return False
  99. if not isinstance(snapshot_store, dict):
  100. logger.error("Refusing auth.json restore from %s: top level is not an object", src)
  101. return False
  102. if not (
  103. isinstance(snapshot_store.get("providers"), dict)
  104. or isinstance(snapshot_store.get("credential_pool"), dict)
  105. or isinstance(snapshot_store.get("systems"), dict)
  106. ):
  107. logger.error("Refusing auth.json restore from %s: unrecognized auth-store shape", src)
  108. return False
  109. # Legacy "systems" stores are still valid snapshot input. Mirror auth._load_auth_store's
  110. # migration without asking that loader to create a .corrupt sidecar inside the snapshot.
  111. if (
  112. not isinstance(snapshot_store.get("providers"), dict)
  113. and not isinstance(snapshot_store.get("credential_pool"), dict)
  114. and isinstance(snapshot_store.get("systems"), dict)
  115. ):
  116. systems = snapshot_store["systems"]
  117. providers = {"nous": systems["nous_portal"]} if "nous_portal" in systems else {}
  118. # Match auth._load_auth_store's migration shape exactly: the legacy
  119. # "systems" container itself is obsolete and must not be written back.
  120. snapshot_store = {
  121. "providers": providers,
  122. "active_provider": "nous" if providers else None,
  123. }
  124. try:
  125. from hermes_cli.auth import _auth_store_lock, _load_auth_store, _save_auth_store
  126. from hermes_cli.auth_oauth_grants import (
  127. merge_snapshot_auth_preserving_live_single_use_grants,
  128. )
  129. target = _auth_restore_target(dst)
  130. if target is None:
  131. return False
  132. with _auth_store_lock(target_path=target):
  133. live_store = _load_auth_store(target)
  134. restored = merge_snapshot_auth_preserving_live_single_use_grants(
  135. snapshot_store, live_store
  136. )
  137. _save_auth_store(restored, target_path=target)
  138. return True
  139. except Exception as exc:
  140. logger.error("Failed to restore %s safely: %s", dst, exc)
  141. return False
  142. def _safe_restore_db(src: Path, dst: Path) -> bool:
  143. """Restore only while holding the destination authority's maintenance reservation.
  144. Returns ``False``, without touching *dst*, when *src* fails the SQLite
  145. integrity check, when the runtime owns the profile, or when a canonical
  146. ``state.db`` epoch floor cannot be proven.
  147. """
  148. from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
  149. from hermes_cli.backup import verify_sqlite_integrity
  150. # backup() copies pages without validating their contents; its fallback
  151. # copies bytes even when SQLite rejected the source. Neither may touch the
  152. # destination until the snapshot passes the existing bounded integrity policy.
  153. source_check = verify_sqlite_integrity(src)
  154. if not source_check["valid"]:
  155. logger.error("Refusing SQLite restore from %s: %s", src, source_check["message"])
  156. return False
  157. try:
  158. with exclusive_maintenance([dst.absolute().parent, dst.resolve().parent]):
  159. with _restore_epoch_source(src, dst) as prepared:
  160. return _restore_db_pages(prepared, dst)
  161. except (OwnershipConflict, OSError, sqlite3.Error) as exc:
  162. logger.error("%s", exc)
  163. return False
  164. def _restore_epoch(path: Path) -> int:
  165. if not path.exists():
  166. return 0
  167. with closing(sqlite3.connect(read_only_db_uri(path), uri=True)) as conn:
  168. if not conn.execute("SELECT 1 FROM sqlite_master WHERE name='runtime_epoch'").fetchone():
  169. return 0
  170. row = conn.execute("SELECT epoch FROM runtime_epoch WHERE singleton=1").fetchone()
  171. return int(row[0]) if row else 0
  172. def _restore_destination_epoch(dst: Path) -> int:
  173. try:
  174. return _restore_epoch(dst)
  175. except (OSError, sqlite3.Error, ValueError) as exc:
  176. raise OSError(
  177. f"Cannot read the canonical runtime epoch at {dst}; refusing in-place restore. "
  178. "Use `hermes sessions recover --source <snapshot> --output <separate-path>` "
  179. "to salvage transcripts into a separate output."
  180. ) from exc
  181. @contextmanager
  182. def _restore_epoch_source(src: Path, dst: Path):
  183. """Stage an epoch floor into the image BEFORE publishing it atomically.
  184. Updating the destination after backup() would leave a crash window that
  185. reuses an older worker epoch. Never modify the user's source snapshot.
  186. Full restores retain all admissions/worker receipts; normal startup advances
  187. this floor and reconciles started work to unknown, not replayable queued work.
  188. An unreadable destination cannot prove its epoch floor: use transcript salvage
  189. into a separate output rather than silently restoring with a recycled epoch.
  190. """
  191. if dst.name != 'state.db' and dst.resolve().name != 'state.db':
  192. yield src
  193. return
  194. floor = _restore_destination_epoch(dst)
  195. if floor <= _restore_epoch(src):
  196. yield src
  197. return
  198. with tempfile.TemporaryDirectory(prefix='.restore-epoch-', dir=dst.parent) as work:
  199. prepared = Path(work) / 'state.db'
  200. if not _safe_copy_db(src, prepared):
  201. raise OSError('Unable to stage a restore with a safe runtime epoch')
  202. with closing(sqlite3.connect(prepared)) as conn:
  203. conn.execute("""CREATE TABLE IF NOT EXISTS runtime_epoch (
  204. singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
  205. epoch INTEGER NOT NULL CHECK (epoch > 0), instance_id TEXT NOT NULL)""")
  206. conn.execute("""INSERT INTO runtime_epoch VALUES(1,?, 'offline-restore')
  207. ON CONFLICT(singleton) DO UPDATE SET epoch=excluded.epoch,
  208. instance_id=excluded.instance_id""", (floor,))
  209. conn.commit()
  210. prepared.chmod(src.stat().st_mode)
  211. yield prepared
  212. def _restore_db_pages(src: Path, dst: Path) -> bool:
  213. """Restore a SQLite database from snapshot *src* into live *dst*.
  214. Uses SQLite's backup() API to write snapshot pages into the live
  215. database file, preserving the file's inode and WAL state so that
  216. any other process still holding the DB open (gateway, dashboard,
  217. another CLI session) sees the restored data on the next read —
  218. instead of continuing to serve stale cached pages from a replaced
  219. inode.
  220. The old approach was ``unlink() + move()``, which replaced the file
  221. under any live connection. SQLite connections cache pages in
  222. per-connection page caches keyed by inode; after an unlink+move the
  223. old inode still existed (the live connection held a reference), so
  224. that connection continued serving the pre-restore data while new
  225. connections saw the restored snapshot — a partial/inconsistent
  226. state (issue #65942).
  227. By writing pages through the backup API the file inode is preserved,
  228. the WAL journal is updated correctly, and all connections (old and
  229. new) converge on the restored data.
  230. Falls back to the unlink+move approach on failure ONLY when no other
  231. process or in-process connection holds the file: replacing the inode
  232. under a live holder is the #90950 split-brain, so that branch fails
  233. closed (returns ``False``) and the caller reports the file as skipped.
  234. Callers go through ``_safe_restore_db``, which validates *src* and holds
  235. the maintenance reservation and epoch floor around this publication.
  236. """
  237. dst_conn: Optional[sqlite3.Connection] = None
  238. try:
  239. dst_conn = sqlite3.connect(str(dst))
  240. try:
  241. # Force a WAL checkpoint so the backup starts from a clean
  242. # state rather than writing on top of a deep WAL.
  243. dst_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
  244. except sqlite3.Error as exc:
  245. # Optional: backup() still publishes over a deep WAL, just less tidily.
  246. logger.debug("Pre-restore WAL checkpoint of %s skipped: %s", dst, exc)
  247. src_conn = sqlite3.connect(read_only_db_uri(src), uri=True)
  248. try:
  249. src_conn.backup(dst_conn)
  250. finally:
  251. src_conn.close()
  252. dst_conn.close()
  253. # Restore original file permissions from the snapshot
  254. try:
  255. mode = src.stat().st_mode
  256. dst.chmod(mode)
  257. except OSError as exc:
  258. # The data is restored; only the snapshot's file mode could not be carried over.
  259. logger.warning("Restored %s but could not apply the snapshot's file mode: %s", dst, exc)
  260. return True
  261. except (sqlite3.Error, OSError, ValueError) as exc: # ValueError: an unrepresentable path
  262. logger.warning("SQLite safe restore failed for %s -> %s: %s", src, dst, exc)
  263. # Release our own handle on *dst* before the fallback: on Windows an
  264. # open connection blocks unlink() with WinError 32, which would make
  265. # the fallback below dead code exactly when it is needed.
  266. if dst_conn is not None:
  267. try:
  268. dst_conn.close()
  269. except sqlite3.Error as close_exc:
  270. logger.debug("Closing %s before the fallback restore failed: %s", dst, close_exc)
  271. # Fallback: unlink+move (the old approach). This still works for
  272. # the common case where no other process holds the DB open.
  273. from hermes_cli.sqlite_safe_read import (
  274. LiveConnectionError,
  275. offline_file_access,
  276. )
  277. try:
  278. holders = _foreign_db_holder_pids(dst)
  279. if holders:
  280. # Replacing the inode under a live holder is the #90950
  281. # corruption class: the holder keeps writing through a
  282. # deleted-inode fd (split brain), and removing its sidecars
  283. # detaches the WAL index it is checkpointing through. The
  284. # backup-API path above is the live-safe route; if it failed,
  285. # fail closed rather than corrupt.
  286. logger.error(
  287. "Refusing unlink+move restore of %s: process(es) %s still "
  288. "hold the database or its WAL open. Stop them and retry.",
  289. dst, holders,
  290. )
  291. return False
  292. # The foreign-pid scan above deliberately excludes THIS process,
  293. # but an in-process SessionDB (the agent's own handle during
  294. # /snapshot restore, a second SessionDB instance, a read pool)
  295. # is exactly as much of a live holder: unlinking the DB and its
  296. # sidecars under it leaves this process on deleted-inode fds —
  297. # the same #90950 split brain, produced first-party (proven live
  298. # on main: `/proc/self/fd` shows `state.db-wal (deleted)` right
  299. # after this fallback ran under a tracked connection).
  300. # ``offline_file_access`` fails CLOSED when any tracked
  301. # connection to *dst* is live and holds the connection-lifecycle
  302. # lock across the whole swap so no new connection can appear
  303. # mid-replace.
  304. with offline_file_access(dst, what="unlink+move restore of"):
  305. tmp = dst.parent / f".{dst.name}.snap_restore"
  306. shutil.copy2(src, tmp)
  307. dst.unlink(missing_ok=True)
  308. # Drop the destination's sidecars before installing the
  309. # snapshot. The snapshot is a checkpointed ``sqlite3.backup()``
  310. # image (see ``_safe_copy_db``) that owns no WAL, so any
  311. # ``-wal``/``-shm`` still sitting here describes the database we
  312. # just unlinked — an ungracefully killed gateway leaves them
  313. # behind, which is exactly when a restore gets run. SQLite
  314. # replays that foreign WAL over the restored file on the next
  315. # open and the database comes up "malformed" (or silently
  316. # resurrects post-snapshot rows). Same reasoning as
  317. # ``_EXCLUDED_SUFFIXES``, applied to the restore destination.
  318. for _sidecar_suffix in ("-wal", "-shm", "-journal"):
  319. dst.with_name(dst.name + _sidecar_suffix).unlink(missing_ok=True)
  320. shutil.move(str(tmp), str(dst))
  321. return True
  322. except LiveConnectionError as exc2:
  323. logger.error(
  324. "Refusing unlink+move restore of %s: %s Close the in-process "
  325. "database handles (or restart Hermes) and retry.",
  326. dst, exc2,
  327. )
  328. return False
  329. except (OSError, ValueError) as exc2: # ValueError: a path the OS cannot represent
  330. logger.error("Fallback restore also failed for %s -> %s: %s", src, dst, exc2)
  331. return False
  332. def _validate_backup_zip(zf: zipfile.ZipFile) -> tuple[bool, str]:
  333. """Check that a zip looks like a Hermes backup.
  334. Returns (ok, reason).
  335. """
  336. names = zf.namelist()
  337. if not names:
  338. return False, "zip archive is empty"
  339. # Look for telltale files that a hermes home would have
  340. markers = {"config.yaml", ".env", "state.db"}
  341. found = set()
  342. for n in names:
  343. # Could be at the root or one level deep (if someone zipped the directory)
  344. basename = Path(n).name
  345. if basename in markers:
  346. found.add(basename)
  347. if not found:
  348. return False, (
  349. "zip does not appear to be a Hermes backup "
  350. "(no config.yaml, .env, or state databases found)"
  351. )
  352. return True, ""
  353. def _detect_prefix(zf: zipfile.ZipFile) -> str:
  354. """Detect if the zip has a common directory prefix wrapping all entries.
  355. Some tools zip as `.hermes/config.yaml` instead of `config.yaml`.
  356. Returns the prefix to strip (empty string if none).
  357. """
  358. names = [n for n in zf.namelist() if not n.endswith("/")]
  359. if not names:
  360. return ""
  361. # Find common prefix
  362. parts_list = [Path(n).parts for n in names]
  363. # Check if all entries share a common first directory
  364. first_parts = {p[0] for p in parts_list if len(p) > 1}
  365. if len(first_parts) == 1:
  366. prefix = first_parts.pop()
  367. # Only strip if it looks like a hermes dir name
  368. if prefix in {".hermes", "hermes"}:
  369. return prefix + "/"
  370. return ""
  371. def _default_new_file_mode() -> Optional[int]:
  372. """Return the mode ``open(path, "wb")`` gives a file it has to create.
  373. ``tempfile.mkstemp`` always creates at 0600, so staging an import through a
  374. temp file would tighten every *newly created* file to owner-only — the same
  375. hazard ``utils._restore_file_mode`` documents for Docker/NAS volume mounts
  376. that rely on broader permissions. The umask can only be read by setting it,
  377. so this is resolved once per import rather than once per member. The probe
  378. installs a *restrictive* mask rather than 0 so that anything another thread
  379. creates inside the two-syscall window is owner-only, never world-writable.
  380. Returns ``None`` if the umask cannot be read, in which case the caller
  381. leaves mkstemp's mode alone.
  382. """
  383. try:
  384. current = os.umask(0o077)
  385. os.umask(current)
  386. except OSError:
  387. return None
  388. return 0o666 & ~current
  389. def _extract_member_atomically(
  390. zf: zipfile.ZipFile,
  391. member: str,
  392. target: Path,
  393. new_file_mode: Optional[int] = None,
  394. ) -> None:
  395. """Restore one zip member onto *target* with no truncation window.
  396. ``open(target, "wb")`` truncates the user's existing file to zero *before*
  397. any replacement bytes exist. A Ctrl-C, an ENOSPC, a corrupt member, or a
  398. crash between the truncate and the write therefore leaves that file empty
  399. with nothing behind it — during ``hermes import``, which is the
  400. disaster-recovery path a user reaches for *because* they already lost
  401. something. Staging into the target's own directory and publishing with a
  402. rename means the target only ever moves from its old contents to the
  403. complete new contents.
  404. ``atomic_replace`` rather than a bare ``os.replace``: it resolves a
  405. symlinked target first, so a deployment that links ``config.yaml`` into a
  406. dotfiles repo keeps the link instead of having it silently swapped for a
  407. regular file (GitHub #16743), and it falls back to copy/fsync/unlink on
  408. ``EXDEV``/``EBUSY`` for cross-device and bind-mount installs. That
  409. fallback uses ``shutil.copyfile``, which does truncate in place, so on the
  410. cross-device path the guarantee above degrades to today's behaviour rather
  411. than improving on it; closing that belongs in ``utils.atomic_replace``,
  412. where every atomic writer in the repo would benefit, not here.
  413. Permission bits *and* ownership are carried across the replace so routing
  414. through mkstemp does not change the file the caller would otherwise have
  415. produced. ``os.replace`` swaps in a temp file owned by the *writing* user,
  416. so without the chown a ``sudo hermes import`` would silently re-own every
  417. restored file to root — on the disaster-recovery path, and on exactly the
  418. Docker/NAS installs ``utils._restore_file_owner`` documents. Both concerns
  419. delegate to the shared ``utils`` helpers rather than being re-derived here.
  420. The temp file is removed on any failure so a partial import leaves no
  421. residue.
  422. The one bit of the old file *not* carried across is setuid/setgid. The
  423. replacement bytes come out of the zip, so preserving those would let an
  424. archive take over the identity an existing privileged file executes as —
  425. and unlike the other ``utils`` writers, which re-serialize content this
  426. process produced, the trust boundary here is an untrusted archive. The
  427. mask is applied once, before the temp file is chmod'd, so neither the
  428. pre-replace ``fchmod`` nor the post-replace restore can re-elevate the
  429. target.
  430. """
  431. # ``_preserve_file_mode`` returns None when the target does not exist (or
  432. # cannot be stat'd), in which case the umask-derived create-mode applies —
  433. # the same shape as ``atomic_yaml_write``'s ``create_mode`` fallback.
  434. mode = _preserve_file_mode(target)
  435. owner = _preserve_file_owner(target)
  436. if mode is None:
  437. mode = new_file_mode
  438. else:
  439. # Deliberately NOT a faithful mode copy: setuid/setgid are dropped.
  440. # ``_preserve_file_mode`` returns ``stat.S_IMODE``, i.e. all twelve
  441. # bits, and the content replacing this file comes from the archive.
  442. # Carrying the elevated bits across would let archive-controlled bytes
  443. # take over an existing setuid/setgid file, so ``hermes import`` would
  444. # hand whoever produced the zip the identity that file runs as. Nothing
  445. # constrains that to Hermes' own state either: the ``_external/`` branch
  446. # of ``run_import`` publishes members anywhere under ``$HOME``. The
  447. # sticky bit is kept — it is inert on a regular file.
  448. mode &= ~(stat.S_ISUID | stat.S_ISGID)
  449. # Truncate the stem: mkstemp adds ~16 characters, and a member already near
  450. # NAME_MAX would otherwise fail here on a write that used to succeed.
  451. fd, tmp_name = mkstemp_beside(target, prefix=f".{target.name[:80]}.", suffix=".partial")
  452. try:
  453. with os.fdopen(fd, "wb") as dst:
  454. if mode is not None:
  455. # Apply the mode to the temp file BEFORE the replace so the
  456. # target never transits through mkstemp's 0600, and so
  457. # ``atomic_replace``'s EXDEV/EBUSY ``shutil.copystat`` fallback
  458. # copies the intended bits rather than 0600. fchmod is
  459. # Unix-only; Windows takes the path-based chmod.
  460. if hasattr(os, "fchmod"):
  461. os.fchmod(dst.fileno(), mode)
  462. else:
  463. os.chmod(tmp_name, mode)
  464. # Stream instead of ``src.read()``: a multi-gigabyte state.db member
  465. # must not be held in memory in one piece.
  466. with zf.open(member) as src:
  467. shutil.copyfileobj(src, dst)
  468. dst.flush()
  469. os.fsync(dst.fileno())
  470. real_path = Path(atomic_replace(tmp_name, target))
  471. # Owner first, mode second — the ordering ``atomic_yaml_write`` uses,
  472. # because chown drops setuid/setgid and a mode restore that ran first
  473. # would be partly undone. Here ``mode`` no longer carries those bits,
  474. # so the two agree: neither step can re-elevate the restored file.
  475. _restore_file_owner(real_path, owner)
  476. _restore_file_mode(real_path, mode)
  477. except BaseException:
  478. try:
  479. os.unlink(tmp_name)
  480. except OSError:
  481. pass
  482. raise
  483. def _count_session_rows(path: Path) -> Optional[tuple[int, int]]:
  484. """Return ``(sessions, messages)`` stored in the session database *path*.
  485. Read-only and best effort. ``None`` means "unknown" — a missing file, a
  486. database that is not a Hermes session store, or one that cannot be read.
  487. Callers must never read ``None`` as "zero rows": acting on an unreadable
  488. database would mask the very loss this count exists to surface. Same
  489. contract as :func:`_count_cron_jobs`.
  490. """
  491. if not path.is_file():
  492. return None
  493. try:
  494. conn = sqlite3.connect(read_only_db_uri(path), uri=True)
  495. except sqlite3.Error:
  496. return None
  497. try:
  498. sessions = conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0]
  499. messages = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
  500. return int(sessions), int(messages)
  501. except (sqlite3.Error, TypeError, ValueError):
  502. return None
  503. finally:
  504. conn.close()
  505. def _import_db_member(
  506. zf: zipfile.ZipFile,
  507. member: str,
  508. target: Path,
  509. new_file_mode: Optional[int] = None,
  510. ) -> None:
  511. from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
  512. try:
  513. with exclusive_maintenance([target.absolute().parent, target.resolve().parent]):
  514. _import_db_member_exclusive(zf, member, target, new_file_mode)
  515. except OwnershipConflict as exc:
  516. raise OSError(str(exc)) from exc
  517. def _import_db_member_exclusive(
  518. zf: zipfile.ZipFile,
  519. member: str,
  520. target: Path,
  521. new_file_mode: Optional[int] = None,
  522. ) -> None:
  523. """Publish a SQLite ``.db`` member onto *target* without replacing its inode.
  524. ``_extract_member_atomically`` publishes with a rename. For an ordinary
  525. file that is the safest write available; for a live SQLite database it is
  526. the #65942 / #90950 corruption class. A gateway, dashboard, or WebUI
  527. process holding the database open keeps its descriptor on the now-unlinked
  528. inode: it goes on serving pre-import pages and writing sessions that no
  529. other process will ever see, and any sidecar WAL left beside the new file
  530. describes the database that was just unlinked. Nothing fails, so nothing
  531. is reported — the sessions simply are not there afterwards (issue #100960).
  532. ``hermes import`` is the disaster-recovery path, so that failure mode lands
  533. on users who have already lost something once. Route the member through
  534. the same ``_safe_restore_db`` page copy that ``/snapshot restore`` has used
  535. since #65942: the live inode is preserved, every open connection converges
  536. on the imported data, and the sidecars are handled there.
  537. Raises ``OSError`` when the database could not be replaced safely, so the
  538. caller reports a skipped file instead of counting a silent success.
  539. """
  540. if not target.exists():
  541. # "Missing" is not "unheld": a gateway or dashboard that had the database open when it
  542. # was unlinked still writes the deleted inode (the ``(deleted)`` fingerprint of #90950).
  543. # Publishing a fresh inode here re-creates the same split brain the branch below exists
  544. # to prevent, so refuse and name the holders instead (#110179).
  545. holders = _foreign_db_holder_pids(target)
  546. if holders:
  547. raise OSError(
  548. f"{target.name} was deleted but is still open in PID(s) "
  549. f"{', '.join(str(pid) for pid in sorted(holders))}; publishing a new file would "
  550. "leave them writing an invisible database. Stop those processes and re-run the import."
  551. )
  552. _extract_member_atomically(zf, member, target, new_file_mode)
  553. return
  554. # The database keeps its own mode/ownership: the bytes come from the
  555. # archive but the file does not, so the archive has no say in either.
  556. mode = _preserve_file_mode(target)
  557. owner = _preserve_file_owner(target)
  558. fd, tmp_name = tempfile.mkstemp(
  559. dir=str(target.parent), prefix=f".{target.name[:80]}.", suffix=".dbimport"
  560. )
  561. try:
  562. with os.fdopen(fd, "wb") as dst:
  563. # Stream: a multi-gigabyte state.db member must not be held in
  564. # memory in one piece.
  565. with zf.open(member) as src:
  566. shutil.copyfileobj(src, dst)
  567. dst.flush()
  568. os.fsync(dst.fileno())
  569. if not _safe_restore_db(Path(tmp_name), target):
  570. from hermes_cli.backup import verify_sqlite_integrity
  571. # Re-check only on failure so the user gets the real cause; the
  572. # detailed integrity message was already logged by _safe_restore_db.
  573. if not verify_sqlite_integrity(Path(tmp_name))["valid"]:
  574. raise OSError(
  575. "the archived database failed its integrity check; the existing "
  576. "database was left untouched."
  577. )
  578. raise OSError(
  579. "live-safe restore refused or failed; the existing database was "
  580. "left untouched. Stop the gateway/dashboard processes holding it "
  581. "open and re-run the import."
  582. )
  583. _restore_file_owner(target, owner)
  584. _restore_file_mode(target, mode)
  585. finally:
  586. try:
  587. os.unlink(tmp_name)
  588. except OSError:
  589. pass
  590. def _get_config_path_value(data: dict[str, Any], dotted: tuple[str, ...]) -> Any:
  591. node: Any = data
  592. for key in dotted:
  593. if not isinstance(node, dict):
  594. return None
  595. node = node.get(key)
  596. return node
  597. def _set_config_path_value(data: dict[str, Any], dotted: tuple[str, ...], value: Any) -> None:
  598. node = data
  599. for key in dotted[:-1]:
  600. child = node.get(key)
  601. if not isinstance(child, dict):
  602. child = {}
  603. node[key] = child
  604. node = child
  605. node[dotted[-1]] = value
  606. def _count_cron_jobs(path: Path) -> Optional[int]:
  607. """Return the number of cron jobs stored in ``path``.
  608. The canonical on-disk shape is ``{"jobs": [...]}`` (see ``cron/jobs.py``).
  609. A legacy bare-list shape (``[...]``) is also honoured.
  610. Returns:
  611. The job count for any *valid, readable* JSON document, or ``None`` if
  612. the file is missing or cannot be parsed. ``None`` means "unknown" —
  613. callers must not treat it as "zero jobs", because acting on an
  614. unreadable file could mask a real corruption the user needs to see.
  615. """
  616. if not path.is_file():
  617. return None
  618. try:
  619. # utf-8-sig: same dialect as cron/jobs.load_jobs — Windows editors
  620. # may leave a UTF-8 BOM that plain utf-8 json.load rejects. Without
  621. # it a BOM'd jobs.json counts as "unreadable" (None) and the
  622. # post-update cron-loss auto-restore safety net silently disables.
  623. with open(path, "r", encoding="utf-8-sig") as f:
  624. data = json.load(f)
  625. except (OSError, json.JSONDecodeError):
  626. return None
  627. if isinstance(data, dict):
  628. jobs = data.get("jobs", [])
  629. return len(jobs) if isinstance(jobs, list) else None
  630. if isinstance(data, list):
  631. return len(data)
  632. return None