backup_sqlite.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. """WAL-safe SQLite snapshots. Direct execution needs only the standard library.
  2. Desktop invokes this file before stopping its backend, even when application
  3. imports cannot load. Full and quick backups use the same SQLite copy operation.
  4. """
  5. import json
  6. import logging
  7. import os
  8. import signal
  9. import sqlite3
  10. import sys
  11. import threading
  12. import time
  13. from contextlib import suppress
  14. from datetime import datetime, timezone, UTC
  15. from pathlib import Path
  16. from typing import Optional
  17. logger = logging.getLogger(__name__)
  18. class _SQLiteBackupTimeout(RuntimeError):
  19. """Raised when a SQLite snapshot remains busy past its deadline."""
  20. def _close_quietly(conn: Optional[sqlite3.Connection]) -> None:
  21. if conn is not None:
  22. with suppress(Exception):
  23. conn.close()
  24. class _SQLiteBackupRestarted(RuntimeError):
  25. """Raised when a foreign write restarted a stepped snapshot from page one."""
  26. # A rollback-journal reader blocks every writer's commit while it holds the read
  27. # lock, so that copy releases it after each step; a waiting writer's PENDING lock
  28. # refuses the next step, so it waits for one step at most. Activity writes give
  29. # up after 0.5s (SessionDB._ACTIVITY_WRITE_PATIENCE_S); every hold stays under it.
  30. _ROLLBACK_STEP_SECONDS = 0.1
  31. _ROLLBACK_FIRST_STEP_PAGES = 256
  32. _ROLLBACK_BUSY_SLEEP_SECONDS = 0.01
  33. # Each foreign write restarts a stepped copy, so a continuous writer gets a bounded
  34. # number of attempts, then one whole-copy hold if it fits the budget, else a refusal.
  35. _ROLLBACK_MAX_RESTARTS = 20
  36. _ROLLBACK_LOCK_HOLD_SECONDS = 0.25
  37. def _uses_wal(src: Path) -> bool:
  38. # Header byte 18 (file format write version) is 2 for WAL; reading it takes no lock.
  39. with open(src, "rb") as handle:
  40. header = handle.read(19)
  41. return len(header) == 19 and header[18] == 2
  42. class _CopyRate:
  43. def __init__(self) -> None:
  44. self.seconds = 0.0
  45. self.pages = 0
  46. self.total = 0
  47. def pages_per_second(self) -> float:
  48. return self.pages / max(self.seconds, 1e-6)
  49. def _stepped_attempt(conn: sqlite3.Connection, backup_conn: sqlite3.Connection, pages: int,
  50. check_busy, rate: _CopyRate) -> bool:
  51. """One stepped backup; False when a foreign write restarted it."""
  52. last_remaining: Optional[int] = None
  53. last_tick, last_busy = time.monotonic(), False
  54. def _progress(status: int, remaining: int, total: int) -> None:
  55. nonlocal last_remaining, last_tick, last_busy
  56. check_busy(status, remaining, total)
  57. now = time.monotonic()
  58. # Copy time only: drop the sleep sqlite3 takes after a busy step.
  59. step_seconds = max(0.0, now - last_tick - (_ROLLBACK_BUSY_SLEEP_SECONDS if last_busy else 0.0))
  60. last_tick, last_busy = now, status in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)
  61. if status not in (sqlite3.SQLITE_OK, sqlite3.SQLITE_DONE):
  62. return
  63. restarted = last_remaining is not None and remaining >= last_remaining
  64. rate.seconds += step_seconds
  65. rate.pages += total - remaining if last_remaining is None or restarted else last_remaining - remaining
  66. rate.total = total
  67. if restarted:
  68. raise _SQLiteBackupRestarted()
  69. last_remaining = remaining
  70. try:
  71. conn.backup(backup_conn, pages=pages, progress=_progress, sleep=_ROLLBACK_BUSY_SLEEP_SECONDS)
  72. except _SQLiteBackupRestarted:
  73. return False
  74. return True
  75. def _stepped_rollback_journal_copy(conn: sqlite3.Connection, backup_conn: sqlite3.Connection,
  76. check_busy) -> None:
  77. rate = _CopyRate()
  78. pages = _ROLLBACK_FIRST_STEP_PAGES
  79. for _attempt in range(_ROLLBACK_MAX_RESTARTS + 1):
  80. if _stepped_attempt(conn, backup_conn, pages, check_busy, rate):
  81. return
  82. pages = max(_ROLLBACK_FIRST_STEP_PAGES, int(rate.pages_per_second() * _ROLLBACK_STEP_SECONDS))
  83. # Floored: a run where no step completed has no measured rate, and the estimate must not divide by 0.
  84. expected_hold = rate.total / max(rate.pages_per_second(), 1.0)
  85. if expected_hold > _ROLLBACK_LOCK_HOLD_SECONDS:
  86. raise RuntimeError(
  87. "state.db uses a rollback journal and is written continuously; a consistent snapshot "
  88. f"would block those writes for about {expected_hold:.1f}s, so it was not taken. "
  89. "Retry when no agent turn is running.")
  90. conn.backup(backup_conn, pages=-1, progress=check_busy, sleep=_ROLLBACK_BUSY_SLEEP_SECONDS)
  91. class _OffThreadCopy:
  92. """One long sqlite3 call run off the main thread. Python runs signal handlers only on the main
  93. thread and only between bytecodes, so a one-step backup (a single C ``sqlite3_backup_step``,
  94. GIL released) would hold a SIGTERM until the whole copy finished; the main thread waits here in
  95. short joins instead, so the Desktop's pre-flight deadline interrupts the snapshot promptly."""
  96. def __init__(self, copy) -> None:
  97. self._copy, self._error = copy, None
  98. # health: allow HX012 -- stdlib-only file (the Desktop runs it when app imports cannot load); the copy reads no profile state
  99. self.thread = threading.Thread(target=self._run, name="sqlite-snapshot", daemon=True)
  100. def _run(self) -> None:
  101. try:
  102. self._copy()
  103. except BaseException as exc: # health: allow BLE001 -- handed to the waiting main thread, which re-raises it
  104. self._error = exc
  105. def run(self) -> None:
  106. self.thread.start()
  107. while self.thread.is_alive():
  108. self.thread.join(0.05)
  109. if self._error is not None:
  110. raise self._error
  111. def running(self) -> bool:
  112. return self.thread.is_alive()
  113. def _safe_copy_db(src: Path, dst: Path, *, timeout_seconds: float = 10.0) -> bool:
  114. """Copy a SQLite database with the backup() API (WAL-safe consistent snapshot).
  115. Fails closed when no consistent snapshot can be made: copying only the main file loses WAL data.
  116. """
  117. conn = backup_conn = None
  118. copier: Optional[_OffThreadCopy] = None
  119. try:
  120. # sqlite3.connect() creates a missing destination with the process
  121. # umask, which is commonly 0022 (0644). Snapshot databases contain
  122. # session and tool state, so create the inode owner-only before SQLite
  123. # writes its first byte. O_NOFOLLOW also refuses a planted symlink on
  124. # platforms that support it. Tighten an existing internal staging
  125. # file as well (NamedTemporaryFile callers already create it 0600).
  126. if os.name != "nt":
  127. open_flags = os.O_WRONLY | os.O_CREAT
  128. if hasattr(os, "O_NOFOLLOW"):
  129. open_flags |= os.O_NOFOLLOW
  130. secure_fd = os.open(dst, open_flags, 0o600)
  131. try:
  132. os.fchmod(secure_fd, 0o600)
  133. finally:
  134. os.close(secure_fd)
  135. # timeout=0.0 disables sqlite3's implicit busy wait so the progress callback owns the
  136. # full locked-source deadline instead of adding the default timeout before each callback.
  137. conn = sqlite3.connect(f"{src.resolve().as_uri()}?mode=ro", uri=True, timeout=0.0,
  138. check_same_thread=False)
  139. backup_conn = sqlite3.connect(str(dst), check_same_thread=False)
  140. busy_deadline = time.monotonic() + max(0.0, timeout_seconds)
  141. def _check_backup_progress(status: int, _remaining: int, _total: int) -> None:
  142. nonlocal busy_deadline
  143. now = time.monotonic()
  144. if status in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED):
  145. if now >= busy_deadline:
  146. raise _SQLiteBackupTimeout(f"database remained locked for {timeout_seconds:g} seconds")
  147. else:
  148. busy_deadline = now + max(0.0, timeout_seconds)
  149. if _uses_wal(src):
  150. # One step: SQLite restarts a multi-step backup whenever another connection writes the
  151. # source, and a WAL reader never blocks a writer however long the copy takes.
  152. # Run off the main thread so a SIGTERM is serviced during the copy (_OffThreadCopy).
  153. copier = _OffThreadCopy(lambda: conn.backup(
  154. backup_conn, pages=-1, progress=_check_backup_progress, sleep=0.1))
  155. copier.run()
  156. else:
  157. _stepped_rollback_journal_copy(conn, backup_conn, _check_backup_progress)
  158. return True
  159. except Exception as exc:
  160. logger.warning("SQLite safe copy failed for %s: %s", src, exc)
  161. # Windows won't remove the partial destination while SQLite still has it open.
  162. _close_quietly(backup_conn)
  163. backup_conn = None
  164. with suppress(OSError):
  165. dst.unlink(missing_ok=True)
  166. return False
  167. finally:
  168. # Interrupted mid-copy (SIGTERM -> SystemExit): the worker is still inside the C copy with
  169. # both connections, and close() would wait on their mutex until the copy ends. The process
  170. # is exiting; leave them to it, and the caller unlinks the staging file.
  171. if copier is None or not copier.running():
  172. _close_quietly(backup_conn)
  173. _close_quietly(conn)
  174. # Longer than the Desktop's 30-minute pre-flight cap: a staging file this old
  175. # has no live writer. A hard-killed pre-flight (Windows TerminateProcess)
  176. # cannot unlink its own staging copy, and each one is a full-size database.
  177. _ABANDONED_STAGING_SECONDS = 2 * 3600
  178. def _reclaim_abandoned_staging(home: Path, prefix: str) -> None:
  179. horizon = time.time() - _ABANDONED_STAGING_SECONDS
  180. for staged in home.glob(f"{prefix}*.partial*"): # + the copy's -journal
  181. with suppress(OSError):
  182. if staged.stat().st_mtime < horizon:
  183. staged.unlink()
  184. def preflight_state_db(home: Path) -> dict:
  185. """Publish an emergency snapshot; do not prune recovery files on failure."""
  186. source = home / "state.db"
  187. if not source.exists():
  188. return {"path": None, "message": "state.db not found (fresh install?)"}
  189. prefix = "state.db.pre-update-emergency-"
  190. _reclaim_abandoned_staging(home, prefix)
  191. stamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%fZ")
  192. destination = home / f"{prefix}{stamp}-{os.getpid()}.bak"
  193. # Named before it exists, so a SIGTERM at any point after creation reaches the
  194. # cleanup below; _safe_copy_db creates it owner-only and refuses a symlink.
  195. staged = home / f"{prefix}{stamp}-{os.getpid()}.partial"
  196. try:
  197. if not _safe_copy_db(source, staged):
  198. raise RuntimeError("SQLite safe copy failed; previous emergency snapshots were retained")
  199. connection = sqlite3.connect(str(staged))
  200. try:
  201. result = connection.execute("PRAGMA quick_check").fetchall()
  202. if result != [("ok",)]:
  203. raise RuntimeError(f"SQLite snapshot integrity check failed: {result}")
  204. finally:
  205. connection.close()
  206. size = staged.stat().st_size
  207. os.replace(staged, destination)
  208. finally:
  209. staged.unlink(missing_ok=True)
  210. # A copy stopped mid-write leaves the staging copy's rollback journal behind too.
  211. Path(f"{staged}-journal").unlink(missing_ok=True)
  212. for old in sorted(home.glob(f"{prefix}*.bak"), reverse=True)[2:]:
  213. try:
  214. old.unlink()
  215. except OSError as exc:
  216. logger.warning("Could not prune emergency snapshot %s: %s", old, exc)
  217. return {"path": str(destination), "bytes": size}
  218. if __name__ == "__main__":
  219. # The Desktop stops a pre-flight past its deadline with SIGTERM; exit through
  220. # the staging cleanup instead of dying with a full-size .partial left behind.
  221. signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))
  222. print(json.dumps(preflight_state_db(Path(sys.argv[1]))))