hermes_state_wal.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669
  1. """SQLite journal-mode and PRAGMA policy for state.db (split from hermes_state).
  2. Patchable helpers are looked up as module globals at call time, so tests patch ``hermes_state_wal.<name>``.
  3. """
  4. from __future__ import annotations
  5. import contextlib
  6. import functools
  7. import logging
  8. import os
  9. import sqlite3
  10. import sys
  11. import threading
  12. import time
  13. from typing import Any, Dict, Optional
  14. from hermes_cli.sqlite_runtime import is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable
  15. from hermes_state_errors import is_sqlite_lock_error
  16. # Log-record parity with the origin module (caplog tests pin "hermes_state").
  17. logger = logging.getLogger("hermes_state")
  18. # WAL needs mmap shared memory + fcntl byte-range locks. Network filesystems (NFS, SMB/CIFS, some FUSE, WSL1) raise
  19. # ``locking protocol``; ZFS corrupts the -shm file under concurrent bursts (COW + mmap) -> ``disk I/O error``.
  20. # Either would silently break everything on state.db/kanban.db, so fall back to DELETE (readers block on writes).
  21. # "not authorized": some FUSE mounts block the WAL pragma outright.
  22. _WAL_INCOMPAT_MARKERS = ("locking protocol", "not authorized", "disk i/o error")
  23. # SQLite's default journal_size_limit is -1 (unlimited); see _apply_wal_size_limit.
  24. _WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
  25. # Once-per-process-per-db_label dedup sets (kanban_db.connect() runs on every kanban operation, so an undeduped
  26. # line would repeat per connection). Tests clear these via ``hermes_state_wal.<name>``.
  27. _wal_fallback_warned_paths: set[str] = set()
  28. _wal_fallback_warned_lock = threading.Lock()
  29. # Dedup for the both-pragmas-failed WARNING (WAL *and* DELETE rejected, e.g. APFS external SSDs under contention).
  30. _wal_delete_fallback_failed_paths: set[str] = set()
  31. _wal_delete_fallback_failed_lock = threading.Lock()
  32. # Dedup for the probe-unknown WARNING (on-disk journal mode unreadable, nothing touched).
  33. _wal_probe_unknown_paths: set[str] = set()
  34. _wal_probe_unknown_lock = threading.Lock()
  35. _wal_reset_bug_warned_paths: set[str] = set()
  36. _wal_reset_bug_warned_lock = threading.Lock()
  37. _delete_overridden_warned_paths: set[str] = set()
  38. _delete_overridden_warned_lock = threading.Lock()
  39. _journal_upgrade_warned_paths: set = set()
  40. _journal_upgrade_warned_lock = threading.Lock()
  41. _CANNOT_VERIFY_DELETE_MSG = ("could not verify journal mode before applying configured journal_mode=delete (database "
  42. "is locked — possible concurrent openers); refusing to downgrade a database this process "
  43. "does not exclusively own")
  44. def _mode_from_row(row) -> str:
  45. """Lower-cased mode from a ``PRAGMA journal_mode`` row, ``""`` if no row."""
  46. return str(row[0]).strip().lower() if row and row[0] is not None else ""
  47. def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]:
  48. """Read the journal mode from the DB header; ``None`` if undeterminable (new DB, or PRAGMA failed) ->
  49. callers take their fail-closed "refuse to downgrade" branch. ``disk i/o error`` can be transient on
  50. virtualized block devices (XFS on cloud hosts), so it is retried a few times first."""
  51. for _ in range(4):
  52. try:
  53. row = conn.execute("PRAGMA journal_mode").fetchone()
  54. except sqlite3.OperationalError as exc:
  55. if "disk i/o error" not in str(exc).lower():
  56. return None
  57. last_exc = exc
  58. time.sleep(0.05)
  59. continue
  60. mode = row[0] if row else None
  61. if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes
  62. try:
  63. mode = mode.decode("ascii")
  64. except UnicodeDecodeError:
  65. return None
  66. return str(mode).strip().lower() if mode is not None else None
  67. logger.debug("_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc)
  68. return None
  69. def _apply_wal_size_limit(conn: sqlite3.Connection) -> None:
  70. """Bound the WAL so it returns space after big transactions. With the default (-1) a checkpointed WAL is
  71. reused in place, never truncated, so ``state.db-wal`` keeps the high-water mark of the largest
  72. transaction ever (a 3 GB optimize left a 3 GB WAL). Best-effort: failure only costs disk slack."""
  73. try:
  74. conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
  75. except sqlite3.OperationalError as exc: # pragma: no cover - defensive
  76. logger.debug("journal_size_limit not applied: %s", exc)
  77. def _darwin_pragma(conn: sqlite3.Connection, pragma: str) -> None:
  78. """Best-effort PRAGMA on macOS only (no-op elsewhere, never raises)."""
  79. if sys.platform == "darwin":
  80. with contextlib.suppress(sqlite3.OperationalError):
  81. conn.execute(pragma)
  82. def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
  83. """Enable ``PRAGMA checkpoint_fullfsync`` on macOS. Apple's ``fsync(2)`` guarantees neither data-on-platter nor
  84. ordering, so without ``F_FULLFSYNC`` a launchd shutdown can turn a "durable" checkpoint into a malformed
  85. ``state.db``. Checkpoint boundaries only (~+0.1 ms/commit vs ~+4 ms for ``fullfsync=1``).
  86. During a launchd *system* shutdown/reboot the OS page cache is dropped (effectively a power-loss event
  87. for in-flight pages), so a WAL checkpoint whose ``fsync()`` "reported" durable may never have hit the
  88. platter — corrupting ``state.db`` with a malformed image. This is the trigger in issue #30636 ("SIGTERM
  89. during launchd shutdown under high load"), distinct from a plain in-session kill (which the page cache
  90. survives and SQLite recovers from).
  91. """
  92. _darwin_pragma(conn, "PRAGMA checkpoint_fullfsync=1")
  93. def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None:
  94. """Enforce ``PRAGMA synchronous=FULL`` on macOS: with NORMAL a WAL checkpoint
  95. racing process termination leaves half-written btree pages (``btreeInitPage
  96. error 11``). Called after every WAL activation so a prior NORMAL never sticks."""
  97. _darwin_pragma(conn, "PRAGMA synchronous=FULL")
  98. def _apply_wal_companions(conn: sqlite3.Connection) -> None:
  99. """The settings every WAL activation carries: size limit + macOS barriers."""
  100. _apply_wal_size_limit(conn)
  101. _apply_macos_checkpoint_barrier(conn)
  102. _enforce_macos_synchronous_full(conn)
  103. def is_sqlite_wal_reset_vulnerable(version_info: Optional[tuple] = None) -> bool:
  104. """True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2; fixed 3.51.3+, backports 3.50.7 /
  105. 3.44.6). Pre-WAL libraries are safe. https://sqlite.org/wal.html#walresetbug"""
  106. return _is_sqlite_wal_reset_vulnerable(sqlite3.sqlite_version_info if version_info is None else version_info)
  107. def sqlite_source_id() -> str:
  108. """Return ``sqlite_source_id()``, or an empty string when unavailable."""
  109. try:
  110. with contextlib.closing(sqlite3.connect(":memory:")) as conn:
  111. row = conn.execute("SELECT sqlite_source_id()").fetchone()
  112. except sqlite3.Error:
  113. return ""
  114. return str(row[0]) if row and row[0] is not None else ""
  115. def _database_has_content(conn: sqlite3.Connection) -> bool:
  116. """Whether the file already holds pages (existing vs brand-new DB); lock-free header read. Fail-quiet False: the
  117. only caller gates a warning on this and an unknown-answer warning would fire on every fresh database."""
  118. try:
  119. row = conn.execute("PRAGMA page_count").fetchone()
  120. return bool(row) and row[0] is not None and int(row[0]) > 0
  121. except (sqlite3.Error, TypeError, ValueError):
  122. return False
  123. def resolve_journal_mode() -> str:
  124. """The configured ``database.journal_mode`` (``wal`` default; ``delete`` for filesystems without WAL-safe
  125. durability: macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``."""
  126. try:
  127. from hermes_cli.config import load_config_readonly
  128. database = (load_config_readonly() or {}).get("database", {})
  129. raw = database.get("journal_mode", "wal") if isinstance(database, dict) else "wal"
  130. except Exception:
  131. return "wal"
  132. mode = raw.strip().lower() if isinstance(raw, str) else ""
  133. return mode if mode in ("wal", "delete") else "wal"
  134. class WalUnsupportedError(sqlite3.OperationalError):
  135. """Raised by :func:`apply_wal_with_fallback` under ``require_wal=True`` when
  136. the filesystem cannot provide WAL (SQLITE_PROTOCOL raised, or macOS-NFS silent
  137. refusal) or the on-disk mode cannot be verified (probe blocked by a concurrent
  138. opener). Subclasses ``OperationalError`` so DB-init handlers still catch it."""
  139. # Docker Desktop / OrbStack / Podman expose host bind mounts to the guest as ``fuse.virtiofs`` or ``9p``. WAL's
  140. # -shm file must be coherent shared memory for every opener; across the VM boundary it is not, and under write
  141. # pressure the failure is SILENT (zero-filled pages), so the reactive ``_WAL_INCOMPAT_MARKERS`` fallback in
  142. # ``_enable_wal`` never gets a signal — WAL must be refused before the pragma. Port of openclaw/openclaw#120597.
  143. # Detection is Linux ``/proc/self/mountinfo`` only (statvfs carries no fstype); elsewhere, or when the table is
  144. # unreadable, the answer is False and behaviour is unchanged. Nothing else (ext4/btrfs/xfs/zfs/tmpfs/overlay/nfs)
  145. # is ever flagged here — those keep the existing reactive paths.
  146. _CROSS_VM_FSTYPES = frozenset({"virtiofs", "fuse.virtiofs", "9p", "9p2000", "9p2000.l", "9p2000.u"})
  147. _cross_vm_fs_cache: dict[str, bool] = {} # per DB directory; kanban_db.connect() opens per operation
  148. _cross_vm_fs_cache_lock = threading.Lock()
  149. _cross_vm_warned_paths: set[str] = set()
  150. _cross_vm_warned_lock = threading.Lock()
  151. _cross_vm_existing_wal_warned_paths: set[str] = set()
  152. _cross_vm_existing_wal_warned_lock = threading.Lock()
  153. def _mountinfo_fstype(directory: str, mountinfo_path: str = "/proc/self/mountinfo") -> str:
  154. """fstype of the longest mount point that is a prefix of ``directory`` (``""`` if unreadable / no match)."""
  155. try:
  156. with open(mountinfo_path, "r", encoding="utf-8", errors="replace") as fh:
  157. lines = fh.read().splitlines()
  158. except OSError:
  159. return ""
  160. best_len, best_fstype = -1, ""
  161. for line in lines:
  162. # proc(5): <id> <parent> <maj:min> <root> <mount point> <opts> [optional...] - <fstype> <source> <super opts>
  163. fields, _, tail = line.partition(" - ")
  164. parts = fields.split()
  165. if len(parts) < 5 or not tail:
  166. continue
  167. mount_point = parts[4]
  168. if "\\" in mount_point: # octal escapes (\040 = space)
  169. mount_point = mount_point.encode("latin-1", "ignore").decode("unicode_escape")
  170. if directory == mount_point or directory.startswith(mount_point.rstrip("/") + "/"):
  171. if len(mount_point) > best_len:
  172. best_len, best_fstype = len(mount_point), tail.split()[0]
  173. return best_fstype.lower()
  174. def _detect_cross_vm_fs(directory: str, mountinfo_path: str = "/proc/self/mountinfo") -> bool:
  175. """True only when ``directory`` sits on a virtiofs/9p mount per ``mountinfo_path``."""
  176. if sys.platform != "linux":
  177. return False
  178. return _mountinfo_fstype(directory, mountinfo_path) in _CROSS_VM_FSTYPES
  179. def _path_on_cross_vm_fs(path: str) -> bool:
  180. """True when ``path`` resides on a virtiofs/9p (cross-VM) filesystem; cached per resolved directory."""
  181. try:
  182. directory = os.path.dirname(os.path.realpath(path)) or "/"
  183. except (OSError, ValueError):
  184. return False
  185. with _cross_vm_fs_cache_lock:
  186. cached = _cross_vm_fs_cache.get(directory)
  187. if cached is None:
  188. cached = _detect_cross_vm_fs(directory)
  189. with _cross_vm_fs_cache_lock:
  190. _cross_vm_fs_cache[directory] = cached
  191. return cached
  192. def _connection_db_file(conn: sqlite3.Connection) -> str:
  193. """Filesystem path of the ``main`` database, ``""`` for in-memory / unknown."""
  194. try:
  195. row = conn.execute("PRAGMA database_list").fetchone()
  196. return str(row[2]) if row and row[2] else ""
  197. except (sqlite3.OperationalError, IndexError, TypeError):
  198. return ""
  199. def _verify_configured_delete(actual: str) -> str:
  200. """Raise unless SQLite reported ``delete`` for an explicit operator request."""
  201. if actual != "delete":
  202. raise sqlite3.OperationalError(f"could not set configured journal_mode=delete (got {actual or 'no result'})")
  203. return actual
  204. def apply_wal_with_fallback(conn: sqlite3.Connection, *, db_label: str = "state.db", require_wal: bool = False) -> str:
  205. """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
  206. Returns the mode actually set — or, when the read-only mode probe is blocked by a concurrent opener, ``"wal"``
  207. as the assumed mode with nothing touched (``require_wal=True`` raises instead). Shared by :class:`SessionDB`
  208. and ``hermes_cli.kanban_db_connect.connect``. WAL-incompatible filesystems either raise ``OperationalError`` ("locking protocol" / "disk I/O error") or —
  209. macOS NFS / SMB / AgentFS — silently refuse and stay in DELETE; either way log ERROR once per process per
  210. ``db_label`` and fall back. ``require_wal=True`` raises :class:`WalUnsupportedError` instead. WAL-reset-bug
  211. builds (https://sqlite.org/wal.html#walresetbug) never enable WAL on non-WAL files; an already-WAL DB keeps WAL
  212. with a warning. Gate deliberately RETAINED: re-measured on the bundled 3.50.4 there is no evidence WAL is safer.
  213. Invariant on every path: never downgrade to DELETE if the on-disk header reports WAL or cannot be read — other
  214. gateway/cron/worker connections may hold the DB open, and a live downgrade destroys their uncheckpointed
  215. commits.
  216. An earlier revision of the lock-cancellation fix (#71724) reverted it on the theory that DELETE was "the
  217. mode that corrupts", but that comparison was confounded: the clean WAL result came from SQLite 3.53.1,
  218. which carries BOTH the WAL-reset fix AND 3.51.0's defenses against close()-broken POSIX locks, so it
  219. says nothing about 3.50.4. Re-measured on the actually-bundled 3.50.4 with the lock fix in place, WAL
  220. and DELETE are both clean (0/3 each) — i.e. there is no evidence that WAL is safer here, and upstream
  221. still documents the WAL-reset bug as real through 3.51.2 with serious consequences. Until a fixed
  222. runtime is delivered, keep new databases out of WAL.
  223. """
  224. config_module = sys.modules.get("hermes_cli.config")
  225. if getattr(getattr(config_module, "__spec__", None), "_initializing", False):
  226. config_module = None
  227. config_lock = getattr(config_module, "_CONFIG_LOCK", None)
  228. acquired = config_lock is not None and config_lock.acquire(blocking=False)
  229. try:
  230. if config_lock is not None and not acquired:
  231. # A config writer cannot hold up work on an initialized database.
  232. # Preserve its observed mode; defer configuration-driven transitions.
  233. current_mode = _on_disk_journal_mode(conn)
  234. if current_mode == "wal":
  235. if is_sqlite_wal_reset_vulnerable():
  236. return _apply_delete_for_wal_reset_bug(conn, db_label=db_label)
  237. _apply_wal_companions(conn)
  238. return "wal"
  239. if current_mode == "delete" and not require_wal and _database_has_content(conn):
  240. if is_sqlite_wal_reset_vulnerable():
  241. _log_wal_reset_bug_once(db_label, kept_wal=False)
  242. return "delete"
  243. configured = resolve_journal_mode()
  244. finally:
  245. if acquired:
  246. config_lock.release()
  247. # Vulnerable SQLite: never enable WAL on non-WAL files (configured mode resolved first so an explicit DELETE
  248. # request is still verified).
  249. if is_sqlite_wal_reset_vulnerable():
  250. return _apply_delete_for_wal_reset_bug(conn, db_label=db_label, require_delete=configured == "delete")
  251. # Read-only probe (no flock/checkpoint/WAL-SHM unlink): WAL-init must not unlink files other connections hold.
  252. current_mode = _on_disk_journal_mode(conn)
  253. if current_mode == "wal":
  254. if configured == "delete":
  255. # Never-live-downgrade keeps WAL; tell the operator their delete did not apply.
  256. _log_configured_delete_overridden_once(db_label)
  257. _warn_existing_wal_on_cross_vm_fs(conn)
  258. _apply_wal_companions(conn)
  259. return "wal"
  260. if configured == "delete":
  261. # #68545: honor the canonical database.journal_mode setting. Existing on-disk WAL databases were
  262. # returned above and are never live-downgraded.
  263. if current_mode is None:
  264. # Probe failed (locked/busy): ownership not provably exclusive. Fail loudly.
  265. raise sqlite3.OperationalError(_CANNOT_VERIFY_DELETE_MSG)
  266. return _verify_configured_delete(_set_journal_mode_no_wait(conn, "DELETE"))
  267. if current_mode is None:
  268. # Probe failed (locked/busy): ownership not provably exclusive, same as the DELETE branch above. A fresh
  269. # 0-page DB probes "delete" cleanly, so this is a real file some other connection holds — running WAL-init
  270. # would unlink its -wal/-shm sidecars. Touch nothing: the connection inherits whatever mode the header has.
  271. if require_wal:
  272. raise WalUnsupportedError("could not verify the on-disk journal mode (database is locked — possible "
  273. "concurrent openers); cannot guarantee WAL")
  274. _log_once("wal_probe_unknown", db_label)
  275. return "wal"
  276. # Cross-VM bind mount (virtiofs/9p): WAL corrupts silently there, so refuse to ENABLE it. On-disk WAL
  277. # databases were returned above (never live-downgrade); a 0-page / DELETE file just stays DELETE.
  278. db_file = _connection_db_file(conn)
  279. if db_file and _path_on_cross_vm_fs(db_file):
  280. if require_wal:
  281. raise WalUnsupportedError("journal_mode=WAL refused: database is on a cross-VM filesystem (virtiofs/9p) "
  282. "where WAL shared-memory silently corrupts")
  283. _log_once("cross_vm_fs", db_label)
  284. return _set_journal_mode_no_wait(conn, "DELETE") or "delete"
  285. return _enable_wal(conn, db_label, require_wal, current_mode)
  286. def _enable_wal(conn: sqlite3.Connection, db_label: str, require_wal: bool, current_mode: Optional[str]) -> str:
  287. """Flip a non-WAL, non-vulnerable connection to WAL, or fall back to DELETE."""
  288. # Decide BEFORE the flip whether it overwrites a mode somebody chose (probe and page_count are only readable
  289. # while the file is untouched). A 0-page DB has no prior choice, and every caller reaches this before schema.
  290. upgrading_existing_db = current_mode is not None and current_mode != "wal" and _database_has_content(conn)
  291. def _wal_activated() -> str:
  292. if upgrading_existing_db:
  293. _log_journal_mode_upgrade_once(db_label, current_mode)
  294. _apply_wal_companions(conn)
  295. return "wal"
  296. try:
  297. # ``PRAGMA journal_mode=WAL`` RETURNS the resulting mode: macOS NFS, SMB/CIFS and the AgentFS overlay
  298. # refuse WITHOUT raising. Trust the row, not the absence of an exception.
  299. mode = _mode_from_row(conn.execute("PRAGMA journal_mode=WAL").fetchone())
  300. if mode == "wal":
  301. return _wal_activated()
  302. silent_exc = WalUnsupportedError(f"journal_mode=WAL refused without raising (still {mode!r})")
  303. if require_wal:
  304. raise silent_exc
  305. _log_wal_fallback_once(db_label, silent_exc)
  306. return mode or "delete"
  307. except WalUnsupportedError:
  308. raise # the require_wal silent-refusal raise above — propagate unchanged
  309. except sqlite3.OperationalError as exc:
  310. msg = str(exc).lower()
  311. if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
  312. raise # unrelated OperationalError — don't silently swallow
  313. # ``disk i/o error`` is ambiguous: on ZFS / APFS-CoW it is a deterministic WAL-incompatibility (SHM
  314. # corruption under concurrent connection bursts — #55305, #71498), but it can also be a one-shot
  315. # transient EIO (page-cache pressure, brief lock contention). Treating a transient EIO as a
  316. # permanent downgrade signal produced the mixed-journal-mode corruption pattern fixed in 5c49cd0ed0
  317. # (process A downgrades to DELETE while sibling processes set WAL). Disambiguate by retrying the
  318. # pragma a couple of times: transient EIO clears and we return "wal"; the deterministic filesystem
  319. # cases keep failing and fall through to the guarded DELETE fallback.
  320. if "disk i/o error" in msg:
  321. # Retry twice: EIO is either deterministic WAL-incompatibility (ZFS / APFS-CoW) or a one-shot transient,
  322. # and treating a transient as a permanent downgrade produced mixed-mode corruption (A downgrades to
  323. # DELETE while siblings set WAL). A non-EIO retry error propagates.
  324. for _ in range(2):
  325. time.sleep(0.05)
  326. try:
  327. row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
  328. except sqlite3.OperationalError as retry_exc:
  329. if "disk i/o error" not in str(retry_exc).lower():
  330. raise
  331. exc = retry_exc
  332. continue
  333. if _mode_from_row(row) == "wal":
  334. return _wal_activated()
  335. break
  336. # Never downgrade if WAL is on disk or the mode cannot be read (probe blocked by a concurrent opener) —
  337. # ownership is not provably exclusive either way.
  338. if _on_disk_journal_mode(conn) in ("wal", None):
  339. raise
  340. if require_wal:
  341. raise WalUnsupportedError(str(exc)) from exc
  342. _log_wal_fallback_once(db_label, exc)
  343. try:
  344. _set_journal_mode_no_wait(conn, "DELETE")
  345. except sqlite3.OperationalError as delete_exc:
  346. # Filesystems that reject BOTH pragmas (APFS external SSDs under heavy contention raise
  347. # "disk I/O error" on DELETE too). The connection keeps its current mode — typically the
  348. # SQLite default DELETE — and stays usable for reads/writes, so propagating would crash
  349. # every DB-init caller (SessionDB, kanban_db, ResponseStore) for no benefit. Log once
  350. # per db_label and return the mode actually in effect, read back rather than guessed.
  351. _log_once("wal_delete_fallback_failed", db_label, exc, delete_exc)
  352. try:
  353. read_back = _mode_from_row(conn.execute("PRAGMA journal_mode").fetchone())
  354. except sqlite3.OperationalError:
  355. read_back = ""
  356. return read_back or "delete"
  357. return "delete"
  358. def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str:
  359. """Execute ``PRAGMA journal_mode=<mode>`` without waiting on other openers.
  360. The ONLY place a non-WAL journal-mode switch may be issued. ``busy_timeout=0`` turns SQLite's exclusivity
  361. requirement into a concurrent-opener detector: leaving WAL needs exclusive access, so if ANY other connection
  362. holds the DB the pragma fails immediately with ``database is locked`` instead of sneaking the flip between a
  363. writer's transactions (how uncheckpointed WAL commits die). Callers must treat a raised ``OperationalError`` as
  364. "not exclusively owned: leave the mode alone", never as retryable. Returns the reported mode, ``""`` if no row."""
  365. try:
  366. row = conn.execute("PRAGMA busy_timeout").fetchone()
  367. previous_timeout = int(row[0]) if row and row[0] is not None else 0
  368. except (sqlite3.OperationalError, TypeError, ValueError):
  369. previous_timeout = 0
  370. conn.execute("PRAGMA busy_timeout=0")
  371. try:
  372. return _mode_from_row(conn.execute(f"PRAGMA journal_mode={mode}").fetchone())
  373. finally:
  374. with contextlib.suppress(sqlite3.OperationalError):
  375. conn.execute(f"PRAGMA busy_timeout={previous_timeout}")
  376. def _warn_existing_wal_on_cross_vm_fs(conn: sqlite3.Connection) -> None:
  377. """#110848: the fresh-DB cross-VM refusal never sees a DB that was already WAL before the check existed (or was
  378. created on a native volume and then moved). Keeping WAL is right (never live-downgrade); staying silent is not.
  379. Both the vulnerable-SQLite and the regular already-WAL paths return early, so both must call this."""
  380. db_file = _connection_db_file(conn)
  381. if db_file and _path_on_cross_vm_fs(db_file):
  382. # Keyed (and labelled) by path, not db_label: a gateway serving several profiles must hear about each one.
  383. _log_once("cross_vm_fs_existing_wal", db_file)
  384. def _apply_delete_for_wal_reset_bug(conn: sqlite3.Connection, *, db_label: str, require_delete: bool = False) -> str:
  385. """Avoid enabling WAL when the linked SQLite has the WAL-reset bug.
  386. Already-WAL on disk: keep WAL (no live downgrade) and warn. Mode unreadable (probe blocked by a concurrent
  387. opener): not provably exclusive — leave it and warn; treating "could not read" as "not WAL" once flipped a live
  388. WAL state.db to DELETE under a writer, destroying its uncheckpointed commits. Otherwise set DELETE without
  389. waiting out openers and warn; an explicit operator request additionally verifies SQLite accepted DELETE."""
  390. current = _on_disk_journal_mode(conn)
  391. if current == "wal":
  392. _log_wal_reset_bug_once(db_label, kept_wal=True)
  393. if require_delete:
  394. # Upgrading SQLite doesn't help here; emit the actionable message last.
  395. _log_configured_delete_overridden_once(db_label)
  396. _warn_existing_wal_on_cross_vm_fs(conn)
  397. _apply_wal_companions(conn)
  398. return "wal"
  399. if current is None:
  400. if require_delete:
  401. raise sqlite3.OperationalError(_CANNOT_VERIFY_DELETE_MSG)
  402. _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
  403. return "wal"
  404. actual = ""
  405. try:
  406. actual = _set_journal_mode_no_wait(conn, "DELETE")
  407. except sqlite3.OperationalError as exc:
  408. if require_delete:
  409. raise
  410. if is_sqlite_lock_error(exc):
  411. # A concurrent opener appeared between probe and flip: leave the mode as is.
  412. _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
  413. return current or "delete"
  414. # Best-effort otherwise: DELETE is already the default for new file-backed DBs.
  415. if require_delete:
  416. _verify_configured_delete(actual)
  417. _log_wal_reset_bug_once(db_label, kept_wal=False)
  418. return "delete"
  419. def _wal_reset_repair_hint() -> str:
  420. """Repair hint matching what ``hermes update`` can actually do for this install type.
  421. See #75153.
  422. """
  423. try:
  424. from hermes_cli.config import detect_install_method, get_project_root, recommended_update_command_for_method
  425. method = detect_install_method(get_project_root())
  426. cmd = recommended_update_command_for_method(method)
  427. if method in {"git", "unknown"}:
  428. return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`"
  429. return f"update the container image with `{cmd}`" if method == "docker" else cmd # else nix/nixos
  430. except Exception:
  431. return "install a Python build bundled with SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6) and restart Hermes"
  432. # Once-per-(process, db_label) log table. Levels are deliberate: falling back to DELETE and an ignored
  433. # ``journal_mode: delete`` are real losses (ERROR); a non-WAL -> WAL flip is normally desirable (WARNING).
  434. _WAL_RESET_BUG_ACTIONS = {
  435. "indeterminate": ("journal mode could not be verified or exclusively switched (database is locked — possible "
  436. "concurrent openers); leaving the journal mode untouched (no live downgrade under concurrent "
  437. "openers)"),
  438. "kept_wal": "is already in WAL mode — leaving WAL in place (no live downgrade under concurrent openers)",
  439. "delete": "using journal_mode=DELETE instead of enabling WAL",
  440. }
  441. _ONCE_LOGS = {
  442. "wal_reset_bug": (_wal_reset_bug_warned_lock, "_wal_reset_bug_warned_paths", logging.WARNING,
  443. # Install-type-aware so the warning never promises a repair path that doesn't exist for git/pip installs.
  444. "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset corruption bug "
  445. "(https://sqlite.org/wal.html#walresetbug) — %s. Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
  446. "%s. See `hermes doctor`. This warning fires once per process per database."),
  447. "journal_upgrade": (_journal_upgrade_warned_lock, "_journal_upgrade_warned_paths", logging.WARNING,
  448. # journal_mode is a property of the FILE: switching an existing DB to WAL rewrites its header and outlives
  449. # the process. Operators set DELETE on the file (the WAL-reset-bug mitigation) and nothing told them the
  450. # next open would silently put WAL back.
  451. "%s: on-disk journal_mode was %s and has been switched to WAL. This rewrites the database header and "
  452. "persists after this process exits. If %s was a deliberate choice (for example the mitigation for the SQLite "
  453. "WAL-reset bug, or a WAL-unsafe filesystem), setting it with PRAGMA on the file will not survive -- every "
  454. "open re-applies the configured mode. Set `database.journal_mode: delete` in config.yaml to make it stick. "
  455. "This message fires once per process per database."),
  456. "wal_fallback": (_wal_fallback_warned_lock, "_wal_fallback_warned_paths", logging.ERROR,
  457. # Under kanban dispatcher + workers a DELETE-mode write blocks readers as SQLITE_BUSY.
  458. "%s: WAL journal_mode unsupported on this filesystem (%s) — falling back to journal_mode=DELETE (slower "
  459. "rollback-journal mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
  460. "https://www.sqlite.org/wal.html for details. This message fires once per process per database."),
  461. "wal_delete_fallback_failed": (_wal_delete_fallback_failed_lock, "_wal_delete_fallback_failed_paths", logging.WARNING,
  462. # Both pragmas rejected (observed on APFS external SSDs under heavy contention): the connection keeps
  463. # whatever mode it has (typically the SQLite default DELETE) and stays usable for reads/writes, so this
  464. # is WARNING, not ERROR — propagating would crash every DB-init caller. Deduped per db_label because
  465. # kanban_db.connect() runs on every kanban operation.
  466. "%s: both WAL and DELETE journal_mode failed (WAL: %s; DELETE: %s) — continuing with the connection's "
  467. "current journal mode (reads/writes still work). Typically seen on APFS external SSDs under heavy "
  468. "contention. This message fires once per process per database."),
  469. "delete_overridden": (_delete_overridden_warned_lock, "_delete_overridden_warned_paths", logging.ERROR,
  470. # Never-live-downgrade keeps WAL; without this the operator never learns their delete had no effect.
  471. "%s: database.journal_mode=delete is configured but the on-disk database is already WAL; keeping WAL (a live "
  472. "downgrade under open connections can corrupt the DB). To apply journal_mode=DELETE, stop every Hermes "
  473. "process using this database and run `hermes sessions set-journal-mode delete` (add `--db PATH` for a "
  474. "store other than state.db). This message fires once per process per database."),
  475. "wal_probe_unknown": (_wal_probe_unknown_lock, "_wal_probe_unknown_paths", logging.WARNING,
  476. # WARNING, not ERROR: the connection inherits the header's mode, so an already-WAL file (the common case)
  477. # keeps working; only a true-DELETE file stays DELETE for this connection.
  478. "%s: could not verify the on-disk journal mode (database is locked / busy); not issuing a journal-mode "
  479. "set-pragma while another connection may hold the file (it could unlink the -wal/-shm sidecars that "
  480. "connection still uses). Leaving the file untouched; this connection inherits the on-disk mode. This message fires once per process per database."),
  481. "cross_vm_fs": (_cross_vm_warned_lock, "_cross_vm_warned_paths", logging.WARNING,
  482. # The DB keeps working in DELETE mode; the loss is concurrency, so WARNING with the actionable fix.
  483. "%s: database directory is on a cross-VM filesystem (virtiofs/9p — typical for Docker Desktop / OrbStack / "
  484. "Podman host bind mounts). SQLite WAL shared-memory is not coherent across the VM boundary and can silently "
  485. "corrupt the database, so journal_mode=DELETE is used instead. To restore WAL concurrency, move the database "
  486. "onto a native volume (e.g. a named Docker volume) instead of a host bind mount. This message fires once per "
  487. "process per database."),
  488. "cross_vm_fs_existing_wal": (_cross_vm_existing_wal_warned_lock, "_cross_vm_existing_wal_warned_paths", logging.ERROR,
  489. # ERROR, unlike the fresh-DB refusal: this database IS running WAL on the corrupting filesystem and only the
  490. # operator can fix it (a live downgrade under other openers would destroy their uncheckpointed commits).
  491. "%s: existing WAL-mode database is on a cross-VM filesystem (virtiofs/9p — typical for Docker Desktop / "
  492. "OrbStack / Podman host bind mounts). SQLite WAL shared-memory is not coherent across the VM boundary and "
  493. "concurrent writers can silently corrupt the database. Hermes does not live-downgrade an on-disk WAL database. "
  494. "Fix one of two ways: stop every Hermes process using this database and run `hermes sessions "
  495. "set-journal-mode delete` (set `database.journal_mode: delete` in config.yaml to keep it), "
  496. "or move the database onto a native volume (e.g. a named Docker volume). This message fires once per process "
  497. "per database."),
  498. }
  499. def _log_once(kind: str, db_label: str, *args: Any) -> None:
  500. """Emit ``_ONCE_LOGS[kind]`` once per (process, db_label). Callable *args* are
  501. resolved only after the dedupe check, so install-method probes run once."""
  502. lock, set_name, level, message = _ONCE_LOGS[kind]
  503. seen = globals()[set_name]
  504. with lock:
  505. if db_label in seen:
  506. return
  507. seen.add(db_label)
  508. logger.log(level, message, db_label, *(a() if callable(a) else a for a in args))
  509. def _log_wal_reset_bug_once(db_label: str, *, kept_wal: bool, indeterminate: bool = False) -> None:
  510. """Log once per (process, db_label) about the WAL-reset vulnerability path."""
  511. action = _WAL_RESET_BUG_ACTIONS["indeterminate" if indeterminate else "kept_wal" if kept_wal else "delete"]
  512. _log_once("wal_reset_bug", db_label, sqlite3.sqlite_version, sys.executable, action, _wal_reset_repair_hint)
  513. def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None:
  514. """Single WARNING per (process, db_label) about a non-WAL -> WAL flip."""
  515. _log_once("journal_upgrade", db_label, previous_mode, previous_mode)
  516. # Single ERROR per (process, db_label): WAL fallback / configured delete ignored (DB already WAL).
  517. _log_wal_fallback_once = functools.partial(_log_once, "wal_fallback")
  518. _log_configured_delete_overridden_once = functools.partial(_log_once, "delete_overridden")
  519. # Operators write synchronous as a name; mapped so a typo becomes a warning, not a silently different level.
  520. _SYNCHRONOUS_LEVELS: dict[str, int] = {"OFF": 0, "NORMAL": 1, "FULL": 2, "EXTRA": 3}
  521. _SYNCHRONOUS_NAMES: dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()}
  522. _SYNCHRONOUS_FULL = 2
  523. def resolve_synchronous_level(raw_value: Any) -> Optional[int]:
  524. """Map ``database.synchronous`` (``OFF``/``NORMAL``/``FULL``/``EXTRA`` any
  525. case, or ``0``-``3``) to its PRAGMA integer; None for anything else so the
  526. caller warns and leaves the level untouched (guessing at durability is worse)."""
  527. if isinstance(raw_value, bool):
  528. # bool is an int subclass and YAML turns bare `on`/`off` into one; "off" is a real choice, True meaningless.
  529. return 0 if raw_value is False else None
  530. if isinstance(raw_value, int):
  531. return raw_value if raw_value in _SYNCHRONOUS_NAMES else None
  532. text = str(raw_value).strip()
  533. if text.upper() in _SYNCHRONOUS_LEVELS:
  534. return _SYNCHRONOUS_LEVELS[text.upper()]
  535. with contextlib.suppress(TypeError, ValueError):
  536. return int(text) if int(text) in _SYNCHRONOUS_NAMES else None
  537. return None
  538. def _apply_synchronous_pragma(conn: sqlite3.Connection, raw_value: Any, *, db_label: str) -> None:
  539. """Set ``PRAGMA synchronous`` from config, never below FULL on macOS.
  540. Kept out of the integer loop in :func:`apply_database_pragmas`: this PRAGMA decides whether a commit is on
  541. the platter, so an unrecognised value must not fall through to "SQLite default" the way a bad
  542. ``cache_size`` can. Darwin floor: this runs after :func:`_enforce_macos_synchronous_full`, so a configured
  543. ``NORMAL`` would silently undo the btree protection — raising is allowed, lowering is refused out loud."""
  544. if (level := resolve_synchronous_level(raw_value)) is None:
  545. logger.warning("%s: ignoring unrecognized database.synchronous=%r (expected OFF, NORMAL, FULL, EXTRA, or 0-3)",
  546. db_label, raw_value)
  547. return
  548. if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL:
  549. logger.warning("%s: refusing database.synchronous=%s on macOS; keeping FULL. Darwin's fsync() does not "
  550. "guarantee write ordering, so a lower level readmits the half-written btree pages FULL exists "
  551. "to prevent.", db_label, _SYNCHRONOUS_NAMES[level])
  552. return
  553. with contextlib.suppress(sqlite3.OperationalError):
  554. conn.execute(f"PRAGMA synchronous={level}")
  555. def apply_database_pragmas(conn: sqlite3.Connection, *, db_label: str = "state.db") -> None:
  556. """Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``.
  557. Journal mode is NOT handled here (owned by :func:`apply_wal_with_fallback`). ``database:`` keys: ``cache_size``
  558. (negative = KiB, positive = pages), ``mmap_size`` (bytes, 0 = off), ``temp_store`` (0-3), ``wal_autocheckpoint``
  559. (pages), ``journal_size_limit`` (bytes), ``synchronous`` (unset leaves the compile-time default, which differs
  560. between bundled/distro/Homebrew builds). Best-effort: failures are ignored so DB init never breaks on a
  561. malformed section. Applied to ALL connection types: writer, read_only, WAL readers."""
  562. try:
  563. from hermes_cli.config import cfg_get, load_config_readonly # local: avoids a circular import
  564. cfg = load_config_readonly()
  565. except Exception:
  566. return
  567. for pragma_name in ("cache_size", "mmap_size", "temp_store", "wal_autocheckpoint", "journal_size_limit"):
  568. if (raw_value := cfg_get(cfg, "database", pragma_name, default=None)) is None:
  569. continue
  570. try:
  571. value = int(str(raw_value).strip())
  572. except (TypeError, ValueError):
  573. logger.warning("%s: ignoring non-integer database.%s=%r", db_label, pragma_name, raw_value)
  574. continue
  575. with contextlib.suppress(sqlite3.OperationalError):
  576. conn.execute(f"PRAGMA {pragma_name}={value}")
  577. # Last: sizing pragmas cannot change durability (see _apply_synchronous_pragma).
  578. if (raw_synchronous := cfg_get(cfg, "database", "synchronous", default=None)) is not None:
  579. _apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label)