hermes_state_repair.py 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173
  1. """state.db repair, backup and writability preflight (split from hermes_state).
  2. Patchable helpers are looked up as module globals at call time, so tests patch ``hermes_state_repair.<name>``.
  3. """
  4. from __future__ import annotations
  5. import contextlib
  6. import datetime
  7. import hashlib
  8. import itertools
  9. import json
  10. import logging
  11. import os
  12. import re
  13. import shutil
  14. import sqlite3
  15. import stat
  16. import time
  17. from pathlib import Path
  18. from typing import Any, Dict, List, Optional, Tuple
  19. from hermes_constants import get_hermes_home
  20. from hermes_startup_watchdog import report_startup_progress
  21. from hermes_state_holders import read_only_db_uri
  22. from hermes_state_common import (
  23. _acquire_db_flock, _clear_lock_holder_record, _describe_lock_holder, _read_lock_holder_record,
  24. is_advisory_lock_contention,
  25. )
  26. # Log-record parity with the origin module (caplog tests pin "hermes_state").
  27. logger = logging.getLogger("hermes_state")
  28. _REPAIR_LOCK_POLL_SECONDS = 0.1
  29. # Snapshot copies are data transfer, not locking: bounded separately at 10 MiB/s (historical two-minute floor).
  30. _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024
  31. # ── Repair-loop bounding + dead-backup hygiene (#86747) ───────────────────── ``_claim_repair_attempt``
  32. # above is an in-memory set: it bounds the loop only WITHIN one process. A corruption class the strategies
  33. # cannot heal (b-tree page damage) failed repair on EVERY process start, and each pass took a fresh ~900MB
  34. # forensic backup — 105 attempts / 89GB of identical dead copies in the reporting install. Two persistent
  35. # bounds fix the class: * a sidecar attempt ledger (``<db>.repair-attempts.json``) that refuses further
  36. # surgery after ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` failures on the SAME damaged file (fingerprint = size +
  37. # a bounded content sample; any successful repair or replacement changes it and resets the count); * backup
  38. # dedupe + a retention cap in ``_backup_db_file`` — an identical damaged file is never copied twice, and
  39. # only the newest ``_MAX_MALFORMED_BACKUPS`` forensic copies are kept.
  40. _MAX_PERSISTENT_REPAIR_ATTEMPTS = 3
  41. _MAX_MALFORMED_BACKUPS = 3
  42. # Sidecars copied with a damaged DB and pruned with it. ``-journal``: DELETE mode (the NFS/SMB/FUSE/ZFS and
  43. # WAL-reset-bug fallback) leaves a hot journal whenever a transaction was open; without it the copy cannot roll back.
  44. _DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal")
  45. # Head/tail bytes sampled by ``_db_fingerprint``: changes on any real repair/truncation/restore, O(1) on a multi-GB file.
  46. _FINGERPRINT_SAMPLE_BYTES = 65536
  47. # Header ranges that move on ordinary commits, not on repair — file change counter (24-27), version-valid-for
  48. # (92-95) — masked out of the sample. A malformed-SCHEMA DB still accepts writes and DELETE mode writes the main
  49. # file directly, so without the mask any live write re-keys the ledger and the repair budget resets to 1
  50. # forever. The page-1 sqlite_master b-tree (repair identity) sits after byte 100.
  51. _FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96))
  52. # Headroom for the forensic backup (a full raw copy; a repair loop on a large state.db is a disk amplifier).
  53. # Proportional, not a flat multi-GB floor: a refused backup is a HARD STOP, and a big reserve would make repair
  54. # never run on small volumes.
  55. # The backup is a full raw copy of the damaged DB (plus its -wal/-shm sidecars), so a repair loop on a large
  56. # state.db is a disk amplifier: the reporting incident wrote ~98MB every ~10s until the volume was nearly
  57. # full, which would have taken down every agent on the host. Require the copy itself plus a small slice of
  58. # the volume, clamped to a modest floor. See #69603.
  59. _REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor
  60. _REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume
  61. _FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
  62. _MANUAL_RECOVER_HINT = ("Free disk space, then retry (or recover manually with "
  63. "`hermes sessions recover --source {db_path} --inspect-only` first).")
  64. def _sidecars(db_path: Path):
  65. """The three SQLite sidecar paths for *db_path* (present or not)."""
  66. return (db_path.with_name(db_path.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES)
  67. def _read_offline(db_path: Path, what: str, reader) -> Optional[str]:
  68. """Run *reader()* under ``hermes_cli.sqlite_safe_read.offline_file_access``.
  69. ``close()`` on ANY raw descriptor cancels every POSIX advisory lock this process holds on the file,
  70. including a peer connection's RESERVED lock (``sqlite_safe_read`` rule 1), so a raw read is only safe with
  71. no live connection; ``None`` when that makes it unsafe or the file is unreadable. Scaffold/embed installs
  72. without hermes_cli have no tracked connections."""
  73. try:
  74. from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access
  75. except ImportError:
  76. offline_file_access, LiveConnectionError = (lambda _p, **_k: contextlib.nullcontext()), OSError
  77. try:
  78. with offline_file_access(db_path, what=what):
  79. return reader()
  80. except (LiveConnectionError, OSError):
  81. return None
  82. def _claim_repair_attempt(db_path: Path) -> bool:
  83. """Claim the one-shot per-process repair attempt for *db_path*: True for the first caller, False
  84. afterwards (bounds the repair/reopen loop and stops concurrent callers racing surgery on one file)."""
  85. from hermes_state import _repair_attempt_lock, _repair_attempted_paths
  86. with _repair_attempt_lock:
  87. if str(db_path) in _repair_attempted_paths:
  88. return False
  89. _repair_attempted_paths.add(str(db_path))
  90. return True
  91. def _open_lock_file(db_path: Path, suffix: str, what: str, tail: str):
  92. """Open ``<db>.<suffix>`` for locking; on failure warn and return None."""
  93. lock_path = db_path.with_name(db_path.name + suffix)
  94. try:
  95. lock_path.parent.mkdir(parents=True, exist_ok=True)
  96. return lock_path, lock_path.open("a+b")
  97. except OSError as exc:
  98. logger.warning(f"Could not open state.db {what} lock %s (%s) — {tail}", lock_path, exc)
  99. return lock_path, None
  100. def _msvcrt_lock(handle, flag_name: str) -> None:
  101. import msvcrt
  102. handle.seek(0)
  103. msvcrt.locking(handle.fileno(), getattr(msvcrt, flag_name), 1) # type: ignore[attr-defined]
  104. def _try_lock_nonblocking(handle) -> None:
  105. """Take the advisory lock on *handle* without waiting (raises on contention)."""
  106. from hermes_state import _IS_WINDOWS
  107. if _IS_WINDOWS:
  108. _msvcrt_lock(handle, "LK_NBLCK")
  109. else:
  110. import fcntl
  111. fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  112. def _release_lock_handle(handle, *, clear_record: bool = False) -> None:
  113. """Drop the advisory lock on *handle* (best effort) and close it."""
  114. from hermes_state import _IS_WINDOWS
  115. with contextlib.closing(handle), contextlib.suppress(OSError): # best-effort release; always close
  116. if _IS_WINDOWS:
  117. _msvcrt_lock(handle, "LK_UNLCK")
  118. else:
  119. import fcntl
  120. if clear_record:
  121. _clear_lock_holder_record(handle)
  122. fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
  123. def _acquire_repair_lock_windows(lock_path: Path, handle, timeout: float):
  124. """Windows counterpart of ``_acquire_db_flock``: True / False (timed out) / None (non-contention error)."""
  125. deadline = time.monotonic() + timeout
  126. while True:
  127. try:
  128. _try_lock_nonblocking(handle)
  129. return True
  130. except (BlockingIOError, OSError) as exc:
  131. if not is_advisory_lock_contention(exc):
  132. logger.warning("Could not acquire state.db repair lock %s (%s) — skipping schema surgery on a "
  133. "non-contention error.", lock_path, exc)
  134. return None
  135. if time.monotonic() >= deadline:
  136. return False
  137. time.sleep(_REPAIR_LOCK_POLL_SECONDS)
  138. @contextlib.contextmanager
  139. def _cross_process_repair_lock(db_path: Path):
  140. """Serialize state.db schema surgery across processes.
  141. Yields True when this process holds the repair lock, False when the bounded acquire timed out or the lock
  142. file could not be opened; on False the caller must NOT do surgery (unlocked surgery IS the interleaving
  143. this prevents). ``flock``: the kernel drops it when the holder dies (a pidfile would wedge every future
  144. repair); a forked child inheriting the fd is the exception, so the acquire records pid + start time and
  145. breaks a provably dead holder's lock. Bounded because a live repairer can sit in ``VACUUM`` for minutes.
  146. An unopenable lock file (no space/inodes/descriptors) fails closed too: a sibling that opened ITS handle
  147. before the disk filled may be inside surgery.
  148. The acquire is still bounded because a *live* repairer can legitimately sit in ``VACUUM`` for minutes on
  149. a large DB, and an unbounded wait would hang the caller's open with no traceback (the failure shape of
  150. #36644).
  151. """
  152. from hermes_state import _IS_WINDOWS, _REPAIR_LOCK_TIMEOUT_SECONDS
  153. lock_path, handle = _open_lock_file(
  154. db_path, ".repair.lock", "repair", "skipping schema surgery rather than running it without cross-process authority.")
  155. if handle is None:
  156. yield False
  157. return
  158. acquired = False
  159. try:
  160. if _IS_WINDOWS:
  161. acquired = _acquire_repair_lock_windows(lock_path, handle, _REPAIR_LOCK_TIMEOUT_SECONDS)
  162. else:
  163. acquired, handle = _acquire_db_flock(str(lock_path), handle, _REPAIR_LOCK_TIMEOUT_SECONDS,
  164. _REPAIR_LOCK_POLL_SECONDS, "state.db repair lock")
  165. if acquired is None:
  166. acquired = False # non-contention failure already logged with its errno
  167. elif not acquired:
  168. logger.warning("state.db repair lock %s held by another process for more than %.0fs — skipping schema "
  169. "surgery in this process to avoid racing the repairer. Recorded holder: %s.",
  170. lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS,
  171. _describe_lock_holder(None if _IS_WINDOWS else _read_lock_holder_record(handle)))
  172. yield acquired
  173. finally:
  174. if acquired:
  175. _release_lock_handle(handle, clear_record=True)
  176. else:
  177. handle.close()
  178. def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]:
  179. """Non-blocking cross-process lock for one auto-maintenance pass (None = skip the pass: otherwise two startups
  180. both pass the interval check and the second prunes a row the first has only just closed recoverably)."""
  181. _lock_path, handle = _open_lock_file(db_path, ".auto-maintenance.lock", "auto-maintenance", "skipping automatic maintenance.")
  182. try:
  183. if handle is not None:
  184. _try_lock_nonblocking(handle)
  185. return handle
  186. except (BlockingIOError, OSError):
  187. handle.close()
  188. return None
  189. _release_auto_maintenance_lock = _release_lock_handle # release a _try_acquire_auto_maintenance_lock handle
  190. def _bump_schema_cookie(conn: sqlite3.Connection) -> None:
  191. """Increment the schema cookie after direct ``sqlite_master`` surgery.
  192. Ordinary DDL bumps it and peers compare it before running a prepared statement (how they discard a cached
  193. schema); ``writable_schema=ON`` edits do NOT, so live connections would keep firing triggers into
  194. ``messages_fts*`` shadow tables that no longer exist. Best-effort, never raises."""
  195. try:
  196. current = conn.execute("PRAGMA schema_version").fetchone()[0]
  197. # Wrap within SQLite's 32-bit signed range; peers compare for equality.
  198. conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}")
  199. except (sqlite3.DatabaseError, TypeError, IndexError) as exc:
  200. logger.warning("Could not bump state.db schema cookie: %s", exc)
  201. def _mask_volatile_header(head: bytes) -> bytes:
  202. """Zero the commit-counter fields so ordinary writes don't re-key the ledger."""
  203. if len(head) < 96:
  204. return head
  205. buf = bytearray(head)
  206. for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES:
  207. buf[start:end] = b"\x00" * (end - start)
  208. return bytes(buf)
  209. def _repair_backup_headroom_bytes(total_bytes: int) -> int:
  210. """Free space required *beyond* the copy itself, for a volume of *total_bytes*."""
  211. return max(_REPAIR_BACKUP_MIN_FREE_BYTES, int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION))
  212. def _disk_budget(db_path: Path, refusal: str) -> tuple[Optional[str], int, int, int]:
  213. """``(error, bundle_bytes, free_bytes, headroom_bytes)`` for *db_path*'s volume (main file plus every PRESENT
  214. sidecar); *error* is set (and the sizes zero) on stat()/disk_usage() failure. Fails CLOSED: the nearly-full
  215. volume these guards exist for is exactly where they are most likely to fail."""
  216. try:
  217. need = db_path.stat().st_size + sum(p.stat().st_size for p in _sidecars(db_path) if p.exists())
  218. usage = shutil.disk_usage(db_path.parent)
  219. except OSError as exc:
  220. return (f"could not determine free space on {db_path.parent} ({exc}); refusing the {refusal} rather than risk "
  221. "filling the volume"), 0, 0, 0
  222. return None, need, usage.free, _repair_backup_headroom_bytes(usage.total)
  223. def _repair_scratch_space_error(db_path: Path) -> Optional[str]:
  224. """Return an error unless snapshot, VACUUM and promotion can fit safely."""
  225. error, snapshot_bytes, free, headroom = _disk_budget(db_path, "repair snapshot")
  226. if error is not None:
  227. return error
  228. # VACUUM on the staged DB may need up to 2x the database size (SQLite docs); the same reserve then covers
  229. # transactional promotion into the live DB.
  230. if free >= snapshot_bytes + (2 * snapshot_bytes) + headroom:
  231. return None
  232. return (f"only {free / 1e9:.2f}GB free on {db_path.parent}; the repair snapshot needs up to "
  233. f"{snapshot_bytes / 1e9:.2f}GB, VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and "
  234. f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, then retry.")
  235. def _backup_free_space_error(db_path: Path) -> Optional[str]:
  236. """Disk guard for the forensic copy: reason to refuse, or None. A full raw copy on a nearly-full volume (which a
  237. preceding repair loop may itself have caused) can finish off the disk and every process on the machine."""
  238. hint = _MANUAL_RECOVER_HINT.format(db_path=db_path)
  239. error, need, free, headroom = _disk_budget(db_path, "forensic copy")
  240. if error is not None:
  241. return f"{error}. {hint}"
  242. if free - need >= headroom:
  243. return None
  244. return (f"only {free / 1e9:.2f}GB free on {db_path.parent}; copying the damaged DB needs {need / 1e9:.2f}GB and must "
  245. f"leave {headroom / 1e9:.2f}GB headroom. {hint}")
  246. def _repair_snapshot_timeout_seconds(source_path: Path) -> float:
  247. """Bound one SQLite snapshot by source size incl. sidecars (a WAL can hold committed rows not yet in the
  248. main file), so a healthy large-database copy is not cut off by the repair-lock timeout."""
  249. from hermes_state import _REPAIR_LOCK_TIMEOUT_SECONDS
  250. source_bytes = 0
  251. for candidate in (source_path, *_sidecars(source_path)):
  252. with contextlib.suppress(FileNotFoundError): # a sidecar may vanish mid-walk
  253. source_bytes += candidate.stat().st_size
  254. return max(_REPAIR_LOCK_TIMEOUT_SECONDS, source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND)
  255. def _repair_failure_consumes_attempt(exc: BaseException) -> bool:
  256. """Whether a pre-strategy SQLite failure proves deterministic corruption.
  257. Lock contention, timeouts, disk-full and I/O failures are environmental — a retry may succeed, so they
  258. must not burn the repair ledger. Only SQLite's corruption/image result codes prove deterministic damage."""
  259. if not isinstance(exc, sqlite3.DatabaseError):
  260. return False
  261. if isinstance(error_code := getattr(exc, "sqlite_errorcode", None), int):
  262. # Extended result codes keep the primary code in the low byte.
  263. return (error_code & 0xFF) in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB)
  264. # Older sqlite3 without result-code attributes: narrow message match only, never turning generic
  265. # "disk is full"/"readonly" into permanent failures.
  266. return any(m in str(exc).lower() for m in ("file is not a database", "database disk image is malformed"))
  267. def _repair_ledger_path(db_path: Path) -> Path:
  268. return db_path.with_name(db_path.name + ".repair-attempts.json")
  269. def _db_fingerprint(db_path: Path) -> Optional[str]:
  270. """Cheap identity for a damaged DB file: size + a bounded content sample.
  271. EXCLUDES mtime: a malformed-schema DB still accepts writes, so live writers, checkpoints and the
  272. strategies move mtime between passes; keyed on mtime every pass looked like a NEW file and the attempt
  273. counter reset forever. Hashing a multi-GB file per open is the cost this ledger avoids, so sample the
  274. head/tail slices any real repair, truncation or restore must change.
  275. ``None`` = identity unavailable (:func:`_read_offline`; a live peer is expected here since this runs
  276. BEFORE ``_backup_db_file``'s live-connection guard). Callers MUST NOT substitute a differently-shaped key:
  277. the ledger compares for equality, so alternating shapes never match and the unbounded loop returns."""
  278. def _sample() -> str:
  279. st = db_path.stat()
  280. with open(db_path, "rb") as fh:
  281. head = fh.read(_FINGERPRINT_SAMPLE_BYTES)
  282. fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES))
  283. tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) if st.st_size > _FINGERPRINT_SAMPLE_BYTES else b""
  284. return f"{st.st_size}:{hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32]}"
  285. return _read_offline(db_path, "fingerprint", _sample)
  286. def _backup_content_identity(db_path: Path) -> Optional[str]:
  287. """Recovery-image identity for forensic-backup dedupe: whole file + sidecars.
  288. A DIFFERENT relation from :func:`_db_fingerprint` (never conflate them): the fingerprint answers "same
  289. repair epoch?" from head+tail only, and a live writer can commit into an *interior* page while preserving
  290. size and both 64 KiB slices — reusing a backup on that basis hands the operator a snapshot predating real
  291. user data. A forensic copy must claim byte identity, so this digests the ENTIRE main file plus every
  292. present sidecar (the WAL can hold uncheckpointed frames). ``None`` when a live connection makes the read
  293. unsafe (:func:`_read_offline`) — the caller then takes a fresh backup, never a false reuse.
  294. See #87409.
  295. """
  296. def _digest() -> str:
  297. hasher = hashlib.sha256()
  298. members = [("main", db_path), *((sfx, p) for sfx, p in zip(_DB_SIDECAR_SUFFIXES, _sidecars(db_path)) if p.exists())]
  299. for label, path in members:
  300. # Length-delimit every member (main file included) so the concatenation is prefix-free; otherwise a
  301. # main-file tail could coincide with a main+sidecar split and dedupe two images together.
  302. hasher.update(f"\0{label}:{path.stat().st_size}\0".encode())
  303. with open(path, "rb") as fh:
  304. for chunk in iter(lambda: fh.read(1024 * 1024), b""):
  305. hasher.update(chunk)
  306. return hasher.hexdigest()
  307. return _read_offline(db_path, "backup-identity", _digest)
  308. def _read_repair_ledger(db_path: Path) -> dict[str, Any]:
  309. with contextlib.suppress(OSError, ValueError):
  310. raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8"))
  311. return raw if isinstance(raw, dict) else {}
  312. return {}
  313. def _persistent_repair_attempts_exhausted(db_path: Path) -> bool:
  314. """Whether *db_path* has already burned its cross-restart repair budget.
  315. True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` failures against the CURRENT
  316. fingerprint. Never raises; a missing/corrupt ledger or unstatable DB reads as "not exhausted" (the
  317. in-process claim and cross-process lock still bound one run). Fingerprint unavailable (live connection) ->
  318. compare the SIZE the ledger recorded, otherwise a peer connection hides an exhausted budget on every pass."""
  319. ledger = _read_repair_ledger(db_path)
  320. recorded = ledger.get("fingerprint")
  321. fp = _db_fingerprint(db_path)
  322. try: # size is the one key component that needs no raw read
  323. same = recorded == fp if fp is not None else (
  324. isinstance(recorded, str) and recorded.startswith(f"{db_path.stat().st_size}:"))
  325. except OSError:
  326. return False
  327. return same and int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS
  328. def _persistent_repair_exhausted_error(db_path: Path) -> str:
  329. """The stable operator-facing diagnostic for an exhausted repair budget. The ``hermes`` commands
  330. carry the profile selector: a bare ``hermes`` follows ``active_profile`` (#105887)."""
  331. from hermes_constants import profile_cli_selector
  332. profile_arg = profile_cli_selector()
  333. return (f"automatic repair has already failed {_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — the "
  334. f"corruption is beyond the schema/FTS repair strategies (likely b-tree page damage). Manual recovery "
  335. f"required: restore a backup, or salvage with `hermes {profile_arg}sessions recover --source {db_path} "
  336. f"--inspect-only`, then (if it reports recoverable) `hermes {profile_arg}sessions recover --source {db_path} "
  337. f"--output recovered-state.db` (recovery snapshots the damaged file first, then runs the page-level "
  338. f"`.recover` lane on the copy; do NOT point a raw `sqlite3` shell at the live database). "
  339. f"Delete {_repair_ledger_path(db_path).name} to force another automatic attempt.")
  340. def _record_repair_outcome(db_path: Path, *, repaired: bool, fingerprint: Optional[str] = None) -> None:
  341. """Update the persistent attempt ledger after a repair pass. Never raises.
  342. Keys on the post-attempt fingerprint (what the NEXT exhaustion probe sees). If a live connection makes it
  343. unavailable, keep the recorded key and still increment — dropping the pass lets a peer reset the budget
  344. every time. Never write a differently shaped key."""
  345. ledger_path = _repair_ledger_path(db_path)
  346. try:
  347. if repaired:
  348. ledger_path.unlink(missing_ok=True)
  349. return
  350. ledger = _read_repair_ledger(db_path)
  351. recorded = ledger.get("fingerprint")
  352. fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path)
  353. if fp is None:
  354. if not isinstance(recorded, str):
  355. # No prior key to extend and no safe way to mint one; the in-process claim and cross-process lock
  356. # still bound this run.
  357. return
  358. fp = recorded
  359. attempts = int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1
  360. stamp = datetime.datetime.now().isoformat(timespec="seconds")
  361. ledger_path.write_text(json.dumps({"fingerprint": fp, "failed_attempts": attempts, "last_attempt": stamp}), encoding="utf-8")
  362. except Exception as exc: # pragma: no cover - best effort
  363. logger.warning("Could not update state.db repair ledger: %s", exc)
  364. def _existing_malformed_backups(db_path: Path) -> list[Path]:
  365. """Timestamped forensic backups of *db_path*, newest first."""
  366. prefix = f"{db_path.name}.malformed-backup-"
  367. try:
  368. found = [p for p in db_path.parent.iterdir() if p.name.startswith(prefix) and not p.name.endswith(_DB_SIDECAR_SUFFIXES)]
  369. except OSError:
  370. return []
  371. return sorted(found, key=lambda p: p.name, reverse=True)
  372. def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None:
  373. """Delete all but the *keep* newest forensic backups (and sidecars)."""
  374. for stale in _existing_malformed_backups(db_path)[keep:]:
  375. for victim in (stale, *_sidecars(stale)):
  376. try:
  377. victim.unlink(missing_ok=True)
  378. except OSError as exc: # pragma: no cover - best effort
  379. logger.warning("Could not prune stale DB backup %s: %s", victim, exc)
  380. def _publish_backup_bundle(db_path: Path, staging: Path, backup_path: Path) -> None:
  381. """Copy DB + sidecars to *staging* names, then rename each into place.
  382. ORDER MATTERS: the main DB name is the bundle's commit marker (what ``_existing_malformed_backups``
  383. counts), so sidecars publish FIRST and the main DB LAST — a failure partway never leaves a countable
  384. backup over a missing sidecar. On failure, staging files AND anything promoted are removed."""
  385. main = (db_path, staging, backup_path)
  386. sidecars = [(sidecar, staging.with_name(staging.name + suffix), backup_path.with_name(backup_path.name + suffix))
  387. for suffix, sidecar in zip(_DB_SIDECAR_SUFFIXES, _sidecars(db_path)) if sidecar.exists()]
  388. published: list[Path] = []
  389. try:
  390. for src, staged, _dst in (main, *sidecars):
  391. shutil.copy2(src, staged)
  392. for _src, staged, dst in (*sidecars, main):
  393. os.replace(staged, dst)
  394. published.append(dst)
  395. except Exception:
  396. for victim in (*(staged for _s, staged, _d in (*sidecars, main)), *published):
  397. with contextlib.suppress(OSError): # best effort; a failure here is never the caller's error
  398. victim.unlink(missing_ok=True)
  399. raise
  400. def _backup_db_file(db_path: Path) -> tuple[Optional[Path], Optional[str]]:
  401. """Raw-copy a (possibly malformed) DB plus sidecars to a timestamped backup.
  402. Raw bytes on purpose: the DB won't open cleanly, so preserve them exactly for forensics. Returns ``(backup_path,
  403. None)`` or ``(None, reason)``; repair treats a refused backup as a HARD STOP because the bundle is the recovery
  404. path when every strategy fails. Refuses while a connection to this DB is live in the process: the raw read would
  405. ``close()`` a descriptor and cancel that connection's POSIX advisory locks (``hermes_cli.sqlite_safe_read``) —
  406. real case: one SessionDB enters repair while the gateway holds others.
  407. Dedupe: reuse the newest backup when byte-identical to the current recovery image (``_backup_content_identity``
  408. — NOT mtime, NOT ``_db_fingerprint``); a repair loop once re-copied the same bytes on every restart. Staging
  409. names live OUTSIDE the ``.malformed-backup-`` prefix: inside it they count as a backup, sort NEWEST (prune kept
  410. partials, deleted intact copies) and dedupe could return one with no real forensic copy on disk.
  411. Refusal reasons (``_backup_free_space_error`` / ``_MANUAL_RECOVER_HINT``) point operators at the safe lane,
  412. `hermes sessions recover --source <db> --inspect-only`, never at a raw sqlite3 shell on the live file.
  413. See #69603.
  414. """
  415. with contextlib.suppress(ImportError): # scaffold/embed installs without hermes_cli track no connections
  416. from hermes_cli.sqlite_safe_read import has_live_connection
  417. if has_live_connection(db_path):
  418. reason = (f"a connection to {db_path} is still open in this process; raw-copying it would cancel that "
  419. "connection's POSIX advisory locks. Close all SessionDB handles first.")
  420. logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason)
  421. return None, reason
  422. stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
  423. backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}")
  424. for seq in itertools.count(1): # same-second collision must not overwrite the earlier forensic copy
  425. if not backup_path.exists():
  426. break
  427. backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}_{seq}")
  428. try:
  429. # Sweep staging debris from an interrupted pass BEFORE the dedupe (it is byte-identical to the damaged
  430. # DB, so dedupe would hand it back as a backup). Also the old ``.incomplete`` spelling, which
  431. # prefix-matches as a backup, sorts NEWEST and would otherwise survive prune forever.
  432. for pattern in (f"{db_path.name}.backup-staging-*", f"{db_path.name}.malformed-backup-*.incomplete*"):
  433. for old in db_path.parent.glob(pattern):
  434. with contextlib.suppress(OSError):
  435. old.unlink(missing_ok=True)
  436. with contextlib.suppress(OSError):
  437. # Hash the source only when there is a candidate: hashing a multi-GB file before copying it is waste.
  438. if newest := _existing_malformed_backups(db_path)[:1]:
  439. src_id = _backup_content_identity(db_path)
  440. if src_id is not None and _backup_content_identity(newest[0]) == src_id:
  441. logger.info("Reusing existing forensic backup %s (identical to the damaged DB).", newest[0])
  442. return newest[0], None
  443. if (reason := _backup_free_space_error(db_path)) is not None:
  444. logger.error("Refusing forensic backup of %s: %s", db_path, reason)
  445. return None, reason
  446. _publish_backup_bundle(db_path, db_path.with_name(f"{db_path.name}.backup-staging-{stamp}"), backup_path)
  447. # Retention cap (#86747): keep only the newest few forensic copies.
  448. _prune_malformed_backups(db_path)
  449. return backup_path, None
  450. except Exception as exc: # pragma: no cover - best effort
  451. logger.warning("Could not back up malformed DB %s: %s", db_path, exc)
  452. return None, f"backup copy failed: {exc}"
  453. def _rw_or_vanished(p: Path) -> bool:
  454. """True when *p* is read-write for this user OR no longer exists.
  455. The preflight lists the files first and checks them after, and a live sibling connection's last ``close()``
  456. unlinks ``-wal``/``-shm`` in between (a concurrent quarantine can rename ``state.db`` aside the same way).
  457. ``os.access`` on the vanished path returns False, which is "gone", not "read-only". The final re-check covers
  458. a sidecar that vanished and was recreated by the next opener before ``exists()`` ran.
  459. """
  460. mode = os.R_OK | os.W_OK
  461. return os.access(p, mode) or not p.exists() or os.access(p, mode)
  462. def preflight_db_writability(db_path: Path, *, db_label: str = "state.db", directory_only: bool = False) -> None:
  463. """Refuse-or-repair read-only DB files BEFORE the first connection opens.
  464. A stray read-only ``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored backup, copied dotfiles) otherwise
  465. surfaces as an opaque "attempt to write a readonly database" inside ``_init_schema``, and the obvious wrong
  466. "fix" (deleting the ``-wal``) loses committed transactions. ``chmod u+rw`` repair only inside the Hermes home
  467. tree (Hermes owns those files; ``chmod`` fails on files the user doesn't own, bounding the repair exactly);
  468. otherwise fail fast naming the file and command. Never deletes/truncates a WAL sidecar — once writable, the
  469. normal open checkpoints it. ``:memory:``/``file:`` skipped. Shared with ``kanban_db``.
  470. Port of Kilo-Org/kilocode#12508's startup preflight. This preflight:
  471. """
  472. if str(db_path) == ":memory:" or str(db_path).startswith("file:"):
  473. return
  474. home: Optional[Path] = None
  475. with contextlib.suppress(Exception): # pragma: no cover - defensive
  476. home = Path(get_hermes_home()).resolve()
  477. # SQLite needs a writable directory in every journal mode (WAL/SHM sidecars, or the DELETE-mode journal).
  478. sidecars = (db_path.with_name(db_path.name + "-wal"), db_path.with_name(db_path.name + "-shm"))
  479. files = () if directory_only else tuple((p, False) for p in (db_path, *sidecars) if p.is_file())
  480. for p, is_dir in [(db_path.parent, True), *files]:
  481. if (is_dir and not p.is_dir()) or _rw_or_vanished(p):
  482. continue
  483. x = "x" if is_dir else ""
  484. in_scope = False
  485. with contextlib.suppress(OSError, ValueError):
  486. in_scope = home is not None and p.resolve().is_relative_to(home)
  487. if in_scope:
  488. os.chmod(p, p.stat().st_mode | stat.S_IRUSR | stat.S_IWUSR | (stat.S_IXUSR if is_dir else 0))
  489. if in_scope and os.access(p, os.R_OK | os.W_OK):
  490. logger.info("%s preflight: repaired read-only %s (chmod u+rw%s)", db_label, p, x)
  491. continue
  492. wal_note = (" Do NOT delete the -wal file — it contains committed data that "
  493. "will be merged into the database once it is writable." if p.name.endswith("-wal") else "")
  494. raise sqlite3.OperationalError(
  495. f"{db_label} is not writable: {'directory' if is_dir else 'file'} {p} is read-only for this user. Hermes "
  496. f"needs read-write access to open the database. Fix with: chmod u+rw{x} '{p}' (files owned by another "
  497. f"user may need sudo/chown).{wal_note}")
  498. def _connect_repair_durable(db_path: Path, *, timeout: float = 5.0) -> sqlite3.Connection:
  499. """``sqlite3.connect`` for the repair/probe paths, with macOS write barriers.
  500. These paths bypass ``SessionDB``/:func:`apply_wal_with_fallback`, so they inherited ``synchronous=NORMAL`` and
  501. no ``checkpoint_fullfsync`` — on Darwin an interrupted ``REINDEX``/``VACUUM``/``writable_schema`` rewrite leaves
  502. half-written b-tree pages. Autocommit (``isolation_level=None``): DDL and ``VACUUM`` are illegal inside an
  503. implicit transaction. Barriers are best-effort: on a malformed schema even ``PRAGMA synchronous=FULL`` raises,
  504. so whole-file rewrites call :func:`_reapply_durability_barriers` once the schema parses again.
  505. Tracked (:func:`hermes_cli.sqlite_safe_read.connect_tracked`) because repair connections hold the
  506. strongest locks in the process (``locking_mode=EXCLUSIVE``, ``BEGIN IMMEDIATE``); an untracked fd let
  507. the byte-level probes ``open()``/``close()`` the live file, which cancels every POSIX advisory lock this
  508. process holds on it (sqlite.org/howtocorrupt §2.2) and lets an external writer commit mid-repair (#63386).
  509. """
  510. from hermes_cli.sqlite_safe_read import connect_tracked
  511. conn = connect_tracked(db_path, tracking_path=db_path, timeout=timeout, isolation_level=None)
  512. _reapply_durability_barriers(conn)
  513. return conn
  514. def _repair_conn(db_path: Path, *, timeout: float = 5.0):
  515. """A :func:`_connect_repair_durable` connection as a context manager, closed on exit."""
  516. return contextlib.closing(_connect_repair_durable(db_path, timeout=timeout))
  517. def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool:
  518. """Best-effort (re)application of the macOS write barriers; True if accepted.
  519. Call before ``VACUUM``/``REINDEX`` once the schema parses: a connection opened
  520. on a malformed schema could not take them at open time. Never raises."""
  521. from hermes_state_wal import _apply_macos_checkpoint_barrier, _enforce_macos_synchronous_full
  522. try:
  523. _apply_macos_checkpoint_barrier(conn)
  524. _enforce_macos_synchronous_full(conn)
  525. return True
  526. except Exception: # schema still unparseable, or anything else — pragmas cannot be set yet
  527. return False
  528. def apply_durability_barriers(conn: sqlite3.Connection) -> bool:
  529. """Durability barriers for guest users of ``state.db`` that must inherit its owner's journal mode. Also
  530. applies the configured ``database.synchronous`` level, a per-connection pragma that otherwise only
  531. rides on the journal-mode setup path guests must not run."""
  532. from hermes_state_wal import _apply_synchronous_pragma
  533. ok = _reapply_durability_barriers(conn)
  534. with contextlib.suppress(Exception):
  535. from hermes_cli.config import cfg_get, load_config_readonly # local: avoids an import cycle
  536. if (raw_synchronous := cfg_get(load_config_readonly(), "database", "synchronous", default=None)) is not None:
  537. _apply_synchronous_pragma(conn, raw_synchronous, db_label="state.db (guest)")
  538. return ok
  539. def _close_unpinned(conn: sqlite3.Connection) -> None:
  540. """Leave EXCLUSIVE locking mode (so the file is never left pinned) and close."""
  541. with contextlib.suppress(Exception):
  542. conn.execute("PRAGMA locking_mode=NORMAL")
  543. conn.close()
  544. def _open_exclusive(db_path: Path, begin: str) -> sqlite3.Connection:
  545. """Zero-timeout connection holding ``locking_mode=EXCLUSIVE`` after a rolled-back
  546. *begin*; closed (unpinned) and re-raised when exclusion cannot be taken."""
  547. conn = _connect_repair_durable(db_path, timeout=0.0)
  548. try:
  549. for statement in ("PRAGMA locking_mode=EXCLUSIVE", begin, "ROLLBACK"):
  550. conn.execute(statement)
  551. except BaseException:
  552. _close_unpinned(conn)
  553. raise
  554. return conn
  555. @contextlib.contextmanager
  556. def _exclusive_repair_db_guard(db_path: Path):
  557. """Yield ``(conn, None)`` — one live connection that excludes writers for repair surgery — or ``(None,
  558. exc)`` when exclusion could not be taken.
  559. ``locking_mode=EXCLUSIVE`` retains file-level exclusion after the short ``BEGIN EXCLUSIVE`` is rolled
  560. back; the rollback is essential because ``Connection.backup`` uses this connection as *source* and later
  561. as the promotion *destination*, both transaction-free. It stays open across the snapshot -> strategies ->
  562. promotion window so no writer can commit a change promotion would overwrite. Existing readers make
  563. acquisition fail rather than being disturbed (fail closed). Timeout 0: the cross-process lock already
  564. serializes repairers, and a partial repair is less safe than "stop the gateway and retry"."""
  565. try:
  566. guard = _open_exclusive(db_path, "BEGIN EXCLUSIVE")
  567. except (sqlite3.Error, OSError) as exc:
  568. yield None, exc
  569. return
  570. try:
  571. yield guard, None
  572. finally:
  573. # Releasing the exclusive locks before close keeps a close-time checkpoint from being mistaken for a
  574. # repair write by callers that reopen immediately.
  575. _close_unpinned(guard)
  576. def _copy_database_snapshot(source_path: Path, destination_path: Path, *,
  577. source_connection: Optional[sqlite3.Connection] = None,
  578. destination_connection: Optional[sqlite3.Connection] = None) -> None:
  579. """Copy one complete SQLite snapshot without replacing either file inode: the online backup API folds
  580. committed WAL frames into the source snapshot and writes the destination in one transaction (rolled
  581. back if interrupted), so ``state.db`` is never swapped out from under handles that refer to it."""
  582. # Deadline first: a sidecar vanishing mid-stat must not leak a just-opened descriptor.
  583. deadline_seconds = _repair_snapshot_timeout_seconds(source_path)
  584. deadline = time.monotonic() + deadline_seconds
  585. def _check_deadline(_status: int, _remaining: int, _total: int) -> None:
  586. if time.monotonic() >= deadline:
  587. raise TimeoutError(f"timed out copying SQLite repair snapshot after {deadline_seconds:.0f}s")
  588. with contextlib.ExitStack() as owned: # closes only connections opened here, destination first
  589. source = source_connection or owned.enter_context(_repair_conn(source_path))
  590. if destination_connection is not None and destination_connection.in_transaction:
  591. # sqlite3_backup needs a transaction-free destination (the guard holds exclusion via locking_mode).
  592. raise sqlite3.ProgrammingError("SQLite repair backup destination has an active transaction")
  593. destination = destination_connection or owned.enter_context(_repair_conn(destination_path))
  594. source.backup(destination, pages=256, progress=_check_deadline, sleep=_REPAIR_LOCK_POLL_SECONDS)
  595. def _schema_not_built(exc: BaseException) -> bool:
  596. """``no such table/column``: FTS5 / core tables not created yet (brand new file mid-init)."""
  597. return any(m in str(exc).lower() for m in ("no such table", "no such column"))
  598. # Hermes-owned FTS5 objects: the virtual tables and their shadow b-trees. Full-matched, so a
  599. # user-created lookalike (``archive_fts_data``) is not swept into the rebuildable set.
  600. _FTS_OBJECT_RE = re.compile(
  601. r"messages_fts(_trigram|_cjk)?(_data|_idx|_content|_docsize|_config|_segdir|_segments)?"
  602. )
  603. _INTEGRITY_TREE_RE = re.compile(r"\bTree (\d+)\b")
  604. _INTEGRITY_MISSING_INDEX_RE = re.compile(r"missing from index (\S+)")
  605. def integrity_damage_is_structural(integrity_lines, master_rows) -> bool:
  606. """True when any damaged object named by ``PRAGMA integrity_check`` output lies outside
  607. the FTS shadow set: a ``Tree N`` id mapped through ``sqlite_master.rootpage``, an index in
  608. ``row N missing from index X``, or the file's own freelist. Unparseable lines are not
  609. counted (the FTS wording stays, which is incomplete rather than wrong). #88587: an FTS
  610. rebuild cannot repair a canonical b-tree, and the ``.malformed-backup`` it leaves behind
  611. is a snapshot of the same damage."""
  612. name_by_rootpage = {int(rp): name for rp, _type, name in master_rows if rp}
  613. for line in integrity_lines:
  614. text = str(line)
  615. if text.startswith("Freelist"):
  616. return True
  617. tree = _INTEGRITY_TREE_RE.search(text)
  618. if tree:
  619. name = name_by_rootpage.get(int(tree.group(1)), "")
  620. if name and not _FTS_OBJECT_RE.fullmatch(name):
  621. return True
  622. missing = _INTEGRITY_MISSING_INDEX_RE.search(text)
  623. if missing and not _FTS_OBJECT_RE.fullmatch(missing.group(1)):
  624. return True
  625. return False
  626. def state_db_has_structural_damage(db_path: Path) -> bool:
  627. """Read-only ``integrity_check`` + ``sqlite_master`` rootpage map on a fresh connection;
  628. ``integrity_damage_is_structural`` over the result. A check that RAISES instead of
  629. reporting (a torn page under the walk) is structural too: no FTS-only fixture does that
  630. while ``messages``/``sessions`` read cleanly, and the FTS rebuild ladder cannot help.
  631. Cannot-open / locked stays False so the caller keeps the FTS path."""
  632. try:
  633. conn = sqlite3.connect(read_only_db_uri(db_path), uri=True, timeout=1.0)
  634. except sqlite3.Error:
  635. return False
  636. try:
  637. master_rows = [tuple(r) for r in conn.execute(
  638. "SELECT rootpage, type, name FROM sqlite_master WHERE rootpage > 0").fetchall()]
  639. lines = [str(r[0]) for r in conn.execute("PRAGMA integrity_check").fetchall()]
  640. except sqlite3.OperationalError:
  641. return False
  642. except sqlite3.DatabaseError:
  643. return True
  644. finally:
  645. conn.close()
  646. return integrity_damage_is_structural(
  647. itertools.chain.from_iterable(line.splitlines() for line in lines), master_rows)
  648. def _db_opens_cleanly(db_path: Path) -> Optional[str]:
  649. """Probe a DB on a fresh connection. Returns None if healthy, else a reason.
  650. Runs the first statement that trips the malformed-schema parse (``PRAGMA journal_mode``),
  651. ``integrity_check``, a ``sessions`` read, FTS5 MATCH probes and a rolled-back ``messages`` write — so FTS5
  652. index corruption (reads and ``integrity_check`` pass, every ``INSERT INTO messages`` fails through the FTS
  653. triggers) is reported as unhealthy.
  654. See #50502.
  655. """
  656. from hermes_state import SessionDB, load_fts5_cjk_extension
  657. # ── Strategy 0.5: rebuild stale B-tree indexes (#63386) ── PRAGMA integrity_check can report "wrong #
  658. # of entries in index" when a B-tree index (e.g. idx_sessions_handoff_state) falls out of sync with its
  659. # base table. REINDEX rewrites the index b-tree from the canonical table rows using the existing index
  660. # definition, fixing the mismatch without touching data or FTS schema.
  661. conn = _connect_repair_durable(db_path)
  662. try:
  663. with contextlib.closing(conn):
  664. # Best-effort tokenizer load: messages_fts_cjk needs cjk_unicode61 before any statement can touch it;
  665. # tokenizer absence must never classify as corruption.
  666. load_fts5_cjk_extension(conn)
  667. conn.execute("PRAGMA journal_mode").fetchone()
  668. rows = conn.execute("PRAGMA integrity_check").fetchall()
  669. problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
  670. if problems:
  671. return "; ".join(problems[:3])
  672. conn.execute("SELECT COUNT(*) FROM sessions").fetchone()
  673. # FTS5 read probe: partial shadow-table corruption makes MATCH/snippet/rank raise while integrity_check
  674. # reports healthy. MATCH '""' (empty phrase) parses, scans zero rows and exercises the shadow tables;
  675. # FTS5 rejects MATCH ''.
  676. for fts_table in _FTS_TABLES:
  677. try:
  678. conn.execute(f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1").fetchone()
  679. except sqlite3.DatabaseError as exc:
  680. # Builds without fts5/trigram raise "no such module|tokenizer"; calling that corruption would
  681. # send the DB into repair, whose final fallback deletes messages_fts%. "no such table/column" =
  682. # not built yet.
  683. benign = SessionDB._is_fts5_unavailable_error(exc) or _schema_not_built(exc)
  684. if not (isinstance(exc, sqlite3.OperationalError) and benign):
  685. # This is the corruption class #66724 actually wants caught: partial shadow-table
  686. # damage where MATCH / snippet / rank queries raise DatabaseError("database disk
  687. # image is malformed") while reads of the FTS5 table itself parse fine.
  688. return f"fts5 read probe failed on {fts_table}: {exc}"
  689. # FTS write probe: drive a row through the messages_fts* triggers in a transaction that is always
  690. # rolled back. The trigger INSERT alone only buffers the row in FTS5's in-memory segment; the
  691. # ``flush`` command writes that segment to ``<fts>_idx``/``_data`` exactly as a committed append
  692. # would, so a stale ``_idx`` row at the next segid (IntegrityError "constraint failed", the #100227
  693. # class: integrity_check and MATCH both clean, every real append fails) is hit here rather than
  694. # by the user's next message.
  695. probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}"
  696. try:
  697. conn.execute("BEGIN IMMEDIATE")
  698. conn.execute("INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
  699. (probe_session_id, "_health_probe", time.time()))
  700. conn.execute("INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)",
  701. (probe_session_id, "user", "_fts_health_probe", time.time()))
  702. for fts_table in _FTS_TABLES:
  703. try:
  704. conn.execute(f"INSERT INTO {fts_table}({fts_table}) VALUES('flush')")
  705. except sqlite3.OperationalError as exc:
  706. if not (SessionDB._is_fts5_unavailable_error(exc) or _schema_not_built(exc)):
  707. raise
  708. except sqlite3.DatabaseError as exc:
  709. # IntegrityError is a DatabaseError sibling of OperationalError, not a child: catching only the
  710. # latter let the trigram-segment collision report "healthy".
  711. # Missing messages/sessions tables = brand new file mid-init, not corruption. "no such tokenizer":
  712. # this process lacks the cjk extension the DB's index needs — capability gap; a tokenizer-less
  713. # SessionDB drops the triggers itself.
  714. if _schema_not_built(exc) or "no such tokenizer: cjk_unicode61" in str(exc).lower():
  715. return None
  716. return f"fts5 write probe failed: {exc}"
  717. finally:
  718. with contextlib.suppress(sqlite3.Error):
  719. conn.execute("ROLLBACK")
  720. return None
  721. except sqlite3.DatabaseError as exc:
  722. return str(exc)
  723. def _live_writer_holds_db(db_path: Path) -> bool:
  724. """True when another process (or a connection outside this call) still holds ``db_path``.
  725. The foreign-holder scan (``hermes_state_holders``) is the authority: any other process with the DB or a
  726. WAL sidecar open, a deleted WAL generation, or an unknown/uninspectable holder fails CLOSED. The SQLite
  727. probe (``locking_mode=EXCLUSIVE`` + ``BEGIN IMMEDIATE``) is only an additional positive signal — it cannot
  728. see a ``journal_mode=DELETE`` reader and cannot run on a malformed file, which is why the scan comes first."""
  729. import hermes_state_holders as _state_holders
  730. return _state_holders.live_writer_holds_db(db_path, connect_repair_durable=_connect_repair_durable)
  731. def _repair_skip(report: dict[str, Any], verb: str, error: str, exc: Optional[BaseException] = None) -> dict[str, Any]:
  732. """Record *error* on *report* and log it as ``state.db repair <verb>``. An
  733. *exc* proving deterministic corruption consumes the persistent repair budget
  734. (private ``_repair_attempted`` marker, popped by the caller)."""
  735. if exc is not None and _repair_failure_consumes_attempt(exc):
  736. report["_repair_attempted"] = True
  737. report["error"] = error
  738. logger.error(f"state.db repair {verb}: %s", error)
  739. return report
  740. def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> dict[str, Any]:
  741. """Repair a state.db whose ``sqlite_master`` is malformed or whose FTS indexes reject writes.
  742. Two corruption classes: malformed schema / "duplicate object definition" (even ``PRAGMA`` fails), and FTS
  743. write-corruption (reads and ``integrity_check`` pass, writes fail through ``messages_fts*`` triggers).
  744. ``_REPAIR_STRATEGIES`` run least-destructive first on a complete snapshot; a success is copied back
  745. transactionally, so canonical rows are never modified by a failed attempt. A raw backup is taken first
  746. unless ``backup=False``. Serialised across processes (gateway, Desktop backend and CLI open the same file;
  747. concurrent ``writable_schema`` surgery is itself a corruption source). Returns ``{repaired, strategy,
  748. backup_path, error}``.
  749. See #50502.
  750. """
  751. report: dict[str, Any] = {"repaired": False, "strategy": None, "backup_path": None, "error": None}
  752. # Startup-watchdog lease: repair is I/O-bound (near-zero CPU), which the watchdog's CPU fallback would
  753. # misread as a parked deadlock. One lease (clamped to _MAX_LEASE_S=900) beats per-chunk renewal complexity.
  754. report_startup_progress(900.0, phase="state_db_repair")
  755. db_path = Path(db_path)
  756. if not db_path.exists():
  757. report["error"] = f"{db_path} does not exist"
  758. return report
  759. # Cross-restart cap: the in-memory claim bounds one process, but unhealable b-tree damage used to re-run
  760. # surgery + a fresh backup on EVERY restart.
  761. # Cross-restart attempt cap (#86747): the in-memory claim bounds one process, but a corruption class the
  762. # strategies below cannot heal (b-tree page damage) previously re-ran the whole surgery — and took a
  763. # fresh multi-hundred-MB forensic backup — on EVERY restart, forever. After
  764. # _MAX_PERSISTENT_REPAIR_ATTEMPTS failures against the same damaged file, stop retrying and surface a
  765. # terminal, actionable error.
  766. if _persistent_repair_attempts_exhausted(db_path):
  767. return _repair_skip(report, "skipped", _persistent_repair_exhausted_error(db_path))
  768. with _cross_process_repair_lock(db_path) as holding_lock:
  769. if not holding_lock:
  770. # Another process holds the lock (or the lock file was unopenable); it may have healed the file
  771. # already, so re-probe before failing.
  772. if _db_opens_cleanly(db_path) is None:
  773. report["repaired"], report["strategy"] = True, "repaired_by_other_process"
  774. else:
  775. report["error"] = ("could not obtain the state.db repair lock (held by another process, or the lock "
  776. "file was unopenable); skipped schema surgery to avoid racing a concurrent repairer")
  777. return report
  778. result = report
  779. # Recheck exhaustion after acquisition: a queued repairer can have recorded the final failure while
  780. # this process waited.
  781. if _persistent_repair_attempts_exhausted(db_path):
  782. _repair_skip(report, "skipped", _persistent_repair_exhausted_error(db_path))
  783. # WAL-holder preflight: fail closed for active readers before a backup is taken. Not the race defence — the
  784. # exclusive guard in the locked routine excludes writers through promotion and sees DELETE-mode readers too.
  785. elif _live_writer_holds_db(db_path):
  786. _repair_skip(report, "skipped", "a live writer still holds state.db; skipped schema surgery to avoid tearing "
  787. "b-tree pages under a concurrent writer. Stop the gateway (hermes gateway stop) and retry.")
  788. else:
  789. # Probe journal mode BEFORE surgery: a rebuilt file comes back in the default (delete) mode and nothing
  790. # else records the flip. Unprobeable (damaged file) -> database.journal_mode is the restore target.
  791. # The open-time WAL-reset gate never sees this flip because it happens inside the repair path
  792. # (distinct from the open-time flip #89393 warns about).
  793. before_mode = _probe_journal_mode_for_repair(db_path)
  794. result = _repair_state_db_schema_locked(db_path, backup=backup, report=report,
  795. journal_mode_before=before_mode)
  796. if result.get("repaired"):
  797. result["journal_mode_before"] = before_mode
  798. # Environmental aborts (before a strategy mutates the snapshot) are retriable, not proof of exhaustion;
  799. # the private marker stays out of the public report. The ledger update stays under the cross-process
  800. # lock so two repairers cannot lose each other's updates; a queued loser must not record at all.
  801. if result.pop("_repair_attempted", False) or result.get("repaired"):
  802. _record_repair_outcome(db_path, repaired=bool(result.get("repaired")))
  803. return result
  804. def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]:
  805. """Best-effort journal-mode probe: ``wal``/``delete``, or ``None`` when the file cannot be opened or
  806. probed (malformed header, concurrent opener's locks — both expected on the repair path); callers then
  807. fall back to ``database.journal_mode``."""
  808. from hermes_state_wal import _on_disk_journal_mode
  809. try:
  810. with _repair_conn(db_path) as conn:
  811. return _on_disk_journal_mode(conn)
  812. except (sqlite3.Error, OSError):
  813. return None
  814. def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str], *, conn=None) -> None:
  815. """Re-apply the journal mode after schema surgery.
  816. ``conn`` must be the exclusive repair guard connection when called from the repair path: opening a fresh
  817. connection AFTER the guard released let a writer still holding the unlinked old ``-wal`` inode coexist with
  818. a brand-new ``state.db-wal`` — two generations of one store. The reopen is the hazard, not the mode.
  819. A rebuilt file comes back in the default (delete) mode; without this a corruption event silently moves a
  820. WAL store out of WAL (the open-time WAL-reset gate never sees a flip made inside repair). Routed through
  821. :func:`apply_wal_with_fallback`, not a direct pragma, so it inherits the WAL-reset gate (a vulnerable
  822. runtime deliberately keeps DELETE; the journal_mode-changed WARNING is expected there), the macOS-NFS
  823. silent-refusal handling and the WAL companions. ``before_mode`` is only for the log comparison; the target
  824. is ``database.journal_mode``. Best-effort: the repair already succeeded, so failures log at WARNING.
  825. See #89674.
  826. See #89393.
  827. The transactional promotion already leaves the destination in its pre-repair mode, so on that path this
  828. is mostly the WAL-companion re-assertion; the reopen is the hazard, not the mode. See #101064.
  829. """
  830. from hermes_state_wal import apply_wal_with_fallback
  831. try:
  832. if conn is None:
  833. with _repair_conn(db_path) as owned:
  834. after = apply_wal_with_fallback(owned, db_label=db_path.name)
  835. else:
  836. after = apply_wal_with_fallback(conn, db_label=db_path.name)
  837. if before_mode and after != before_mode:
  838. logger.warning("state.db repair changed journal_mode %r -> %r (pre-surgery probe %r; restore resolved "
  839. "through apply_wal_with_fallback per database.journal_mode and the WAL-reset gate)",
  840. before_mode, after, before_mode)
  841. except (sqlite3.Error, OSError) as exc:
  842. logger.warning("state.db repair at %s: post-surgery journal-mode restore failed (%s); verify with PRAGMA "
  843. "journal_mode on the next open", db_path, exc)
  844. def _repair_state_db_schema_locked(
  845. db_path: Path, *, backup: bool, report: dict[str, Any], journal_mode_before: Optional[str] = None,
  846. ) -> dict[str, Any]:
  847. """Repair strategies for :func:`repair_state_db_schema`; caller holds the cross-process repair lock.
  848. Strategies run on a SCRATCH COPY, copied back through SQLite's transactional backup API only once proven to open
  849. cleanly, so a failed repair cannot modify or lose committed data (a WAL checkpoint of committed frames on guard
  850. release is not a repair mutation). WHY not in place: the final strategy ends in ``VACUUM``, which rebuilds the
  851. file from the schema SQLite can still parse — when the damage IS in the schema b-tree (the ``malformed database
  852. schema ()`` class) every table hanging off the unreadable part is silently dropped, the probe still reports
  853. malformed, and repair returned ``repaired=False`` having destroyed what it was asked to save.
  854. The pre-repair backup (#69603) does not close this: it is a forensic artefact that nothing reads back,
  855. so recovery still depends on a human noticing a ``.malformed-backup-*`` file and knowing what to do with
  856. it. Not mutating the original in the first place is the property that holds without a human in the loop.
  857. """
  858. scratch = db_path.with_name(f"{db_path.name}.repair-scratch")
  859. if (cleanup_error := _unlink_db_triple(scratch)) is not None:
  860. return _repair_skip(report, "aborted", f"could not remove a stale repair snapshot before probing state.db: {cleanup_error}")
  861. # Re-probe under the lock: a process we queued behind may have just repaired the file; redoing surgery
  862. # would undo it (the repair/re-corrupt cascade).
  863. if _db_opens_cleanly(db_path) is None:
  864. report["repaired"], report["strategy"] = True, "already_healthy"
  865. return report
  866. if backup:
  867. bpath, backup_error = _backup_db_file(db_path)
  868. report["backup_path"] = str(bpath) if bpath else None
  869. if bpath is None: # HARD STOP: the forensic image is the recovery path when every strategy fails.
  870. return _repair_skip(report, "aborted", "pre-repair backup refused; aborting schema repair to avoid "
  871. f"mutating the only copy of the damaged DB: {backup_error}")
  872. # The forensic copy precedes this guard on purpose: its live-holder checks would be poisoned by our own
  873. # exclusive connection. Everything touching the repair image or live promotion happens under writer exclusion.
  874. with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error):
  875. if live_guard is None:
  876. return _repair_skip(report, "skipped", "could not acquire exclusive state.db repair ownership; skipped "
  877. f"schema surgery to avoid overwriting a concurrent writer. Stop the gateway and retry: "
  878. f"{guard_error}", exc=guard_error)
  879. if (space_error := _repair_scratch_space_error(db_path)) is not None:
  880. return _repair_skip(report, "aborted", space_error)
  881. try:
  882. # Source = live_guard: it owns the exclusion, and a second connection could be blocked by our own
  883. # EXCLUSIVE lock on some SQLite builds.
  884. _copy_database_snapshot(db_path, scratch, source_connection=live_guard)
  885. except (OSError, sqlite3.Error, TimeoutError) as exc:
  886. _unlink_db_triple(scratch)
  887. return _repair_skip(report, "aborted", f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}", exc=exc)
  888. try:
  889. # Private marker for the outer wrapper: a strategy failure consumes the persistent budget; a
  890. # promotion failure is classified separately.
  891. report["_repair_attempted"] = True
  892. _run_repair_strategies(scratch, report)
  893. if report.get("repaired"):
  894. # Never ``os.replace`` the live DB: Windows rejects replacement under open handles and POSIX would
  895. # leave those handles on the old inode. The guard keeps writer exclusion throughout.
  896. try:
  897. _copy_database_snapshot(scratch, db_path, destination_connection=live_guard)
  898. except (OSError, sqlite3.Error, TimeoutError) as exc:
  899. report.update(repaired=False, strategy=None,
  900. _repair_attempted=_repair_failure_consumes_attempt(exc))
  901. report["error"] = f"repaired snapshot could not be promoted transactionally: {exc}"
  902. logger.error("state.db repair promotion failed: %s", exc)
  903. else:
  904. logger.warning("state.db repaired via '%s' and promoted transactionally: %s",
  905. report.get("strategy"), db_path)
  906. _restore_journal_mode_after_repair(db_path, journal_mode_before, conn=live_guard)
  907. if not report.get("repaired"):
  908. # Logged HERE, not in the strategies: they see the scratch copy, and the message a human acts on
  909. # must name a path that still exists.
  910. logger.error("state.db schema repair could not recover %s automatically (no committed canonical data "
  911. "was modified or lost; backup: %s); manual restore from backup may be required.",
  912. db_path, report["backup_path"])
  913. return report
  914. finally:
  915. # Never leave a half-repaired file beside the DB to be mistaken for the real thing.
  916. if (cleanup_error := _unlink_db_triple(scratch)) is not None:
  917. logger.warning("Could not remove state.db repair snapshot after repair: %s", cleanup_error)
  918. def _unlink_db_triple(path: Path) -> Optional[str]:
  919. """Remove *path* and every SQLite sidecar; return any cleanup failure."""
  920. from hermes_state import _IS_WINDOWS
  921. failures: list[str] = []
  922. for victim in (path, *_sidecars(path)):
  923. for attempt in range(10):
  924. try:
  925. victim.unlink(missing_ok=True)
  926. except OSError as exc:
  927. # Windows may retain a just-closed SQLite handle for a few scheduler ticks; bounded retry.
  928. if isinstance(exc, PermissionError) and _IS_WINDOWS and attempt < 9:
  929. time.sleep(0.05)
  930. continue
  931. failures.append(f"{victim}: {exc}")
  932. break
  933. return "; ".join(failures) or None
  934. # ── Repair strategies, least destructive first (each mutates its connection's DB) ──
  935. def _edit_sqlite_master(conn: sqlite3.Connection, edit) -> None:
  936. """Run *edit* under ``writable_schema=ON``; bump the schema cookie when it
  937. reports a change so live peers discard their cached schema."""
  938. conn.execute("PRAGMA writable_schema=ON")
  939. if edit():
  940. _bump_schema_cookie(conn)
  941. conn.execute("PRAGMA writable_schema=OFF")
  942. conn.commit()
  943. def _strategy_rebuild_fts(conn: sqlite3.Connection) -> None:
  944. """FTS5 'rebuild' rewrites each index from the content table: the least-
  945. destructive fix for an index that rejects writes while reads work."""
  946. from hermes_state import load_fts5_cjk_extension
  947. # The cjk index can only be rebuilt with its tokenizer loaded (best-effort).
  948. load_fts5_cjk_extension(conn)
  949. for table_name in _FTS_TABLES:
  950. with contextlib.suppress(sqlite3.OperationalError): # table absent (FTS disabled / trigram off / cjk absent)
  951. conn.execute(f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')")
  952. def _strategy_reindex(conn: sqlite3.Connection) -> None:
  953. """integrity_check reports "wrong # of entries in index" when a B-tree
  954. index drifts from its base table; REINDEX rewrites it from canonical rows."""
  955. # REINDEX rewrites every index b-tree; take the barriers now that the schema parses.
  956. _reapply_durability_barriers(conn)
  957. conn.execute("REINDEX")
  958. conn.commit()
  959. def _strategy_dedup_schema(conn: sqlite3.Connection) -> None:
  960. """De-duplicate sqlite_master (lowest rowid per type/name wins), keeping FTS."""
  961. def _dedup() -> bool:
  962. dupes = conn.execute(
  963. "SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep FROM sqlite_master GROUP BY type, name HAVING c > 1"
  964. ).fetchall()
  965. for type_, name, _count, keep in dupes:
  966. conn.execute("DELETE FROM sqlite_master WHERE type IS ? AND name IS ? AND rowid <> ?",
  967. (type_, name, keep))
  968. return bool(dupes)
  969. _edit_sqlite_master(conn, _dedup)
  970. def _strategy_drop_fts_vacuum(conn: sqlite3.Connection) -> None:
  971. """Drop all FTS schema and VACUUM; indexes rebuild on the next open. The
  972. destructive one, and why strategies run on a scratch copy: on a damaged
  973. schema b-tree VACUUM silently drops every table hanging off the unreadable part."""
  974. _edit_sqlite_master(conn, lambda: conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'") or True)
  975. # The schema parses now, so the barriers can stick — VACUUM rewrites the whole file.
  976. _reapply_durability_barriers(conn)
  977. conn.execute("VACUUM")
  978. # (name, body, success log, failure log) in escalation order. failure log None =
  979. # final strategy: its failure lands in report["error"] instead of logged-and-skipped.
  980. _REPAIR_STRATEGIES = (
  981. ("rebuild_fts", _strategy_rebuild_fts, "state.db FTS indexes rebuilt in place (schema preserved): %s",
  982. "state.db FTS in-place rebuild pass failed: %s"),
  983. ("reindex_btree", _strategy_reindex, "state.db B-tree indexes rebuilt via REINDEX: %s",
  984. "state.db REINDEX pass failed: %s"),
  985. ("dedup_schema", _strategy_dedup_schema,
  986. "state.db schema repaired by de-duplicating sqlite_master (FTS index preserved): %s",
  987. "state.db dedup repair pass failed: %s"),
  988. ("drop_fts_rebuild", _strategy_drop_fts_vacuum,
  989. "state.db schema repaired by dropping FTS schema; indexes will rebuild from messages on next open: %s", None),
  990. )
  991. def _run_repair_strategies(db_path: Path, report: dict[str, Any]) -> dict[str, Any]:
  992. """Escalating repair attempts, applied to *db_path* IN PLACE — only ever a scratch copy nothing else holds open,
  993. never the user's database. The "could not recover" log lives in the caller so it names the user's database."""
  994. for name, body, success_msg, failure_msg in _REPAIR_STRATEGIES:
  995. try:
  996. with _repair_conn(db_path) as conn:
  997. body(conn)
  998. reason = _db_opens_cleanly(db_path)
  999. except sqlite3.DatabaseError as exc:
  1000. reason = str(exc)
  1001. if failure_msg is not None:
  1002. logger.warning(failure_msg, exc)
  1003. if reason is None:
  1004. report["repaired"], report["strategy"] = True, name
  1005. logger.warning(success_msg, db_path)
  1006. return report
  1007. if failure_msg is None:
  1008. report["error"] = reason
  1009. return report