hermes_state_dbfile.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858
  1. """state.db file-level health helpers, split out of ``hermes_state.py``.
  2. Header probes (application_id / zeroed-file detection), deleted-WAL-sidecar
  3. holder scans, quarantine of zeroed databases, ``collect_state_db_stats`` and
  4. holder-process classification. Helpers that hermes_state itself imports and
  5. calls (``_connect_tracked_db`` & co) are looked up lazily from ``hermes_state`` at
  6. call time, so tests that monkeypatch ``hermes_state.<name>`` keep intercepting.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import errno
  11. import hashlib
  12. import json
  13. import logging
  14. import os
  15. import shutil
  16. import sqlite3
  17. import struct
  18. import sys
  19. import threading
  20. import time
  21. from pathlib import Path
  22. from typing import Any, Callable, Dict, List, Optional, Tuple
  23. from hermes_state_holders import canonical_sqlite_path, read_only_db_uri
  24. from hermes_state_common import (
  25. FTS_REBUILD_DEFERRAL_KEY, stat_db_file_identity as _stat_db_file_identity
  26. )
  27. # Log-record parity with the origin module (caplog tests pin "hermes_state").
  28. logger = logging.getLogger("hermes_state")
  29. def _prepare_connection_retirement():
  30. """Bind a non-finalizing reference before opening a writable SQLite handle.
  31. A Python container is cleared during interpreter shutdown. An unmatched
  32. CPython C reference keeps the exact connection alive through that cleanup,
  33. so SQLite cannot checkpoint its lost WAL generation from a finalizer. This
  34. deliberately retains the connection and its descriptors until process exit;
  35. it does not make already-unlinked WAL data durable after the last fd closes.
  36. """
  37. message = (
  38. "Writable SessionDB on a Python without sqlite3 setconfig (< 3.12) requires CPython with "
  39. "ctypes support to retain a quarantined SQLite connection through interpreter shutdown."
  40. )
  41. if sys.implementation.name != "cpython":
  42. raise RuntimeError(message)
  43. try:
  44. import ctypes
  45. # A private function object avoids changing another caller's signature.
  46. retain = ctypes.pythonapi["Py_IncRef"]
  47. retain.argtypes = (ctypes.py_object,)
  48. retain.restype = None
  49. except (ImportError, AttributeError, OSError) as exc:
  50. raise RuntimeError(message) from exc
  51. return retain
  52. # _read_sqlite_application_id runs on EVERY write (_raise_if_db_replaced) against the LIVE
  53. # state.db. A bare open()/read()/close() there is the howtocorrupt §2.2 bug: close() cancels
  54. # every POSIX advisory lock this process holds on the file, dropping the writer's WAL-mode DMS
  55. # shared lock (see hermes_cli/sqlite_safe_read.py) so another process can treat this writer as
  56. # dead and rerun WAL-index recovery underneath it. So the probe preads through a per-path fd
  57. # cached for the life of the process (opening never cancels locks). When the path is re-pointed
  58. # at a new inode (the very replacement this probe detects) the stale fd is RETIRED, never closed
  59. # — closing it would cancel the live connection's locks. Replacements are rare and halt writes.
  60. _HEADER_PROBE_LOCK = threading.Lock()
  61. _HEADER_PROBE_FDS: dict[str, tuple[int, int, int]] = {} # key -> (fd, dev, ino)
  62. _RETIRED_HEADER_PROBE_FDS: list[int] = [] # intentionally never closed
  63. _FTS_TABLE_NAMES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
  64. def _pread_db_range(db_path: Path, offset: int, length: int) -> Optional[bytes]:
  65. """Lock-safe raw read of a possibly-live SQLite database: POSIX preads from a cached,
  66. never-closed fd (rebound when the path names a new inode); Windows reads plainly, since
  67. advisory-lock cancellation is a POSIX-only hazard."""
  68. from hermes_state import _IS_WINDOWS
  69. if _IS_WINDOWS:
  70. with contextlib.suppress(OSError), db_path.open("rb") as handle:
  71. handle.seek(offset)
  72. return handle.read(length)
  73. return None
  74. key = str(db_path)
  75. try:
  76. st = os.stat(db_path)
  77. except OSError:
  78. return None
  79. with _HEADER_PROBE_LOCK:
  80. cached = _HEADER_PROBE_FDS.get(key)
  81. if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino):
  82. # Path re-pointed at a new file. Retire (never close) the old fd.
  83. _RETIRED_HEADER_PROBE_FDS.append(_HEADER_PROBE_FDS.pop(key)[0])
  84. cached = None
  85. if cached is None:
  86. try:
  87. fd = os.open(db_path, os.O_RDONLY)
  88. except OSError:
  89. return None
  90. try:
  91. fst = os.fstat(fd)
  92. except OSError:
  93. _RETIRED_HEADER_PROBE_FDS.append(fd)
  94. return None
  95. cached = _HEADER_PROBE_FDS[key] = (fd, fst.st_dev, fst.st_ino)
  96. with contextlib.suppress(OSError):
  97. return os.pread(cached[0], length, offset)
  98. return None
  99. def _pread_db_header(db_path: Path, length: int) -> Optional[bytes]:
  100. """Lock-safe raw header read of a possibly-live SQLite database (see :func:`_pread_db_range`)."""
  101. return _pread_db_range(db_path, 0, length)
  102. def _read_sqlite_application_id(db_path: Path) -> Optional[int]:
  103. """application_id from the SQLite header, via the lock-safe :func:`_pread_db_header`."""
  104. from hermes_state_errors import _STATE_DB_APPLICATION_ID_OFFSET
  105. end = _STATE_DB_APPLICATION_ID_OFFSET + 4
  106. header = _pread_db_header(db_path, end)
  107. if header is None or len(header) < end or header[:16] != b"SQLite format 3\x00":
  108. return None
  109. return int(struct.unpack(">I", header[_STATE_DB_APPLICATION_ID_OFFSET:end])[0])
  110. def _stat_sqlite_sidecar_identity(db_path: Path) -> dict[str, tuple]:
  111. """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars."""
  112. base = os.fspath(db_path)
  113. idents = {suffix: _stat_db_file_identity(Path(base + suffix)) for suffix in ("-wal", "-shm")}
  114. return {suffix: ident for suffix, ident in idents.items() if ident is not None}
  115. def _watched_sqlite_sidecar_paths(db_path) -> dict[str, str]:
  116. """Map each sidecar's canonical (/proc-comparable) form to its literal, still-named path,
  117. so a canonical match can be re-``stat``'d for identity rather than trusted as text."""
  118. literal_base = os.path.abspath(os.fspath(db_path))
  119. literal = (literal_base + "-wal", literal_base + "-shm")
  120. watched = {canonical_sqlite_path(path): path for path in literal}
  121. # /proc reports the kernel-resolved dentry, so the watched canonicals must also resolve
  122. # symlinks -- with abspath alone a symlinked HERMES_HOME makes every deleted sidecar
  123. # invisible to the scan. Both spellings are watched: the fully resolved path, which is
  124. # where current SQLite places -wal/-shm when the database file itself is a symlink, and
  125. # the realpath'd parent with the literal basename, which is where they land when SQLite
  126. # names the sidecars after the path it was opened through.
  127. resolved_bases = (
  128. os.path.join(os.path.realpath(os.path.dirname(literal_base)),
  129. os.path.basename(literal_base)),
  130. os.path.realpath(literal_base),
  131. )
  132. for base in resolved_bases:
  133. for suffix in ("-wal", "-shm"):
  134. watched.setdefault(canonical_sqlite_path(base + suffix), base + suffix)
  135. return watched
  136. def _identity_is_truly_unlinked(identity: tuple[int, int], watched_path: str) -> bool:
  137. """The shared verdict: does ``(st_dev, st_ino)`` name a generation the watched path no longer
  138. holds? See :func:`_fd_is_truly_unlinked` for why the test is identity and never link count."""
  139. try:
  140. watched_stat = os.stat(watched_path)
  141. except OSError:
  142. return True
  143. return identity != (watched_stat.st_dev, watched_stat.st_ino)
  144. def _fd_is_truly_unlinked(fd_path: str, watched_path: str) -> bool:
  145. """Confirm a `` (deleted)`` /proc fd target really names an orphaned generation, not the
  146. CURRENT watched sidecar.
  147. The suffix alone is not proof: on OpenZFS a live, still-linked file whose dentry was
  148. unhashed is reported as deleted while it is still the very same inode the watched path
  149. names. Conversely ``st_nlink == 0`` is not proof of the opposite — a stale generation can
  150. keep a surviving hard link (a backup, an operator copy) after the watched path itself is
  151. removed or replaced, leaving ``st_nlink >= 1`` on a truly orphaned inode. So compare
  152. identity, not link count: only an exact ``(st_dev, st_ino)`` match between the fd and the
  153. CURRENT watched path proves they are the same live file. A mismatch, or a watched path
  154. that cannot be stat'd at all, means the fd holds a generation the watched path no longer
  155. names — the guard keeps failing closed."""
  156. try:
  157. fd_stat = os.stat(fd_path)
  158. except OSError as exc:
  159. # ENOENT: the descriptor was closed after /proc was read. ESRCH: the whole
  160. # process exited mid-scan. Neither can keep a retired generation alive, so
  161. # do not turn this scan race into a refusal (mirrors hermes_state_holders).
  162. if exc.errno in (errno.ENOENT, errno.ESRCH):
  163. return False
  164. return True
  165. return _identity_is_truly_unlinked((fd_stat.st_dev, fd_stat.st_ino), watched_path)
  166. def _iter_proc_fd_targets():
  167. """Yield ``(pid, readlink target, fd path)`` for every readable ``/proc/<pid>/fd`` entry."""
  168. for pid_str in os.listdir("/proc"):
  169. if not pid_str.isdigit():
  170. continue
  171. fd_dir = f"/proc/{pid_str}/fd"
  172. try:
  173. fds = os.listdir(fd_dir)
  174. except OSError:
  175. continue # process gone or not ours
  176. for fd in fds:
  177. with contextlib.suppress(OSError):
  178. fd_path = f"{fd_dir}/{fd}"
  179. yield int(pid_str), os.readlink(fd_path), fd_path
  180. # ── macOS fd enumeration (libproc) ──────────────────────────────────────────────────────────────
  181. #
  182. # macOS has no ``/proc`` and never reports a `` (deleted)`` suffix, but libproc can still describe
  183. # ANOTHER process's descriptors: ``proc_listpids`` names the processes, ``proc_pidinfo
  184. # (PROC_PIDLISTFDS)`` lists one process's fds, and ``proc_pidfdinfo(PROC_PIDFDVNODEPATHINFO)``
  185. # returns, for a vnode fd, its ``(st_dev, st_ino)`` together with the vnode's last pathname. Both
  186. # survive unlink (the path keeps its last name, ``st_nlink`` drops to 0), for same-user processes
  187. # without elevation — which is the darwin counterpart of the Linux readlink leg, and the reason
  188. # ``psutil.Process.open_files()`` cannot stand in for it (it hides unlinked descriptors). The
  189. # identity judgement is shared (:func:`_identity_is_truly_unlinked`); only its source differs.
  190. _DARWIN_ALL_PIDS = 1
  191. _DARWIN_PIDLISTFDS = 1
  192. _DARWIN_PIDFDVNODEPATHINFO = 2
  193. _DARWIN_PROC_FD_INFO_SIZE = 8 # struct proc_fdinfo { int32 proc_fd; uint32 proc_fdtype; }
  194. _DARWIN_FD_RECORD_SIZE = 1200 # PROC_PIDFDVNODEPATHINFO_SIZE
  195. # Field offsets inside that 1200-byte record. libproc does not lay the header's structs out
  196. # verbatim (it pads ``vnode_info`` and sizes the record to 1200, not to ``sizeof``'s 1192), so
  197. # these are pinned against live descriptors rather than derived from <libproc.h>. ``vst_dev`` /
  198. # ``vst_ino`` are the same two fields the Linux leg compares; the path is the vnode's last
  199. # pathname, resolved by the kernel (``/private/var/...`` where the caller opened ``/var/...``).
  200. _DARWIN_FD_DEV_OFFSET = 24
  201. _DARWIN_FD_INO_OFFSET = 32
  202. _DARWIN_FD_PATH_OFFSET = 176
  203. _DARWIN_LIBPROC = None
  204. # Wall-clock budget for one darwin holder-scan pass (#113187). A single
  205. # proc_pidfdinfo call can block uninterruptibly in the kernel (a process stuck
  206. # in uninterruptible I/O on a dead network share), so no per-iteration check
  207. # can bound it: the whole pass runs in a daemon thread and is abandoned on
  208. # expiry, failing open per the guard's contract.
  209. _DARWIN_FD_SCAN_TIMEOUT_SECONDS = 5.0
  210. def _darwin_libproc():
  211. """libproc's fd-enumeration entry points, loaded once per process.
  212. macOS exports these from libSystem, so this process's own handle resolves them; a failure
  213. here propagates to the caller's ``except Exception`` and keeps the guard fail-open."""
  214. global _DARWIN_LIBPROC
  215. lib = _DARWIN_LIBPROC
  216. if lib is not None:
  217. return lib
  218. import ctypes
  219. lib = ctypes.CDLL(None, use_errno=True)
  220. lib.proc_listpids.restype = ctypes.c_int
  221. lib.proc_listpids.argtypes = (ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_uint32)
  222. lib.proc_pidinfo.restype = ctypes.c_int
  223. lib.proc_pidinfo.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int)
  224. lib.proc_pidfdinfo.restype = ctypes.c_int
  225. lib.proc_pidfdinfo.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_int)
  226. _DARWIN_LIBPROC = lib
  227. return lib
  228. def _darwin_all_pids(lib) -> list[int]:
  229. """Every pid ``proc_listpids`` will name (the kernel silently omits ones we may not inspect)."""
  230. import ctypes
  231. size = 4096 * 8
  232. while True:
  233. buffer = ctypes.create_string_buffer(size)
  234. used = lib.proc_listpids(_DARWIN_ALL_PIDS, 0, buffer, size)
  235. if used <= 0:
  236. return []
  237. if used < size:
  238. return [pid for pid in struct.unpack_from(f"<{used // 4}i", buffer.raw) if pid > 0]
  239. size *= 2
  240. def _iter_darwin_fd_targets():
  241. """Yield ``(pid, fd, last pathname, (st_dev, st_ino))`` for every vnode fd libproc reports.
  242. The pathname and the identity both stay readable after the path is unlinked, which is what
  243. makes an orphaned WAL generation visible at all on macOS. Processes that cannot be inspected
  244. (gone, or not ours) and descriptors that are not vnodes are skipped silently."""
  245. import ctypes
  246. lib = _darwin_libproc()
  247. for pid in _darwin_all_pids(lib):
  248. size = 4096
  249. while True:
  250. listing = ctypes.create_string_buffer(size)
  251. used = lib.proc_pidinfo(pid, _DARWIN_PIDLISTFDS, 0, listing, size)
  252. if used <= 0:
  253. break # process gone, or not ours to inspect
  254. if used < size:
  255. break
  256. size *= 2
  257. else:
  258. continue
  259. for offset in range(0, used - _DARWIN_PROC_FD_INFO_SIZE + 1, _DARWIN_PROC_FD_INFO_SIZE):
  260. fd = struct.unpack_from("<i", listing.raw, offset)[0]
  261. record = ctypes.create_string_buffer(_DARWIN_FD_RECORD_SIZE)
  262. if lib.proc_pidfdinfo(pid, fd, _DARWIN_PIDFDVNODEPATHINFO, record,
  263. _DARWIN_FD_RECORD_SIZE) <= 0:
  264. continue
  265. raw = record.raw
  266. identity = (struct.unpack_from("<I", raw, _DARWIN_FD_DEV_OFFSET)[0],
  267. struct.unpack_from("<Q", raw, _DARWIN_FD_INO_OFFSET)[0])
  268. target = raw[_DARWIN_FD_PATH_OFFSET:].split(b"\x00", 1)[0].decode("utf-8", "replace")
  269. yield pid, fd, target, identity
  270. def _iter_darwin_sidecar_holders(db_path) -> list[tuple[int, str]]:
  271. """The macOS leg of :func:`iter_deleted_sqlite_sidecar_holders`: libproc enumeration matched
  272. against the watched sidecar paths, judged by identity.
  273. The watched side is resolved with ``os.path.realpath`` because libproc reports the kernel's
  274. path for the vnode, while ``os.path.abspath`` does not resolve symlinks -- a textual compare
  275. of the two silently misses every sidecar under a symlinked prefix (on macOS ``/var`` itself)."""
  276. base = os.path.realpath(os.path.abspath(os.fspath(db_path)))
  277. # APFS/HFS+ are case-insensitive by default and libproc reports the pathname as the opener
  278. # spelled it; ``os.path.normcase`` is the identity on darwin, so fold case here.
  279. watched = {path.casefold(): path for path in (base + "-wal", base + "-shm")}
  280. holders: list[tuple[int, str]] = []
  281. errors: list[BaseException] = []
  282. def _scan() -> None:
  283. try:
  284. for pid, _fd, target, identity in _iter_darwin_fd_targets():
  285. literal = watched.get(target.casefold())
  286. if literal is not None and _identity_is_truly_unlinked(identity, literal):
  287. holders.append((pid, target))
  288. except BaseException as exc:
  289. errors.append(exc)
  290. # ponytail: abandoned daemon thread per timed-out scan; a kernel-blocked
  291. # proc_pidfdinfo cannot be interrupted, so expiry leaks one thread that
  292. # dies with the process. Fail open: partial/empty holders, never a refusal.
  293. worker = threading.Thread(target=_scan, daemon=True)
  294. worker.start()
  295. worker.join(_DARWIN_FD_SCAN_TIMEOUT_SECONDS)
  296. if worker.is_alive():
  297. logger.warning(
  298. "deleted-WAL holder scan timed out after %.1fs for %s; continuing fail-open",
  299. _DARWIN_FD_SCAN_TIMEOUT_SECONDS,
  300. db_path,
  301. )
  302. return []
  303. if errors:
  304. raise errors[0]
  305. return holders
  306. def iter_deleted_sqlite_sidecar_holders(db_path) -> list[tuple[int, str]]:
  307. """Return processes holding an unlinked ``state.db-wal`` / ``-shm`` sidecar for *db_path*.
  308. Linux enumerates ``/proc/<pid>/fd`` (using the `` (deleted)`` suffix as a cheap pre-filter);
  309. macOS enumerates descriptors through libproc, because a retired generation there keeps its
  310. last pathname and no suffix marks it. Either way the verdict is the same identity comparison:
  311. the fd must name a watched sidecar path while its ``(st_dev, st_ino)`` no longer matches what
  312. that path holds. Windows returns ``[]`` -- it cannot unlink a held sidecar, so no retired
  313. generation can exist; any other platform returns ``[]`` as before.
  314. Includes this process: on the open/write refuse path the in-process writer holding the orphan
  315. inode must not mint a replacement WAL (``_foreign_state_db_holders`` skips this PID)."""
  316. if sys.platform == "win32":
  317. return []
  318. holders: list[tuple[int, str]] = []
  319. try:
  320. if sys.platform == "darwin":
  321. holders = _iter_darwin_sidecar_holders(db_path)
  322. elif sys.platform.startswith("linux"):
  323. watched = _watched_sqlite_sidecar_paths(db_path)
  324. for pid, target, fd_path in _iter_proc_fd_targets():
  325. canonical = canonical_sqlite_path(target)
  326. if (" (deleted)" in target and canonical in watched
  327. and _fd_is_truly_unlinked(fd_path, watched[canonical])):
  328. holders.append((pid, target))
  329. except Exception as exc:
  330. logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc)
  331. return holders
  332. def refuse_deleted_wal_generation(db_path) -> None:
  333. """Raise if any process holds a deleted WAL/SHM generation for *db_path*; called
  334. *before* ``sqlite3.connect`` so a second opener cannot mint a replacement WAL inode."""
  335. from hermes_state import DeletedWalGenerationError
  336. from hermes_state_errors import _DELETED_WAL_GENERATION_MSG
  337. if not iter_deleted_sqlite_sidecar_holders(db_path):
  338. return
  339. logger.error(_DELETED_WAL_GENERATION_MSG)
  340. raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
  341. # ── Retired WAL generation capture ──────────────────────────────────────────────────────────────
  342. #
  343. # When a writer's -wal/-shm generation is deleted or replaced underneath it, the frames committed only in
  344. # that WAL survive exactly as long as this process keeps the unlinked inode open. The quarantine (#105670)
  345. # stops them from being checkpointed under wrong page numbers, but the canonical remediation -- "stop the
  346. # writers, then reopen" -- lets the kernel drop the inode with them. So the exact generation is captured
  347. # durably first: located by the recorded (st_dev, st_ino) among this process's OWN descriptors (never by
  348. # pathname, so a sibling's deleted WAL or a newer sidecar minted at the same path cannot be mistaken for
  349. # it), read with pread (no descriptor is closed, moved or truncated) and written with fsync.
  350. RETIRED_GENERATION_DIR_SUFFIX = ".retired-wal-"
  351. RETIRED_GENERATION_MANIFEST = "manifest.json"
  352. RETIRED_GENERATION_MANIFEST_VERSION = 1
  353. # Up to this size the main image is copied whole, so the artifact is a self-contained state.db + -wal
  354. # pair that `hermes sessions recover --source <dir>/state.db` can open. Above it only the 100-byte
  355. # header is kept (the manifest says so): unlike the WAL inode, the main file survives process exit at
  356. # its path, and a multi-GB copy inside a shutdown path is a worse failure than a header-only artifact.
  357. RETIRED_GENERATION_MAIN_IMAGE_MAX_BYTES = 512 * 1024 * 1024
  358. _CAPTURE_CHUNK_BYTES = 8 * 1024 * 1024
  359. _SQLITE_HEADER_BYTES = 100
  360. class RetiredGenerationCaptureError(RuntimeError):
  361. """The retired WAL generation could not be captured durably; nothing was mutated."""
  362. def _fsync_path(path: Path) -> None:
  363. """fsync a file or directory we own. Never used on the live database: opening and closing a
  364. descriptor on a file SQLite has locked would cancel this process's POSIX advisory locks."""
  365. from hermes_state import _IS_WINDOWS
  366. if _IS_WINDOWS:
  367. return # directories cannot be opened; file writes fsync their own handle
  368. fd = os.open(path, os.O_RDONLY)
  369. try:
  370. os.fsync(fd)
  371. finally:
  372. os.close(fd)
  373. def _own_descriptor_for_identity(identity) -> Optional[int]:
  374. """This process's open descriptor for the inode ``(st_dev, st_ino)``, or None.
  375. Exact-generation qualified: pathnames are never consulted. The descriptor stays owned by
  376. SQLite; callers only pread from it."""
  377. if not identity:
  378. return None
  379. wanted = tuple(identity)
  380. for fd_dir in ("/proc/self/fd", "/dev/fd"):
  381. try:
  382. names = os.listdir(fd_dir)
  383. except OSError:
  384. continue
  385. for name in names:
  386. if not name.isdigit():
  387. continue
  388. try:
  389. st = os.fstat(int(name))
  390. except OSError:
  391. continue
  392. if (st.st_dev, st.st_ino) == wanted:
  393. return int(name)
  394. return None # the table was readable: a definitive miss
  395. return None
  396. def _copy_range(read: Callable[[int, int], Optional[bytes]], dest: Path, *, size: int) -> dict[str, Any]:
  397. """Stream ``size`` bytes via ``read(offset, length)`` into ``dest`` (temp file, fsync, rename).
  398. A short read raises ``RetiredGenerationCaptureError``: a truncated copy with a valid-looking
  399. manifest would let the caller settle the handle while the unlinked inode still dies at exit."""
  400. digest = hashlib.sha256()
  401. part = dest.with_name(dest.name + ".part")
  402. offset = 0
  403. try:
  404. with open(part, "wb") as out:
  405. while offset < size:
  406. chunk = read(offset, min(_CAPTURE_CHUNK_BYTES, size - offset))
  407. if not chunk:
  408. break
  409. out.write(chunk)
  410. digest.update(chunk)
  411. offset += len(chunk)
  412. out.flush()
  413. os.fsync(out.fileno())
  414. except Exception:
  415. part.unlink(missing_ok=True)
  416. raise
  417. if offset != size:
  418. part.unlink(missing_ok=True)
  419. raise RetiredGenerationCaptureError(
  420. f"short read copying {dest.name}: {offset} of {size} bytes — the retired generation "
  421. "is NOT fully captured; the handle stays open for a retry"
  422. )
  423. os.replace(part, dest)
  424. return {"file": dest.name, "bytes": offset, "sha256": digest.hexdigest()}
  425. def _copy_descriptor(fd: int, dest: Path, *, size: int) -> dict[str, Any]:
  426. """pread ``fd`` (a descriptor SQLite owns; never closed here) into ``dest``."""
  427. return _copy_range(lambda offset, length: os.pread(fd, length, offset), dest, size=size)
  428. def _copy_main_image(db_path: Path, dest: Path, *, size: int) -> dict[str, Any]:
  429. """Copy the live main file through the lock-safe cached descriptor (see ``_pread_db_range``)."""
  430. return _copy_range(lambda offset, length: _pread_db_range(db_path, offset, length), dest, size=size)
  431. def _parse_sqlite_header(header: bytes) -> dict[str, Any]:
  432. if len(header) < _SQLITE_HEADER_BYTES or header[:16] != b"SQLite format 3\x00":
  433. return {"valid": False}
  434. raw_page_size = struct.unpack(">H", header[16:18])[0]
  435. from hermes_state_errors import _STATE_DB_APPLICATION_ID_OFFSET
  436. fields = {"change_counter": 24, "page_count": 28, "user_version": 60,
  437. "application_id": _STATE_DB_APPLICATION_ID_OFFSET, "version_valid_for": 92}
  438. parsed = {name: struct.unpack(">I", header[off:off + 4])[0] for name, off in fields.items()}
  439. return {"valid": True, "page_size": 65536 if raw_page_size == 1 else raw_page_size, **parsed}
  440. def capture_retired_wal_generation(
  441. db_path, *, sidecar_identity: dict[str, tuple], trigger: str,
  442. ) -> Path:
  443. """Durably capture the lost WAL generation this process still holds open; return the artifact dir.
  444. The artifact sits next to the database as ``<name>.retired-wal-<utc ts>-<pid>/`` and holds the
  445. retired ``<name>-wal`` (and ``-shm`` when still ours), the main image (or its header past the size
  446. cap) and ``manifest.json`` with identities, sizes, digests and the sidecar generation found at the
  447. path at capture time. Nothing is merged: whether those frames belong on top of the main file is
  448. the operator's decision. Raises :class:`RetiredGenerationCaptureError` when the exact generation
  449. cannot be located or written; no descriptor is ever closed, moved or truncated.
  450. """
  451. db_path = Path(db_path)
  452. wal_identity = tuple(sidecar_identity.get("-wal") or ())
  453. if not wal_identity:
  454. raise RetiredGenerationCaptureError(
  455. f"no recorded WAL generation identity for {db_path}; refusing to guess it by pathname")
  456. wal_fd = _own_descriptor_for_identity(wal_identity)
  457. if wal_fd is None:
  458. raise RetiredGenerationCaptureError(
  459. f"this process no longer holds the retired WAL inode {wal_identity} of {db_path}")
  460. try:
  461. wal_size = os.fstat(wal_fd).st_size
  462. except OSError as exc:
  463. raise RetiredGenerationCaptureError(f"cannot stat the retired WAL of {db_path}: {exc}") from exc
  464. stem = f"{db_path.name}{RETIRED_GENERATION_DIR_SUFFIX}{time.strftime('%Y%m%d-%H%M%S', time.gmtime())}-{os.getpid()}"
  465. final = db_path.with_name(stem)
  466. n = 0
  467. while final.exists() or final.with_name(final.name + ".partial").exists():
  468. n += 1
  469. final = db_path.with_name(f"{stem}-{n}")
  470. staging = final.with_name(final.name + ".partial")
  471. try:
  472. staging.mkdir(parents=True, exist_ok=False)
  473. manifest: dict[str, Any] = {
  474. "version": RETIRED_GENERATION_MANIFEST_VERSION,
  475. "database": str(db_path),
  476. "trigger": trigger,
  477. "pid": os.getpid(),
  478. "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
  479. "python": sys.version.split()[0],
  480. "sqlite": sqlite3.sqlite_version,
  481. "wal": {"identity": list(wal_identity),
  482. **_copy_descriptor(wal_fd, staging / (db_path.name + "-wal"), size=wal_size)},
  483. "shm": None,
  484. # The generation living at the path when we captured: a newer writer's, if one was minted.
  485. "path_generation_at_capture": {
  486. suffix: list(ident) for suffix, ident in _stat_sqlite_sidecar_identity(db_path).items()},
  487. "note": ("Frames in the captured WAL were committed by the retired generation. Whether they "
  488. "belong on top of the main file now at the path is an operator decision; inspect "
  489. "the copied image with `hermes sessions recover --inspect-only` first."),
  490. }
  491. shm_identity = tuple(sidecar_identity.get("-shm") or ())
  492. shm_fd = _own_descriptor_for_identity(shm_identity) if shm_identity else None
  493. if shm_fd is not None:
  494. with contextlib.suppress(OSError): # SQLite rebuilds the index; the WAL is what matters
  495. manifest["shm"] = {"identity": list(shm_identity), **_copy_descriptor(
  496. shm_fd, staging / (db_path.name + "-shm"), size=os.fstat(shm_fd).st_size)}
  497. header = _pread_db_range(db_path, 0, _SQLITE_HEADER_BYTES)
  498. if header is None:
  499. raise RetiredGenerationCaptureError(f"cannot read the main image header of {db_path}")
  500. main_size = os.stat(db_path).st_size
  501. main: dict[str, Any] = {"identity": list(_stat_db_file_identity(db_path) or ()) or None,
  502. "size": main_size, "header": _parse_sqlite_header(header)}
  503. if main_size <= RETIRED_GENERATION_MAIN_IMAGE_MAX_BYTES:
  504. main.update(mode="copied", **_copy_main_image(db_path, staging / db_path.name, size=main_size))
  505. else:
  506. header_file = staging / (db_path.name + ".header")
  507. header_file.write_bytes(header)
  508. _fsync_path(header_file)
  509. main.update(mode="header_only", file=header_file.name, bytes=len(header))
  510. manifest["main"] = main
  511. # Late import: utils pulls agent-side deps; the capture must stay dependency-light.
  512. from utils import atomic_json_write
  513. atomic_json_write(staging / RETIRED_GENERATION_MANIFEST, manifest, sort_keys=True)
  514. _fsync_path(staging)
  515. os.replace(staging, final)
  516. _fsync_path(final.parent)
  517. except RetiredGenerationCaptureError:
  518. shutil.rmtree(staging, ignore_errors=True)
  519. raise
  520. except OSError as exc:
  521. shutil.rmtree(staging, ignore_errors=True)
  522. raise RetiredGenerationCaptureError(
  523. f"could not write the retired WAL generation of {db_path} under {staging}: {exc}") from exc
  524. return final
  525. def _connect_tracked_db(path, tracking_path=None, **kwargs):
  526. """``sqlite3.connect`` that registers the open fd so byte-level probes of a live file are
  527. refused (an ``open()``/``close()`` would cancel every POSIX lock, even a running VACUUM's
  528. EXCLUSIVE). The ONLY tolerated fallback is the helper being absent (scaffold/embed installs
  529. without hermes_cli); a real connection failure must propagate — a silent untracked retry
  530. would disable the guard for that connection."""
  531. try:
  532. from hermes_cli.sqlite_safe_read import connect_tracked
  533. except ImportError:
  534. logger.debug("hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
  535. "(byte-probe guard inactive in this install)", path)
  536. return sqlite3.connect(str(path), **kwargs)
  537. # Open through THIS module's sqlite3.connect so tests patching hermes_state.sqlite3.connect keep control.
  538. return connect_tracked(path, tracking_path=tracking_path, connect_fn=sqlite3.connect, **kwargs)
  539. def _preopen_header(path: Path, probe_bytes: int, force: bool) -> Optional[bytes]:
  540. """First ``probe_bytes`` of *path* read WITHOUT opening a SQLite connection, or None when the probe
  541. must not run: special files (a FIFO would block), unreadable paths, or — unless ``force`` — a path
  542. this process already has a connection to (``close()`` cancels every POSIX lock, even a running
  543. VACUUM's EXCLUSIVE; see #97568). ``force=True`` only for offline files (quarantined copies, snapshots)."""
  544. try:
  545. if not path.is_file():
  546. return None
  547. path.stat()
  548. from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
  549. if not force and has_live_connection(path):
  550. return None
  551. return read_header_bytes_preopen(path, length=max(16, probe_bytes), force=force)
  552. except Exception:
  553. return None
  554. def is_zeroed_state_db(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
  555. """Detect the zeroed state.db signature (0-byte or NUL header). Prefers
  556. ``hermes_cli.backup.is_zeroed_sqlite_file``; this copy keeps SessionDB openable without the CLI
  557. package in constrained embed paths."""
  558. with contextlib.suppress(Exception):
  559. from hermes_cli.backup import is_zeroed_sqlite_file
  560. return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
  561. head = _preopen_header(path, probe_bytes, force)
  562. # b"" (0-byte file) is zeroed; all() over an empty header is True.
  563. return head is not None and not head.startswith(b"SQLite format 3") and all(b == 0 for b in head)
  564. def has_invalid_sqlite_header_preopen(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
  565. """A pre-existing state.db whose first page is not SQLite: 0-byte, NUL, or clobbered page 0
  566. (#102198). Zeroed files are the subset :func:`is_zeroed_state_db` names."""
  567. head = _preopen_header(path, probe_bytes, force)
  568. return head is not None and not head.startswith(b"SQLite format 3")
  569. @contextlib.contextmanager
  570. def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
  571. """Acquire the cross-process lock for path.quarantine.lock."""
  572. import platform
  573. lock_path = path.with_name(path.name + ".quarantine.lock")
  574. lock_path.parent.mkdir(parents=True, exist_ok=True)
  575. handle = lock_path.open("a+b")
  576. acquired = False
  577. try:
  578. if platform.system() == "Windows":
  579. import msvcrt
  580. def _lock(mode): # msvcrt locks a byte range from the current position
  581. handle.seek(0)
  582. msvcrt.locking(handle.fileno(), mode, 1)
  583. _try_lock = lambda: _lock(msvcrt.LK_NBLCK)
  584. _unlock = lambda: _lock(msvcrt.LK_UNLCK)
  585. else:
  586. import fcntl
  587. _try_lock = lambda: fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  588. _unlock = lambda: fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
  589. deadline = time.monotonic() + timeout
  590. while not acquired:
  591. try:
  592. _try_lock()
  593. acquired = True
  594. except OSError:
  595. if time.monotonic() >= deadline:
  596. break
  597. time.sleep(0.020)
  598. yield acquired
  599. finally:
  600. try:
  601. if acquired:
  602. _unlock()
  603. except (OSError, AttributeError):
  604. pass
  605. finally:
  606. handle.close()
  607. def quarantine_invalid_state_db(path: Path, *, already_locked: bool = False) -> Optional[Path]:
  608. """Move a non-SQLite state.db (zeroed or clobbered page 0) aside with its -wal/-shm sidecars,
  609. preserving bytes; return the quarantine path. A cross-process lock stops two concurrent
  610. startups racing: the second re-checks under the lock and finds the file gone (or fresh)."""
  611. def _do_quarantine():
  612. if not path.exists():
  613. logger.info("quarantine_invalid_state_db: %s already moved by another process", path)
  614. return None
  615. if not has_invalid_sqlite_header_preopen(path):
  616. logger.info("quarantine_invalid_state_db: %s is no longer invalid (another "
  617. "process quarantined it and a fresh DB was created)", path)
  618. return None
  619. kind = "zeroed" if is_zeroed_state_db(path) else "notadb"
  620. try:
  621. ts = time.strftime("%Y%m%d-%H%M%S")
  622. except Exception:
  623. ts = "unknown"
  624. stem = f"{path.name}.{kind}-{ts}-{os.getpid()}"
  625. dest = path.with_name(f"{stem}.bak")
  626. n = 0
  627. while dest.exists():
  628. n += 1
  629. dest = path.with_name(f"{stem}-{n}.bak")
  630. try:
  631. path.rename(dest)
  632. except OSError as exc:
  633. logger.error("Failed to quarantine invalid %s: %s", path, exc)
  634. return None
  635. for suffix in ("-wal", "-shm"):
  636. side = Path(str(path) + suffix)
  637. if side.exists():
  638. with contextlib.suppress(OSError):
  639. side.rename(Path(str(dest) + suffix))
  640. return dest
  641. if already_locked:
  642. return _do_quarantine()
  643. with quarantine_cross_process_lock(path) as acquired:
  644. if not acquired:
  645. logger.error("quarantine lock for %s not acquired within 5s — refusing to "
  646. "quarantine without the cross-process lock. The invalid file "
  647. "is left in place. If sessions fail to load, run `hermes sessions recover "
  648. "--source <state.db> --inspect-only`, or restore a snapshot with "
  649. "`/snapshot list` / `/snapshot restore <id>` (terminal `hermes` chat only).",
  650. path)
  651. return None
  652. return _do_quarantine()
  653. def collect_state_db_stats(db_path: Path) -> dict[str, Any]:
  654. """Best-effort, strictly read-only stats snapshot of a state.db file: ``mode=ro`` with a short
  655. timeout so it can run against a *live* database without taking a write lock. Every field is
  656. collected independently (a failed pragma/SELECT yields ``None`` for it); never raises.
  657. Deliberately does NOT instantiate :class:`SessionDB` — its constructor runs DDL.
  658. ``wal_size_bytes`` is 0 when the sidecar is absent; ``fts_storage_version`` None means the
  659. legacy inline layout; ``fts_rebuild_deferral`` is the durable blocked-repair diagnostic."""
  660. from hermes_state import _connect_tracked_db
  661. stats: dict[str, Any] = dict.fromkeys((
  662. "page_count", "page_size", "freelist_count", "logical_size_bytes", "wal_size_bytes", "journal_mode",
  663. "messages", "sessions", "fts_tables", "fts_storage_version", "fts_storage_upgrade_needed",
  664. "fts_teardown_pending", "fts_rebuild_pending",
  665. "fts_rebuild_high_water", "fts_rebuild_progress", "fts_rebuild_deferral"))
  666. # WAL sidecar size needs no connection at all.
  667. with contextlib.suppress(OSError):
  668. wal_path = Path(str(db_path) + "-wal")
  669. stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
  670. try:
  671. # A short timeout keeps doctor snappy when a writer holds the lock. The tracked connect
  672. # lets byte-probe helpers see this connection and refuse raw opens that would cancel locks.
  673. conn = _connect_tracked_db(read_only_db_uri(db_path), tracking_path=Path(db_path),
  674. uri=True, timeout=2.0)
  675. except Exception as exc:
  676. logger.debug("collect_state_db_stats: cannot open %s read-only: %s", db_path, exc)
  677. return stats
  678. def _scalar(sql: str, params=()) -> Any:
  679. try:
  680. row = conn.execute(sql, params).fetchone()
  681. return row[0] if row else None
  682. except Exception:
  683. return None
  684. def _int(sql: str, params=()) -> Optional[int]:
  685. value = _scalar(sql, params)
  686. return int(value) if value is not None else None
  687. def _meta_int(key: str) -> Optional[int]:
  688. try: # a non-numeric meta value must yield None, not fail the snapshot
  689. return _int("SELECT value FROM state_meta WHERE key = ?", (key,))
  690. except Exception:
  691. return None
  692. try:
  693. stats["page_count"] = _int("PRAGMA page_count")
  694. stats["page_size"] = _int("PRAGMA page_size")
  695. if stats["page_count"] is not None and stats["page_size"] is not None:
  696. stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
  697. stats["freelist_count"] = _int("PRAGMA freelist_count")
  698. jm = _scalar("PRAGMA journal_mode")
  699. stats["journal_mode"] = str(jm) if jm is not None else None
  700. stats["messages"] = _int("SELECT COUNT(*) FROM messages")
  701. stats["sessions"] = _int("SELECT COUNT(*) FROM sessions")
  702. # FTS table presence via sqlite_master (never SELECTs from the
  703. # virtual tables themselves — a corrupt index must not fail stats).
  704. with contextlib.suppress(Exception):
  705. names = {row[0] for row in conn.execute(
  706. "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)",
  707. _FTS_TABLE_NAMES).fetchall()}
  708. stats["fts_tables"] = {t: (t in names) for t in _FTS_TABLE_NAMES}
  709. # Raw state_meta reads — cheap, and independent of SessionDB.
  710. stats["fts_storage_version"] = _meta_int("fts_storage_version")
  711. with contextlib.suppress(Exception):
  712. from hermes_state_fts import fts_layout_drift
  713. stats["fts_storage_upgrade_needed"] = bool(fts_layout_drift(conn.cursor()))
  714. with contextlib.suppress(Exception):
  715. # Demoted old-index tables still awaiting teardown: an optimize-storage run stopped after its backfill.
  716. stats["fts_teardown_pending"] = conn.execute(
  717. "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' "
  718. "LIMIT 1").fetchone() is not None
  719. stats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")
  720. stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")
  721. stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_water
  722. with contextlib.suppress(Exception):
  723. row = conn.execute("SELECT value FROM state_meta WHERE key = ? LIMIT 1",
  724. (FTS_REBUILD_DEFERRAL_KEY,)).fetchone()
  725. parsed = json.loads(row[0]) if row else None
  726. if isinstance(parsed, dict):
  727. stats["fts_rebuild_deferral"] = parsed
  728. finally:
  729. with contextlib.suppress(Exception):
  730. conn.close()
  731. return stats
  732. def count_db_holders(db_path: Path) -> Optional[int]:
  733. """Best-effort count of distinct PIDs holding ``db_path`` open (``/proc/*/fd`` on Linux, libproc
  734. on macOS); ``None`` on any error or other host, never raises. Uninspectable processes (other
  735. users' without root) are skipped, so this is a lower bound."""
  736. try:
  737. target = os.path.realpath(str(db_path))
  738. if sys.platform == "darwin":
  739. # Identity, not pathname: libproc reports the vnode's last name as the opener spelled it
  740. # (case, symlinked prefix), which is exactly what the sidecar leg had to case-fold around.
  741. st = os.stat(target)
  742. identity = (st.st_dev, st.st_ino)
  743. return len({pid for pid, _fd, _path, ident in _iter_darwin_fd_targets() if ident == identity})
  744. if not sys.platform.startswith("linux"):
  745. return None
  746. return len({pid for pid, link, _fd_path in _iter_proc_fd_targets() if link == target})
  747. except Exception:
  748. return None
  749. def _is_inactive_orphan_desktop_holder(*, ppid: int, age_seconds: float, min_age_seconds: float,
  750. ephemeral_backend: bool, connection_statuses: list[str]) -> bool:
  751. """Pure safety predicate for the narrow Desktop holder reap."""
  752. return (ppid in (0, 1) and age_seconds >= min_age_seconds and ephemeral_backend
  753. and "ESTABLISHED" not in connection_statuses)
  754. def _concrete_state_db_holder_pids(db_path: Path, holders: list[tuple[int, str]]) -> list[int]:
  755. """Return unique PIDs proven to hold this DB or one of its sidecars."""
  756. canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path)))
  757. watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
  758. return list(dict.fromkeys(
  759. pid for pid, path in holders if pid > 0 and canonical_sqlite_path(path) in watched))