sqlite_safe_read.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. """Lock-safe inspection of SQLite database files.
  2. POSIX advisory locks are cancelled **process-wide** by ``close()`` on *any* fd for that file, so a
  3. bare ``open(db_path, "rb") ... close()`` on a live database drops every lock SQLite holds from this
  4. process (a VACUUM's EXCLUSIVE lock, an in-flight BEGIN IMMEDIATE's RESERVED lock). This module
  5. tracks live connections so raw reads happen only when none exist.
  6. """
  7. from __future__ import annotations
  8. import contextlib
  9. import logging
  10. import os
  11. import sqlite3
  12. import threading
  13. from pathlib import Path
  14. from typing import Optional
  15. logger = logging.getLogger(__name__)
  16. # Guards BOTH the registry and the lifecycle syscalls it describes. Reentrant
  17. # because connect_tracked -> _canonical_db_path -> ... stays on one thread.
  18. _live_lock = threading.RLock()
  19. # canonical path -> number of live connections opened by this process
  20. _live_connections: dict[str, int] = {}
  21. class UntrackableConnectionError(RuntimeError):
  22. """A connection to a probe-able database could not be tracked. Raised rather than returning an
  23. untracked connection: on these paths tracking is part of the correctness contract."""
  24. class LiveConnectionError(RuntimeError):
  25. """A raw file operation was attempted on a database with live connections."""
  26. def _key(path: Path | str) -> str:
  27. """Canonicalise a *filesystem* path for use as a registry key."""
  28. try:
  29. return str(Path(path).resolve())
  30. except OSError:
  31. return str(path)
  32. def _canonical_db_path(conn: sqlite3.Connection) -> Optional[str]:
  33. """The on-disk path of ``main`` as SQLite reports it (immune to ``file:`` URIs, relative paths,
  34. symlinks). ``None`` for in-memory/unnamed databases, which cannot be byte-probed."""
  35. try:
  36. row = conn.execute("PRAGMA database_list").fetchone()
  37. except sqlite3.Error:
  38. return None
  39. if not row or len(row) < 3 or not row[2]:
  40. return None
  41. return _key(row[2])
  42. def track_connection(path: Path | str) -> None:
  43. """Record that this process holds a connection to *path* (prefer :func:`connect_tracked`; this
  44. is for callers managing their own connection objects, and for tests)."""
  45. with _live_lock:
  46. _track_key(_key(path))
  47. def _track_key(key: str, delta: int = 1) -> None:
  48. """Adjust the live count for an already-canonical key (caller holds ``_live_lock``)."""
  49. remaining = _live_connections.get(key, 0) + delta
  50. if remaining > 0:
  51. _live_connections[key] = remaining
  52. else:
  53. _live_connections.pop(key, None)
  54. def untrack_connection(path: Path | str) -> None:
  55. """Record that one connection to *path* has been closed."""
  56. with _live_lock:
  57. _track_key(_key(path), -1)
  58. def _live_main_key(key: str) -> Optional[str]:
  59. """The tracked main-database key that makes *key* live, or ``None`` (caller holds ``_live_lock``).
  60. SQLite locks the main file and its WAL sidecars; a raw ``close()`` of any of those
  61. inodes cancels this process's POSIX locks, but the registry is keyed by the main path."""
  62. if key in _live_connections:
  63. return key
  64. for suffix in ("-wal", "-shm"):
  65. if key.endswith(suffix) and key[:-len(suffix)] in _live_connections:
  66. return key[:-len(suffix)]
  67. return None
  68. def has_live_connection(path: Path | str) -> bool:
  69. """Whether this process holds a connection to *path* (or to the database it is a sidecar of).
  70. Point-in-time answer: a raw open/close right after it returns ``False`` can still race a
  71. new connection. Hold :func:`offline_file_access` across the I/O whenever possible."""
  72. with _live_lock:
  73. return _live_main_key(_key(path)) is not None
  74. class _TrackingMixin:
  75. """Untrack-on-close behaviour, mixable into any Connection subclass.
  76. The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent
  77. probe can never observe "no live connection" while this descriptor is still open. Unregister
  78. runs only after ``close()`` succeeds: untracking before a failing close (e.g. cross-thread
  79. ProgrammingError) would leave the FD open while the byte-probe guard thinks nothing is live.
  80. """
  81. _hermes_tracked_path: str | None = None
  82. def close(self) -> None: # type: ignore[misc]
  83. with _live_lock:
  84. path = getattr(self, "_hermes_tracked_path", None)
  85. # Close first; untrack only once the descriptor is actually gone. Untracking before a failing
  86. # close (e.g. cross-thread ProgrammingError) leaves the FD open while the byte-probe guard
  87. # thinks nothing is live — see #75629.
  88. super().close() # type: ignore[misc]
  89. if path is not None:
  90. self._hermes_tracked_path = None
  91. untrack_connection(path)
  92. class TrackedConnection(_TrackingMixin, sqlite3.Connection):
  93. """A ``sqlite3.Connection`` that untracks its path exactly once on close (callers close in many
  94. places, some via ``contextlib.closing``, so counting closes by hand is unreliable)."""
  95. _tracked_factory_cache: dict[type, type] = {}
  96. def _tracking_factory(factory: type) -> type:
  97. """Return *factory* augmented with untrack-on-close. Callers legitimately pass their own
  98. ``Connection`` subclasses (tests simulate FTS5-less or pragma-failing runtimes); leaving them
  99. untracked would quietly unguard the database, so the tracking ``close()`` is mixed in."""
  100. if factory is sqlite3.Connection:
  101. return TrackedConnection
  102. if issubclass(factory, _TrackingMixin):
  103. return factory
  104. cached = _tracked_factory_cache.get(factory)
  105. if cached is None:
  106. cached = type(f"Tracked{factory.__name__}", (_TrackingMixin, factory), {})
  107. _tracked_factory_cache[factory] = cached
  108. return cached
  109. def connect_tracked(
  110. path: Path | str, *, tracking_path: Path | str | None = None, connect_fn=None, **kwargs,
  111. ) -> sqlite3.Connection:
  112. """``sqlite3.connect`` that registers the connection for the lifetime of the fd (released on
  113. ``close()``). Use for any database that might be byte-probed (``state.db``, ``kanban.db``).
  114. Open and registration happen together under ``_live_lock`` so a concurrent
  115. :func:`read_header_bytes_preopen` cannot slip between them and cancel this connection's locks."""
  116. opener = connect_fn if connect_fn is not None else sqlite3.connect
  117. kwargs["factory"] = _tracking_factory(kwargs.get("factory", sqlite3.Connection))
  118. with _live_lock:
  119. conn = opener(str(path), **kwargs)
  120. try:
  121. resolved = _key(tracking_path) if tracking_path is not None else _canonical_db_path(conn)
  122. if resolved is None:
  123. return conn # in-memory / unnamed: nothing on disk to byte-probe
  124. if not isinstance(conn, _TrackingMixin):
  125. # The opener substituted its own factory and discarded ours (test doubles
  126. # simulating FTS5-less runtimes do this). Retag the instance's class so close()
  127. # still releases the registry entry rather than silently losing probe safety.
  128. conn = _retrofit_tracking(conn, resolved)
  129. conn._hermes_tracked_path = resolved
  130. _track_key(resolved)
  131. return conn
  132. except Exception:
  133. try:
  134. # Close via sqlite3 directly: the tracking entry was either never made or is
  135. # being unwound here.
  136. sqlite3.Connection.close(conn)
  137. except Exception:
  138. pass
  139. raise
  140. def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Connection:
  141. """Give an already-open connection untrack-on-close semantics by swapping ``__class__`` for
  142. one mixing in the tracking ``close()`` (used when an opener ignored the factory we asked for)."""
  143. cls = type(conn)
  144. try:
  145. conn.__class__ = _tracking_factory(cls) # type: ignore[assignment]
  146. return conn
  147. except TypeError as exc:
  148. raise UntrackableConnectionError(
  149. f"connection to {resolved} uses factory {cls.__name__}, which "
  150. "cannot release its tracking entry on close; byte-probe safety "
  151. "for this database would be silently lost"
  152. ) from exc
  153. def page_count_bytes(conn: sqlite3.Connection) -> Optional[int]:
  154. """Logical database size in bytes (``page_count * page_size``, the header field at offset 28)
  155. read via PRAGMA over *conn* so no new fd is opened. ``None`` when the pragmas cannot be read."""
  156. try:
  157. page_count = conn.execute("PRAGMA page_count").fetchone()[0]
  158. page_size = conn.execute("PRAGMA page_size").fetchone()[0]
  159. return int(page_count) * int(page_size)
  160. except (sqlite3.Error, TypeError, IndexError, ValueError) as exc:
  161. logger.debug("page_count/page_size unavailable: %s", exc)
  162. return None
  163. def file_length_matches_header(conn: sqlite3.Connection) -> Optional[bool]:
  164. """Whether the file on disk is at least as long as the header claims ("torn extend" check),
  165. without opening the file (PRAGMA over *conn* + ``stat()``). Advisory in WAL mode: a freshly
  166. committed page may still live in ``-wal`` so the main file legitimately lags."""
  167. path_str = _canonical_db_path(conn)
  168. if path_str is None:
  169. return None
  170. logical = page_count_bytes(conn)
  171. if not logical:
  172. return None
  173. try:
  174. return os.path.getsize(path_str) >= logical
  175. except OSError:
  176. return None
  177. def read_header_bytes_preopen(path: Path | str, *, length: int = 100, force: bool = False) -> Optional[bytes]:
  178. """Read the first *length* bytes of *path* -- only when no connection is live. The ONLY
  179. sanctioned byte-level read of a database file, for first-open validation (real SQLite? zeroed?
  180. overwritten?). Check and open/read/close run together under ``_live_lock`` so a connection
  181. cannot be opened between deciding "nothing is live" and closing this descriptor."""
  182. with _live_lock:
  183. if not force and _live_main_key(_key(path)) is not None:
  184. logger.debug(
  185. "refusing byte-level read of %s: a live connection exists in "
  186. "this process and close() would cancel its POSIX locks",
  187. path)
  188. return None
  189. try:
  190. with open(path, "rb") as handle:
  191. return handle.read(length)
  192. except OSError:
  193. return None
  194. @contextlib.contextmanager
  195. def offline_file_access(path: Path | str, *, what: str = "read"):
  196. """Hold the connection-lifecycle lock across a raw read of a database file: checking
  197. :func:`has_live_connection` and *then* doing raw I/O is a check/use race (a connection opened
  198. in between loses its POSIX locks to the raw ``close()``). Held only for the raw I/O."""
  199. with _live_lock:
  200. main = _live_main_key(_key(path))
  201. if main is not None:
  202. subject = "it" if main == _key(path) else f"its main database {main}"
  203. raise LiveConnectionError(
  204. f"Refusing to {what} {path}: a connection to {subject} is still open "
  205. "in this process, and raw file access would cancel that "
  206. "connection's POSIX advisory locks. Close all database "
  207. "handles (stop the gateway/dashboard) and retry.")
  208. yield