hermes_state_registry.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478
  1. """Process-wide shared SessionDB registry.
  2. Each bare ``SessionDB()`` mints its own writer connection, lock, close-time WAL checkpoint
  3. and token writer thread, and one connection's close-time checkpoint can race another's
  4. growth (lost/reordered-page-write corruption). This module owns that boundary: one shared
  5. ``SessionDB`` per resolved path per process, refcounted, with generation-aware retirement
  6. when the file is replaced (snapshot restore, recovery swap).
  7. Lifecycle rules:
  8. - ``acquire(path)`` returns the current generation for *path* and bumps its refcount.
  9. - ``close()`` on a shared instance RELEASES one refcount instead of tearing the
  10. connection down: the registry owns the physical lifecycle and only closes on the
  11. final release, so legacy call sites return their reference instead of leaking it.
  12. - ``release(db)`` decrements the generation *db was acquired from* (object-keyed, so an
  13. inode replacement cannot strand a still-owned generation); the final release of a
  14. retired generation tears it down.
  15. - On inode change the old generation is RETIRED (never lent again) but stays alive until
  16. its holders release. If the replacement open fails the registry keeps NO path entry.
  17. - All teardown happens OUTSIDE the registry lock: a final release's WAL checkpoint must
  18. never stall acquisition for every state.db.
  19. - A final close/checkpoint is serialized with the next open for the same path; no new
  20. generation is published while the previous generation is still tearing down.
  21. - A path can have SEVERAL closes admitted at once (the current generation's final release
  22. plus a retired generation's drain). The path barrier COUNTS them and is lifted only by
  23. the last one to settle, so neither ``acquire`` nor ``close_all`` / ``close_all_under``
  24. can escape while any handle for that path is still inside checkpoint/WAL-unlink.
  25. - Maintenance callers borrow handles with a temporary registry reference instead of
  26. iterating an unpinned snapshot.
  27. """
  28. from __future__ import annotations
  29. import contextlib
  30. import logging
  31. import os
  32. import threading
  33. from pathlib import Path
  34. from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Tuple
  35. from hermes_state_common import stat_db_file_identity as _stat_db_file_identity
  36. if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only
  37. from hermes_state import SessionDB
  38. logger = logging.getLogger(__name__)
  39. class _TeardownBarrier:
  40. """Accounting for every admitted-but-unfinished physical close of one path.
  41. A path can own more than one close at a time: the current generation's final
  42. release and a retired generation's drain are admitted independently under
  43. ``_lock`` and only meet at the lifecycle mutex. One event per path is honest
  44. only if the LAST admitted teardown settles it. Signalling on the first lets
  45. ``close_all()`` return and ``acquire()`` publish a replacement while an older
  46. handle is still inside ``PRAGMA wal_checkpoint``/sidecar unlink -- the exact
  47. overlap (#102827) this registry exists to forbid.
  48. """
  49. __slots__ = ("event", "pending")
  50. def __init__(self) -> None:
  51. self.event = threading.Event()
  52. self.pending = 0
  53. class _Generation:
  54. """One shared SessionDB generation: instance, refcount, file identity."""
  55. __slots__ = ("db", "identity", "path", "refcount", "retired")
  56. def __init__(self, path: Path, db: SessionDB, identity: Optional[tuple[int, int]]) -> None:
  57. self.path = path
  58. self.db = db
  59. self.refcount = 1
  60. self.identity = identity
  61. self.retired = False
  62. _lock = threading.Lock()
  63. # path → live generation; retired generations move to _retired (keyed by id(db)) until
  64. # their last holder releases.
  65. _generations: dict[Path, _Generation] = {}
  66. _retired: dict[int, _Generation] = {}
  67. # Paths whose next generation is being constructed. Construction runs outside _lock
  68. # (schema reconciliation can take seconds), but peers for the SAME file must wait or
  69. # every cold caller opens its own writer before a winner is chosen.
  70. _opening: dict[Path, threading.Event] = {}
  71. # A final close/checkpoint must finish before a replacement writer is opened
  72. # for the same path. The barrier is admitted while holding _lock and lifted
  73. # only after the LAST admitted physical teardown, so acquire cannot slip
  74. # through the generation-removal/open gap and close_all cannot report a
  75. # finished sweep over a close that is still running.
  76. _tearing_down: dict[Path, _TeardownBarrier] = {}
  77. # Open and close are both performed outside _lock. This per-path mutex closes
  78. # the race between checking _tearing_down and entering sqlite3.connect(),
  79. # including retired-generation drains after an inode replacement.
  80. _path_lifecycle_locks: dict[Path, threading.Lock] = {}
  81. def _open_session_db(path: Path) -> SessionDB:
  82. """Construct the SessionDB for *path* (call-time import avoids cycles; tests patch this)."""
  83. from hermes_state import SessionDB
  84. return SessionDB(db_path=path)
  85. def _teardown(db: SessionDB) -> None:
  86. """Close a shared instance, clearing its registry-owned flag first."""
  87. with contextlib.suppress(Exception):
  88. db._shared_registry_owned = False
  89. _close_quietly(db, "Error closing shared SessionDB")
  90. def _close_quietly(db: SessionDB, debug_message: str) -> None:
  91. """close() that never propagates. A lost WAL generation whose capture failed is data at risk,
  92. not teardown noise: the handle stays open and the operator has to act, so that one surfaces."""
  93. try:
  94. db.close()
  95. except Exception as exc:
  96. from hermes_state_dbfile import RetiredGenerationCaptureError
  97. if isinstance(exc, RetiredGenerationCaptureError):
  98. logger.error("SessionDB for %s did not settle at close: %s", _db_path_of(db), exc)
  99. else:
  100. logger.debug(debug_message, exc_info=True)
  101. def _path_lifecycle_lock_locked(path: Path) -> threading.Lock:
  102. """Return the lifecycle mutex for *path* (caller holds ``_lock``)."""
  103. lock = _path_lifecycle_locks.get(path)
  104. if lock is None:
  105. lock = threading.Lock()
  106. _path_lifecycle_locks[path] = lock
  107. return lock
  108. def _admit_teardown_locked(path: Path) -> _TeardownBarrier:
  109. """Register one pending physical close for *path* (caller holds ``_lock``).
  110. Admission shares the lock section that removes the generation, so a peer
  111. release, ``acquire`` or ``close_all`` taking the lock next always sees this
  112. teardown accounted for.
  113. """
  114. barrier = _tearing_down.get(path)
  115. if barrier is None:
  116. barrier = _tearing_down[path] = _TeardownBarrier()
  117. barrier.pending += 1
  118. return barrier
  119. def _finish_teardown(path: Path, barrier: _TeardownBarrier) -> None:
  120. """Settle one admitted teardown; only the last one lifts the path barrier."""
  121. with _lock:
  122. barrier.pending -= 1
  123. if barrier.pending > 0:
  124. return
  125. if _tearing_down.get(path) is barrier:
  126. _tearing_down.pop(path, None)
  127. barrier.event.set()
  128. def _teardown_generation(
  129. path: Path,
  130. db: SessionDB,
  131. *,
  132. barrier: Optional[_TeardownBarrier] = None,
  133. ) -> None:
  134. """Close *db* under its path lifecycle mutex, then settle its barrier slot."""
  135. with _lock:
  136. lifecycle_lock = _path_lifecycle_lock_locked(path)
  137. try:
  138. with lifecycle_lock:
  139. _teardown(db)
  140. finally:
  141. if barrier is not None:
  142. _finish_teardown(path, barrier)
  143. def _db_path_of(db: SessionDB) -> Optional[Path]:
  144. """``Path(db.db_path)`` or None when absent/unconvertible."""
  145. path = getattr(db, "db_path", None)
  146. try:
  147. return None if path is None else Path(path)
  148. except (TypeError, ValueError):
  149. return None
  150. def _finish_opening(path: Path, opening: threading.Event) -> None:
  151. """Drop the per-path construction marker and wake waiters (caller holds _lock)."""
  152. if _opening.get(path) is opening:
  153. _opening.pop(path, None)
  154. opening.set()
  155. def acquire(db_path: Optional[Path] = None) -> SessionDB:
  156. """Return the shared SessionDB for *db_path*, incrementing its refcount. If the file was
  157. replaced (different inode) since the generation opened, that generation is RETIRED
  158. but stays alive for its holders, and a fresh one is opened in its place. Raises
  159. whatever ``SessionDB.__init__`` raises; on a replacement-open failure the registry
  160. holds NO entry for the path."""
  161. from hermes_state import _default_db_path
  162. raw_path = Path(db_path) if db_path is not None else Path(_default_db_path())
  163. try:
  164. path = raw_path.resolve()
  165. except OSError:
  166. path = raw_path
  167. while True:
  168. wait_for: Optional[threading.Event] = None
  169. with _lock:
  170. generation = _generations.get(path)
  171. if generation is not None:
  172. current = _stat_db_file_identity(path)
  173. if current is not None and generation.identity is not None and current != generation.identity:
  174. # File replaced: retire this generation so it is never lent again, then elect one
  175. # caller to open the replacement. It stays alive for its holders, tracked in
  176. # ``_retired`` by ``id(db)`` so their releases find it after the path remaps.
  177. generation.retired = True
  178. del _generations[path]
  179. _retired[id(generation.db)] = generation
  180. else:
  181. generation.refcount += 1
  182. return generation.db
  183. teardown = _tearing_down.get(path)
  184. if teardown is not None:
  185. wait_for = teardown.event
  186. else:
  187. opening = _opening.get(path)
  188. if opening is None:
  189. opening = _opening[path] = threading.Event()
  190. lifecycle_lock = _path_lifecycle_lock_locked(path)
  191. else:
  192. wait_for = opening
  193. if wait_for is not None:
  194. # Another caller is constructing or closing this path; wait without holding the
  195. # global lock. A failed opener signals too, so a waiter can retry.
  196. wait_for.wait()
  197. continue
  198. # Open OUTSIDE the registry lock; the per-path marker prevents redundant writers without
  199. # serialising other files, the lifecycle mutex keeps the open off a same-path close.
  200. try:
  201. with lifecycle_lock:
  202. db = _open_session_db(path)
  203. db._shared_registry_owned = True
  204. identity = _stat_db_file_identity(path)
  205. except BaseException:
  206. with _lock:
  207. _finish_opening(path, opening)
  208. raise
  209. with _lock:
  210. teardown = _tearing_down.get(path)
  211. if teardown is None:
  212. existing = _generations.get(path)
  213. if existing is not None: # Defensive: installed by explicit registry manipulation mid-open.
  214. existing.refcount += 1
  215. winner = existing.db
  216. else:
  217. _generations[path] = _Generation(path, db, identity)
  218. winner = db
  219. _finish_opening(path, opening)
  220. if teardown is not None:
  221. # A shutdown or retired-generation final release was admitted while this opener was
  222. # constructing: never publish into that window — discard the speculative handle and
  223. # go round again once the barrier lifts.
  224. _teardown_generation(path, db)
  225. teardown.event.wait()
  226. continue
  227. if winner is not db:
  228. _teardown_generation(path, db)
  229. return winner
  230. def release(db: SessionDB) -> bool:
  231. """Decrement the refcount of a shared SessionDB. ``True`` if *db* was shared; ``False``
  232. if it is not registry-managed (caller owns close()). The final release tears the
  233. generation down OUTSIDE the registry lock. Lookup is object-keyed, so holders of an
  234. old generation release into its retired record, not into whatever the path names."""
  235. if db is None:
  236. return False
  237. key = id(db)
  238. teardown_barrier: Optional[_TeardownBarrier] = None
  239. with _lock:
  240. generation = _retired.get(key)
  241. if generation is None:
  242. path = _db_path_of(db)
  243. if path is None:
  244. return False
  245. generation = _generations.get(path)
  246. if generation is None or generation.db is not db:
  247. # Not shared (bare SessionDB()); the caller owns close().
  248. return False
  249. generation.refcount -= 1
  250. needs_teardown = generation.refcount <= 0
  251. if needs_teardown:
  252. if generation.retired:
  253. _retired.pop(key, None)
  254. elif _generations.get(generation.path) is generation:
  255. _generations.pop(generation.path, None)
  256. # A retired generation's drain is admitted too: it checkpoints and unlinks the same
  257. # sidecars as the current one, so a replacement writer must not open on top of it.
  258. teardown_barrier = _admit_teardown_locked(generation.path)
  259. # Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and read-pool
  260. # drain must not block acquisition for every other state.db.
  261. if needs_teardown:
  262. _teardown_generation(generation.path, db, barrier=teardown_barrier)
  263. return True
  264. def _path_is_under(path: Path, root: Path) -> bool:
  265. """True when *path* is *root* or a file inside it (normcase, resolved)."""
  266. try:
  267. path_key = os.path.normcase(str(path.resolve()))
  268. root_key = os.path.normcase(str(root.resolve()))
  269. except OSError:
  270. path_key = os.path.normcase(str(path))
  271. root_key = os.path.normcase(str(root))
  272. return path_key == root_key or path_key.startswith(root_key + os.sep)
  273. def _teardown_swept_generations(
  274. generations: list[_Generation],
  275. teardown_barriers: dict[Path, _TeardownBarrier],
  276. active_teardowns: list[_TeardownBarrier],
  277. ) -> int:
  278. """Close *generations* outside the registry lock; wait for already-admitted teardowns."""
  279. by_path: dict[Path, list[_Generation]] = {}
  280. for generation in generations:
  281. by_path.setdefault(generation.path, []).append(generation)
  282. for path, path_generations in by_path.items():
  283. with _lock:
  284. lifecycle_lock = _path_lifecycle_lock_locked(path)
  285. try:
  286. with lifecycle_lock:
  287. for generation in path_generations:
  288. _teardown(generation.db)
  289. finally:
  290. _finish_teardown(path, teardown_barriers[path])
  291. for barrier in active_teardowns:
  292. barrier.event.wait()
  293. return len(generations)
  294. def close_all() -> int:
  295. """Close every shared SessionDB regardless of refcount; returns the count. For gateway
  296. shutdown, after all agents and cron jobs finished. Idempotent."""
  297. teardown_barriers: dict[Path, _TeardownBarrier] = {}
  298. with _lock:
  299. active_teardowns = list(_tearing_down.values())
  300. generations = list(_generations.values()) + list(_retired.values())
  301. for path in {generation.path for generation in generations}:
  302. teardown_barriers[path] = _admit_teardown_locked(path)
  303. _generations.clear()
  304. _retired.clear()
  305. for generation in generations:
  306. generation.retired = True
  307. return _teardown_swept_generations(generations, teardown_barriers, active_teardowns)
  308. def close_all_under(directory: str | Path) -> int:
  309. """Force-close every shared SessionDB whose file lives under *directory*; returns the count.
  310. Profile delete rmtree (and a same-name recreate) fails while this process still holds
  311. ``state.db``. Same contract as ``MemoryStore.release_all_under``: a live holder is
  312. expected to fail afterward; a process that holds none is a no-op returning 0.
  313. A final ``release()`` can drop the generation and admit teardown before the physical
  314. close finishes. Wait for those directory-matching barriers even when no generation
  315. remains, otherwise rmtree still sees the open handle.
  316. """
  317. try:
  318. root = Path(directory).expanduser().resolve()
  319. except OSError:
  320. root = Path(directory).expanduser()
  321. teardown_barriers: dict[Path, _TeardownBarrier] = {}
  322. with _lock:
  323. generations = [
  324. generation
  325. for generation in list(_generations.values()) + list(_retired.values())
  326. if _path_is_under(generation.path, root)
  327. ]
  328. # Collect by directory, not by remaining generations: a last release already
  329. # popped the generation and left only ``_tearing_down``.
  330. active_teardowns = [
  331. barrier for path, barrier in _tearing_down.items()
  332. if _path_is_under(path, root)
  333. ]
  334. selected_paths = {generation.path for generation in generations}
  335. for path in selected_paths:
  336. teardown_barriers[path] = _admit_teardown_locked(path)
  337. for generation in generations:
  338. generation.retired = True
  339. if _generations.get(generation.path) is generation:
  340. _generations.pop(generation.path, None)
  341. _retired.pop(id(generation.db), None)
  342. return _teardown_swept_generations(generations, teardown_barriers, active_teardowns)
  343. def other_generations_for_path(
  344. db_path: Path, *, exclude: Optional[SessionDB] = None
  345. ) -> list[str]:
  346. """Describe every other LIVE SessionDB generation THIS process holds for *db_path*.
  347. The registry is path-keyed, so it can answer the in-process half of "is this store quiet?"
  348. that a ``/proc`` descriptor scan structurally cannot: that scan skips our own pid, so it only
  349. ever proves other PROCESSES are away.
  350. RETIRED generations are deliberately not holders here. A generation is retired only after its
  351. file was replaced, which is exactly when ``SessionDB`` fences it: every write raises
  352. ``StateDbReplacedError`` and the close-time checkpoint is disabled, so it is not the live writer
  353. this gate protects. It also leaves ``_retired`` only when its last holder releases, and a
  354. gateway handle does not release before shutdown — counting it made ONE inode replacement
  355. (recovery swap, backup restore, snapshot) skip auto-VACUUM for that path for the rest of the
  356. process lifetime, turning the unbounded growth this maintenance exists to bound into a
  357. permanent condition.
  358. """
  359. try:
  360. path = Path(db_path).resolve()
  361. except OSError:
  362. path = Path(db_path)
  363. with _lock:
  364. return [
  365. f"in-process live SessionDB generation (refcount {generation.refcount})"
  366. for generation in _generations.values()
  367. if generation.db is not exclude
  368. and generation.path == path
  369. and not generation.retired
  370. ]
  371. def live_shared_session_dbs() -> list[SessionDB]:
  372. """Snapshot of every live (non-retired) shared SessionDB (refcounts untouched), for
  373. in-process maintenance. A concurrent final release may close an instance, in which
  374. case the callee sees ``_conn is None``."""
  375. with _lock:
  376. return [g.db for g in _generations.values() if not g.retired]
  377. @contextlib.contextmanager
  378. def borrow_live_shared_session_dbs() -> Iterator[list[SessionDB]]:
  379. """Borrow live handles with registry references pinned for the whole block.
  380. Maintenance must not operate on the unowned snapshot returned by
  381. :func:`live_shared_session_dbs`: the last real owner could otherwise
  382. release and physically close the connection between the snapshot and the
  383. maintenance call. Each borrowed generation gets one temporary reference;
  384. the ``finally`` block releases it even when maintenance raises.
  385. """
  386. with _lock:
  387. borrowed_generations = [
  388. generation for generation in _generations.values() if not generation.retired
  389. ]
  390. borrowed = [generation.db for generation in borrowed_generations]
  391. for generation in borrowed_generations:
  392. generation.refcount += 1
  393. try:
  394. yield borrowed
  395. finally:
  396. for db in reversed(borrowed):
  397. release(db)
  398. def stats() -> dict[str, int]:
  399. """Registry census for tests and diagnostics (no locks held long)."""
  400. with _lock:
  401. return {
  402. "live_generations": len(_generations), "retired_generations": len(_retired),
  403. "total_refcounts": sum(g.refcount for g in _generations.values()),
  404. }
  405. def release_or_close(db: SessionDB) -> None:
  406. """Release a shared instance, or close it when it is not registry-managed. Drop-in for a
  407. plain ``db.close()``: read-only opens, CLI one-shots and test fakes fall back."""
  408. if not release(db):
  409. _close_quietly(db, "release_or_close fallback close failed")