hermes_state_reconcile.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514
  1. """Additive state.db schema drift: detection, the one-shot browse reconcile, and explicit repair.
  2. Drift (a declared table/column missing) is not corruption. Browsing catches an unowned store up
  3. once, additive DDL only (:func:`reconcile_drift_for_browse`); ``hermes sessions repair`` reconciles
  4. a protected snapshot so failed DDL never leaves the store partly upgraded or escalates into FTS
  5. surgery (:func:`repair_declarative_schema`). Neither runs while a gateway owns the store.
  6. """
  7. import contextlib
  8. import logging
  9. import os
  10. import sqlite3
  11. import threading
  12. from contextlib import closing
  13. from pathlib import Path
  14. def schema_drift_reason(path: Path):
  15. """Return a missing declared table/column, not a corruption diagnosis."""
  16. from hermes_state_schema import schema_read_probe_statements
  17. from hermes_cli.sqlite_safe_read import connect_tracked
  18. with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:
  19. try:
  20. for statement in schema_read_probe_statements():
  21. conn.execute(statement).fetchone()
  22. except sqlite3.OperationalError as exc:
  23. from hermes_state_repair import _schema_not_built
  24. if _schema_not_built(exc):
  25. return str(exc)
  26. raise
  27. return None
  28. def _reconcile_snapshot(path, guard):
  29. from hermes_state_repair import (
  30. _copy_database_snapshot, _db_opens_cleanly, _repair_conn,
  31. _repair_scratch_space_error, _unlink_db_triple,
  32. )
  33. from hermes_state_schema import apply_declared_schema
  34. scratch = path.with_name(path.name + '.reconcile-scratch')
  35. if error := _repair_scratch_space_error(path):
  36. raise RuntimeError(error)
  37. if error := _unlink_db_triple(scratch):
  38. raise RuntimeError(error)
  39. try:
  40. _copy_database_snapshot(path, scratch, source_connection=guard)
  41. with _repair_conn(scratch) as conn:
  42. _require_history_tables(conn)
  43. # The final probe fails closed if an additive change is not possible.
  44. apply_declared_schema(conn.cursor())
  45. conn.commit()
  46. if reason := schema_drift_reason(scratch):
  47. raise RuntimeError(f'schema reconciliation incomplete: {reason}')
  48. if reason := _db_opens_cleanly(scratch):
  49. raise RuntimeError(f'reconciled snapshot is unhealthy: {reason}')
  50. _copy_database_snapshot(scratch, path, destination_connection=guard)
  51. finally:
  52. if error := _unlink_db_triple(scratch):
  53. logging.getLogger(__name__).warning('Could not remove reconciliation snapshot: %s', error)
  54. def _require_history_tables(conn):
  55. tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
  56. if {'sessions', 'messages'} - tables:
  57. raise RuntimeError('canonical history tables are missing; use sessions recover, not additive reconciliation')
  58. def _reconcile_locked(path, backup, report):
  59. from hermes_state_repair import (
  60. _backup_db_file, _db_opens_cleanly, _exclusive_repair_db_guard,
  61. _live_writer_holds_db,
  62. )
  63. if _live_writer_holds_db(path):
  64. raise RuntimeError('a live writer still holds state.db; stop it and retry')
  65. from hermes_cli.sqlite_safe_read import connect_tracked
  66. with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:
  67. _require_history_tables(conn)
  68. if schema_drift_reason(path) is None:
  69. report.update(repaired=True, strategy='already_healthy')
  70. return
  71. # Additive repair is not a substitute for corruption recovery.
  72. if reason := _db_opens_cleanly(path):
  73. raise RuntimeError(reason)
  74. if backup:
  75. bpath, error = _backup_db_file(path)
  76. if bpath is None:
  77. raise RuntimeError(f'pre-repair backup refused: {error}')
  78. report['backup_path'] = str(bpath)
  79. with _exclusive_repair_db_guard(path) as (guard, error):
  80. if guard is None:
  81. raise RuntimeError(f'could not acquire exclusive state.db repair ownership: {error}')
  82. _reconcile_snapshot(path, guard)
  83. report.update(repaired=True, strategy='reconcile_declarative_schema')
  84. # Browse-time reconcile (#140315). A store whose tables/columns lag SCHEMA_SQL is not corrupt:
  85. # a writable open would catch it up, but an idle profile with no gateway never gets one, so every
  86. # read-only surface (sidebar, /api/sessions/<id>/messages?profile=) lost its whole history.
  87. # Catch it up ONCE, additive DDL only, in one short transaction, and only while holding that
  88. # profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse
  89. # by name with the remedy instead of becoming a second one.
  90. #
  91. # The full integrity scan is read-only and scales with file size, so it runs BEFORE the lock, on the
  92. # same 3 s budget (BROWSE_INTEGRITY_BUDGET_S); a gateway booting meanwhile simply takes the lock. A
  93. # store too large to verify and catch up in that budget is refused with a retryable 503 that names
  94. # `hermes sessions repair` (repair_declarative_schema: same additive catch-up, no time limit). The
  95. # scan is tied to the file's (size, mtime) signature: if the store changed by the time the lock is
  96. # held, the scan no longer vouches for it and nothing is written. Under the lock only the cheap drift
  97. # re-check and the DDL run, cut off by BROWSE_RECONCILE_BUDGET_S (a sqlite progress handler; the DDL
  98. # rolls back). Busy waiting does not run the progress handler, so each connection's busy timeout is
  99. # capped at the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).
  100. BROWSE_RECONCILE_BUDGET_S = 3.0
  101. BROWSE_INTEGRITY_BUDGET_S = BROWSE_RECONCILE_BUDGET_S
  102. _BUDGET_PROGRESS_OPS = 1000
  103. class DeclarativeDriftError(RuntimeError):
  104. """The store lags SCHEMA_SQL (missing table/column) and browsing could not catch it up."""
  105. def __init__(self, detail: str, *, owned: bool = False):
  106. super().__init__(detail)
  107. self.owned = owned
  108. def is_declarative_drift_error(exc: BaseException) -> bool:
  109. """``no such table/column`` from a schema read probe: drift, never corruption."""
  110. from hermes_state_repair import _schema_not_built
  111. return isinstance(exc, sqlite3.OperationalError) and _schema_not_built(exc)
  112. def _repair_command(home: Path) -> str:
  113. """The `hermes sessions repair` invocation that targets *home* (no time limit, additive only)."""
  114. from gateway.status import _profile_label_for_home # never raises: None when unknown
  115. label = _profile_label_for_home(home)
  116. if label == 'default':
  117. return 'hermes sessions repair'
  118. if label:
  119. return f'hermes -p {label} sessions repair'
  120. return f'HERMES_HOME={home} hermes sessions repair'
  121. def _owner_remedy(home: Path) -> tuple[str, str]:
  122. from gateway.status import _profile_label_for_home # never raises: None when unknown
  123. label = _profile_label_for_home(home)
  124. if label and label != 'default':
  125. return label, f'hermes -p {label} gateway restart'
  126. return label or str(home), 'hermes gateway restart'
  127. def _owning_gateway_restart(home: Path) -> str:
  128. """Restart command for the gateway that holds *home*'s lock. A multiplexer launched from another
  129. home (usually the default) records that home in ``hermes_home``; `hermes -p <named> gateway
  130. restart` is refused there ("Manage the host gateway instead"), so name the owner's own command."""
  131. from gateway.runtime_ownership import canonical_home
  132. from gateway.status import _read_gateway_lock_record
  133. record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock') or {}
  134. owner_home = record.get('hermes_home')
  135. if isinstance(owner_home, str) and owner_home and canonical_home(Path(owner_home)) != canonical_home(home):
  136. return _owner_remedy(Path(owner_home))[1]
  137. return _owner_remedy(home)[1]
  138. def _lockless_gateway_holder(path: Path) -> str | None:
  139. """A Hermes gateway process holding *path* open without this profile's gateway.lock: a pre-v0.22.1
  140. multiplexer serves named profiles without reserving their per-profile lock, so the lock alone
  141. misses it. Only holders whose command line is a Hermes ``gateway`` run count; plain readers
  142. (Desktop, sibling browses, CLI reads) are not owners. Returns a description, else None."""
  143. from hermes_state_holders import _looks_like_hermes, _read_proc_argv, describe_holder_pid, foreign_state_db_holders
  144. for pid, _target in foreign_state_db_holders(path):
  145. if pid <= 0:
  146. continue
  147. argv = _read_proc_argv(pid)
  148. if argv is None:
  149. try:
  150. import psutil
  151. argv = psutil.Process(pid).cmdline() or None
  152. except (ImportError, OSError, psutil.Error): # gone, denied, or no psutil: not provably ours
  153. argv = None
  154. if argv and _looks_like_hermes(argv) and 'gateway' in argv[1:]:
  155. return describe_holder_pid(pid)
  156. return None
  157. class BrowseBudgetExceeded(RuntimeError):
  158. """The browse reconcile could not finish inside BROWSE_RECONCILE_BUDGET_S; nothing was written."""
  159. class _Budget:
  160. """One monotonic deadline for everything a browse does while holding ``gateway.lock``."""
  161. def __init__(self, seconds: float):
  162. import time
  163. self._clock = time.monotonic
  164. self.deadline = self._clock() + seconds
  165. self.expired = False
  166. def remaining(self) -> float:
  167. return max(self.deadline - self._clock(), 0.0)
  168. def busy_timeout(self) -> float:
  169. return max(self.remaining(), 0.05)
  170. def arm(self, conn) -> None:
  171. def progress():
  172. if self._clock() >= self.deadline:
  173. self.expired = True
  174. return 1 # sqlite raises OperationalError('interrupted'); the caller rolls back
  175. return 0
  176. conn.set_progress_handler(progress, _BUDGET_PROGRESS_OPS)
  177. def check(self) -> None:
  178. if self.expired or self.remaining() <= 0:
  179. self.expired = True
  180. raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget')
  181. def _apply_additive_schema(path: Path, budget: _Budget | None = None) -> None:
  182. """Missing SCHEMA_SQL tables and columns, in ONE ``BEGIN IMMEDIATE`` transaction.
  183. No data migration, no FTS work, no schema_version stamp (that stays the owner's: the
  184. version gates data migrations this path does not run). Every statement is additive and
  185. idempotent. The declared shape is verified inside the transaction before COMMIT: a column
  186. the shared reconciler could not add (it logs and continues) or an expired budget rolls the
  187. store back to exactly what it was, never a partial upgrade."""
  188. from hermes_cli.sqlite_safe_read import connect_tracked
  189. from hermes_state_repair import _reapply_durability_barriers, _schema_not_built
  190. from hermes_state_schema import apply_declared_schema, schema_read_probe_statements
  191. budget = budget or _Budget(BROWSE_RECONCILE_BUDGET_S)
  192. budget.check()
  193. with closing(connect_tracked(path, tracking_path=path, timeout=budget.busy_timeout(),
  194. isolation_level=None)) as conn:
  195. _reapply_durability_barriers(conn)
  196. budget.arm(conn)
  197. conn.execute('BEGIN IMMEDIATE')
  198. try:
  199. cursor = conn.cursor()
  200. apply_declared_schema(cursor)
  201. # Postcondition, still inside the transaction: every declared table/column resolves.
  202. for statement in schema_read_probe_statements():
  203. try:
  204. cursor.execute(statement).fetchone()
  205. except sqlite3.OperationalError as exc:
  206. if _schema_not_built(exc):
  207. raise DeclarativeDriftError(str(exc)) from exc
  208. raise
  209. budget.check()
  210. conn.execute('COMMIT')
  211. except BaseException as exc:
  212. with contextlib.suppress(sqlite3.Error):
  213. conn.execute('ROLLBACK')
  214. if budget.expired and isinstance(exc, sqlite3.OperationalError):
  215. raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget') from exc
  216. raise
  217. BROWSE_RECONCILE_PURPOSE = 'browse_schema_reconcile'
  218. _BROWSE_SIBLING_POLL_S = 0.1
  219. class CorruptStoreError(sqlite3.DatabaseError):
  220. """The lagging store is also damaged: browsing must not write DDL into it. Worded like sqlite's
  221. own malformed-image error so every caller maps it to the corrupt-store status (``hermes doctor``)."""
  222. def __init__(self, detail: str):
  223. super().__init__(f'database disk image is malformed (integrity_check: {detail})')
  224. def _integrity_problem(path: Path, budget: _Budget):
  225. """``PRAGMA integrity_check`` on a read-only connection under *budget*: None when ``ok``, else the
  226. first problems. Full integrity_check, not quick_check: quick_check skips index-vs-table content,
  227. so an index with the right entry count but the wrong keys passes it. Raises
  228. :class:`BrowseBudgetExceeded` when the scan cannot finish in time (fail closed: no write)."""
  229. from hermes_cli.sqlite_safe_read import connect_tracked
  230. budget.check()
  231. try:
  232. with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path,
  233. timeout=budget.busy_timeout())) as conn:
  234. budget.arm(conn)
  235. rows = conn.execute('PRAGMA integrity_check(3)').fetchall()
  236. except sqlite3.DatabaseError as exc:
  237. if budget.expired:
  238. raise BrowseBudgetExceeded('integrity check exceeded the browse time budget') from exc
  239. from hermes_state_errors import is_malformed_db_error
  240. if is_malformed_db_error(exc):
  241. return str(exc)
  242. raise
  243. budget.check()
  244. problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != 'ok']
  245. return '; '.join(problems[:3]) or None
  246. def _store_signature(path: Path):
  247. """(size, mtime_ns) of the main file and its WAL: any committed write changes one of them."""
  248. out = []
  249. for p in (path, path.with_name(path.name + '-wal')):
  250. try:
  251. st = p.stat()
  252. out.append((st.st_size, st.st_mtime_ns))
  253. except FileNotFoundError:
  254. out.append(None)
  255. return tuple(out)
  256. _scan_guard = threading.Lock()
  257. _scan_locks: dict[str, threading.Lock] = {}
  258. # Signatures whose integrity scan could not finish inside BROWSE_INTEGRITY_BUDGET_S. A sidebar polls
  259. # every few seconds; rescanning a multi-GB file on each poll only burns I/O for the same answer.
  260. _scan_too_slow: dict[str, tuple] = {}
  261. def _path_scan_lock(path: Path) -> threading.Lock:
  262. key = str(path.resolve())
  263. with _scan_guard:
  264. return _scan_locks.setdefault(key, threading.Lock())
  265. class _SiblingStillBusy(RuntimeError):
  266. """Another browse/maintenance holder kept gateway.lock past its own budget; carries the drift."""
  267. @contextlib.contextmanager
  268. def _reserving(home: Path):
  269. """``exclusive_maintenance`` for a browse. A lock this user cannot open at all (a root-run
  270. gateway's, #42685) is not drift and not a crash: refuse by name with its recovery text."""
  271. from gateway.runtime_ownership import exclusive_maintenance
  272. with contextlib.ExitStack() as stack:
  273. try:
  274. stack.enter_context(exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE))
  275. except OSError as exc:
  276. raise DeclarativeDriftError(
  277. f'Session store is behind this Hermes version, but its gateway.lock cannot be used '
  278. f'({exc}); nothing was changed.') from exc
  279. yield
  280. @contextlib.contextmanager
  281. def _reserve_for_ddl(home: Path, path: Path, owned):
  282. """Hold gateway.lock for the DDL. A sibling holder (another browse's flock probe or DDL, never a
  283. gateway) is retried, bounded by its own budget, rather than waited on passively: a probe holds
  284. the lock for no work and a sibling's DDL finishes inside BROWSE_RECONCILE_BUDGET_S. Yields True
  285. when the store is already current (nothing to do). Raises ``owned(drift)`` when a gateway turns
  286. out to own it, and :class:`_SiblingStillBusy` past the bound."""
  287. import time
  288. from gateway.runtime_ownership import OwnershipConflict, live_runtime_owner
  289. deadline = time.monotonic() + BROWSE_RECONCILE_BUDGET_S + 0.5
  290. with contextlib.ExitStack() as stack:
  291. while True:
  292. try:
  293. stack.enter_context(_reserving(home))
  294. break
  295. except OwnershipConflict as exc:
  296. pending = schema_drift_reason(path)
  297. if pending is None:
  298. break # a sibling caught it up; the re-check below yields True without the lock
  299. if live_runtime_owner(home) is not None:
  300. raise owned(pending) from exc
  301. if time.monotonic() >= deadline:
  302. raise _SiblingStillBusy(pending) from exc
  303. time.sleep(_BROWSE_SIBLING_POLL_S)
  304. # Re-check (under the lock when held): a gateway that started and released between probes,
  305. # or a sibling browse, may already have caught the store up.
  306. yield schema_drift_reason(path) is None
  307. def _verify_clean_before_lock(path: Path, too_slow):
  308. """The pre-lock phase of the browse catch-up: a full read-only integrity scan OUTSIDE gateway.lock
  309. (it may be slow on a large store and must never block a gateway boot). Returns the file signature
  310. the scan vouches for. Raises :class:`CorruptStoreError` for a damaged store (latched, so polls do
  311. not rescan it) and ``too_slow()`` when the scan cannot finish in its budget (remembered for the
  312. same signature, so a polling sidebar does not rescan a multi-GB file every few seconds)."""
  313. from hermes_state_health import STORAGE_CORRUPT, mark_storage_corrupt, storage_corrupt_reason, storage_state
  314. if storage_state(path) == STORAGE_CORRUPT:
  315. raise CorruptStoreError(storage_corrupt_reason(path) or 'latched corrupt')
  316. key = str(path.resolve())
  317. signature = _store_signature(path)
  318. if _scan_too_slow.get(key) == signature:
  319. raise too_slow()
  320. try:
  321. problem = _integrity_problem(path, _Budget(BROWSE_INTEGRITY_BUDGET_S))
  322. except BrowseBudgetExceeded as exc:
  323. _scan_too_slow[key] = signature
  324. raise too_slow() from exc
  325. if problem:
  326. # A failing column probe hides damage elsewhere in the file: additive DDL is not
  327. # corruption recovery, so never write into a store that is not verified clean.
  328. mark_storage_corrupt(path, problem)
  329. raise CorruptStoreError(problem)
  330. return signature
  331. def reconcile_drift_for_browse(path: Path) -> bool:
  332. """Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.
  333. Raises :class:`DeclarativeDriftError` (``owned=True`` when a running gateway owns the store)
  334. with a user-facing detail naming the drift and the remedy, and :class:`CorruptStoreError`
  335. when the lagging store is also damaged. Never touches a store missing its history tables:
  336. recreating them empty would hide lost sessions."""
  337. from gateway.runtime_ownership import OwnershipConflict, live_runtime_owner
  338. reason = schema_drift_reason(path)
  339. if reason is None:
  340. return False
  341. from hermes_cli.sqlite_safe_read import connect_tracked
  342. with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:
  343. try:
  344. _require_history_tables(conn)
  345. except RuntimeError as exc:
  346. raise DeclarativeDriftError(
  347. f'Session store is missing its history tables ({reason}); this is not upgrade drift. '
  348. 'Run `hermes sessions recover`.') from exc
  349. home = path.parent
  350. label, _ = _owner_remedy(home)
  351. def owned(drift, holder=None):
  352. restart = 'hermes gateway restart' if holder else _owning_gateway_restart(home)
  353. who = f' ({holder})' if holder else ''
  354. return DeclarativeDriftError(
  355. f"Session store for profile '{label}' is behind this Hermes version ({drift}), and a "
  356. f'running gateway owns it{who}, so browsing will not write to it. Restart that gateway '
  357. f'(`{restart}`) to bring the store up to date.', owned=True)
  358. def too_slow():
  359. # A gateway that booted while browse was verifying the store is the real answer: name it.
  360. if live_runtime_owner(home) is not None:
  361. return owned(reason)
  362. # Otherwise no gateway owns the store: never tell the user to restart one that is not
  363. # running. The repair path has no time limit.
  364. return DeclarativeDriftError(
  365. f"Session store for profile '{label}' is behind this Hermes version ({reason}) and is too "
  366. f'large to verify and update within {BROWSE_RECONCILE_BUDGET_S:.0f}s while browsing. Your '
  367. f'history is untouched: nothing was changed and the list was not cleared. Run '
  368. f'`{_repair_command(home)}` to bring it up to date (it has no time limit), then reload.')
  369. # A gateway already owns it: never scan or write, refuse by name at once. Probe the flock itself
  370. # (a record left by a released holder still names a live PID), holding it for no work at all.
  371. try:
  372. with _reserving(home):
  373. pass
  374. except OwnershipConflict as exc:
  375. if live_runtime_owner(home) is not None:
  376. raise owned(reason) from exc
  377. if holder := _lockless_gateway_holder(path):
  378. raise owned(reason, holder)
  379. with _path_scan_lock(path): # one scan per store per process; concurrent polls wait for it
  380. if schema_drift_reason(path) is None:
  381. return False
  382. signature = _verify_clean_before_lock(path, too_slow)
  383. try:
  384. with _reserve_for_ddl(home, path, owned) as current:
  385. if current:
  386. return False
  387. budget = _Budget(BROWSE_RECONCILE_BUDGET_S)
  388. if holder := _lockless_gateway_holder(path): # a legacy gateway started during the scan
  389. raise owned(reason, holder)
  390. if _store_signature(path) != signature:
  391. # Written since the scan (a gateway that came and went): the scan no longer
  392. # vouches for these bytes. Refuse; the next browse scans the new state.
  393. raise DeclarativeDriftError(
  394. f"Session store for profile '{label}' changed while browsing verified it "
  395. f'({reason}); nothing was changed. Retry in a moment; the list was not cleared.')
  396. def not_writable(why):
  397. return DeclarativeDriftError(
  398. f"Session store for profile '{label}' is behind this Hermes version ({reason}), "
  399. f'but {why} is not writable by this user, so browsing cannot add the missing '
  400. f'tables/columns; nothing was changed. Fix the permissions, then reload.')
  401. # The shared column reconciler logs a readonly ALTER failure and continues, so check
  402. # first: the file plus its directory (SQLite creates its journal/WAL beside it).
  403. for target in (path, path.parent):
  404. if not os.access(target, os.W_OK):
  405. raise not_writable(str(target))
  406. try:
  407. _apply_additive_schema(path, budget)
  408. except sqlite3.OperationalError as exc:
  409. if 'readonly' in str(exc).lower():
  410. raise not_writable(str(path)) from exc
  411. raise
  412. except DeclarativeDriftError as exc:
  413. raise DeclarativeDriftError(
  414. f"Session store for profile '{label}' is behind this Hermes version and could not "
  415. f'be caught up ({exc}); nothing was changed. Run `hermes sessions repair` for that '
  416. 'profile.') from exc
  417. except BrowseBudgetExceeded as exc:
  418. raise too_slow() from exc
  419. except _SiblingStillBusy as exc:
  420. raise DeclarativeDriftError(
  421. f"Session store for profile '{label}' is being brought up to date by another Hermes "
  422. f'process ({exc}). Retry in a moment; the list was not cleared.') from exc
  423. if after := schema_drift_reason(path):
  424. raise DeclarativeDriftError(
  425. f"Session store for profile '{label}' is behind this Hermes version and could not be "
  426. f'caught up ({after}). Run `hermes sessions repair` for that profile.')
  427. logging.getLogger(__name__).warning(
  428. 'state.db at %s lagged SCHEMA_SQL (%s); added the missing tables/columns for browsing', path, reason)
  429. return True
  430. def repair_declarative_schema(path: Path, *, backup: bool = True):
  431. """Reuse repair's snapshot/SQLite exclusion beneath the runtime owner lock."""
  432. from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
  433. from hermes_state_repair import _cross_process_repair_lock
  434. report = {'repaired': False, 'strategy': None, 'backup_path': None, 'error': None}
  435. try:
  436. with exclusive_maintenance([path.parent]):
  437. with _cross_process_repair_lock(path) as holding:
  438. if not holding:
  439. raise RuntimeError('could not obtain the state.db repair lock')
  440. _reconcile_locked(path, backup, report)
  441. except (OwnershipConflict, OSError, sqlite3.Error, RuntimeError, TimeoutError) as exc:
  442. report['error'] = str(exc)
  443. return report