hermes_state_health.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114
  1. """Profile-level session-storage health: one process-wide latch per state.db path.
  2. A structurally corrupt state.db used to show up as an empty or partial Desktop sidebar,
  3. green readiness and a 500 from ``/api/sessions``: every surface guessed on its own and
  4. none said "the store is damaged" (#72046). This module is the single place that fact is
  5. recorded. Writers (``SessionDB._halt_db_corrupt``), readers (``SessionDB`` read helpers)
  6. and the readiness probe publish into it; ``gateway.readiness`` (``state_db`` /
  7. ``session_store`` checks, hence ``/api/status`` ``components.storage``) and the session
  8. list endpoints read from it, so Desktop and readiness cannot disagree.
  9. Only structural corruption latches: bare ``SQLITE_CORRUPT`` / ``SQLITE_NOTADB`` with no
  10. FTS provenance. FTS-scoped damage has its own fail-open path with canonical rows intact,
  11. and a malformed-schema row is healed by the web open path, so neither is reported here.
  12. The latch never clears on its own. A corrupt image does not heal, and a store that
  13. flickers between "ok" and "corrupt" is the silent failure this replaces. It resets when
  14. the process restarts, which is the recovery boundary ``StateDbCorruptError`` already
  15. documents (stop Hermes, recover or restore, start again). No marker is written to disk:
  16. the file it would describe is the one that is damaged.
  17. """
  18. from __future__ import annotations
  19. import logging
  20. import sqlite3
  21. import threading
  22. from pathlib import Path
  23. from typing import Dict, Optional
  24. from hermes_state_errors import (
  25. classify_persistence_error,
  26. is_fts_scoped_corruption_error,
  27. is_malformed_schema_error,
  28. )
  29. logger = logging.getLogger(__name__)
  30. STORAGE_OK = "ok"
  31. STORAGE_CORRUPT = "corrupt"
  32. _lock = threading.Lock()
  33. _corrupt: dict[str, str] = {} # resolved db path -> first error text (log only, never served)
  34. def _key(db_path) -> str:
  35. return str(Path(db_path).expanduser().resolve(strict=False))
  36. def is_structural_corruption_error(exc: BaseException) -> bool:
  37. """Canonical B-tree/schema/freelist damage: a corrupt/NOTADB error SQLite does not scope to
  38. the FTS index, and not the malformed-schema case the web open path repairs."""
  39. return (
  40. isinstance(exc, sqlite3.DatabaseError)
  41. and not is_fts_scoped_corruption_error(exc)
  42. and not is_malformed_schema_error(exc)
  43. and classify_persistence_error(exc) == "corrupt"
  44. )
  45. def mark_storage_corrupt(db_path, reason: object) -> None:
  46. """Latch *db_path* as corrupt for the life of this process (idempotent)."""
  47. key = _key(db_path)
  48. with _lock:
  49. if key in _corrupt:
  50. return
  51. _corrupt[key] = str(reason)
  52. logger.error(
  53. "state.db at %s is structurally corrupt (%s); session storage is reported as corrupt "
  54. "until Hermes restarts on a recovered or restored file. Stop Hermes, then run "
  55. "`hermes sessions recover --source %s --inspect-only` or restore a snapshot.",
  56. db_path, reason, db_path,
  57. )
  58. def note_storage_error(db_path, exc: BaseException) -> bool:
  59. """Latch *db_path* when *exc* is structural corruption; True when it was."""
  60. if not is_structural_corruption_error(exc):
  61. return False
  62. mark_storage_corrupt(db_path, exc)
  63. return True
  64. def storage_state(db_path) -> str:
  65. """``"corrupt"`` once this process has seen structural corruption on *db_path*, else ``"ok"``."""
  66. with _lock:
  67. return STORAGE_CORRUPT if _key(db_path) in _corrupt else STORAGE_OK
  68. def storage_corrupt_reason(db_path) -> Optional[str]:
  69. """The first error text latched for *db_path* (logs/diagnostics only), or None."""
  70. with _lock:
  71. return _corrupt.get(_key(db_path))
  72. def reset_storage_state(db_path=None) -> None:
  73. """Forget the latch for *db_path* (all paths when None). For tests and a verified in-process
  74. recovery; nothing in the runtime clears it on its own."""
  75. with _lock:
  76. if db_path is None:
  77. _corrupt.clear()
  78. else:
  79. _corrupt.pop(_key(db_path), None)
  80. __all__ = [
  81. "STORAGE_CORRUPT",
  82. "STORAGE_OK",
  83. "is_structural_corruption_error",
  84. "mark_storage_corrupt",
  85. "note_storage_error",
  86. "reset_storage_state",
  87. "storage_corrupt_reason",
  88. "storage_state",
  89. ]