hermes_state_errors.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. """Exception types and error-classification predicates for the state store.
  2. Shared by hermes_state and its mixins; string predicates match wrapped RPC
  3. strings as well as live sqlite3 exceptions."""
  4. import errno
  5. import re
  6. import sqlite3
  7. # Malformed schema: ``sqlite_master`` itself is inconsistent (typically a DUPLICATE
  8. # ``CREATE VIRTUAL TABLE messages_fts`` row). SQLite parses the whole schema while
  9. # preparing the FIRST statement, so EVERY statement raises (even ``PRAGMA
  10. # journal_mode`` during __init__); only ``PRAGMA writable_schema=ON`` +
  11. # sqlite_master surgery still work. Canonical rows are intact; recovery rebuilds
  12. # only the FTS layer.
  13. _MALFORMED_SCHEMA_MARKERS = ("malformed database schema",)
  14. _MALFORMED_DB_MARKERS = (*_MALFORMED_SCHEMA_MARKERS, "database disk image is malformed")
  15. def is_malformed_db_error(exc: BaseException) -> bool:
  16. """Malformed-schema OR generic corrupt-image error. Diagnostics / offline
  17. recovery only — runtime repair must use :func:`is_malformed_schema_error`."""
  18. return isinstance(exc, sqlite3.DatabaseError) and any(
  19. marker in str(exc).lower() for marker in _MALFORMED_DB_MARKERS
  20. )
  21. # SQLITE_IOERR as a substring (wrapped strings still classify).
  22. _DISK_IO_ERROR_MARKER = "disk i/o error"
  23. # "Store BUSY, not gone" — HTTP callers map these to 503 instead of 500. Corruption
  24. # is deliberately absent: a malformed store must surface, not be retried into a timeout.
  25. _TRANSIENT_SQLITE_MARKERS = (
  26. _DISK_IO_ERROR_MARKER, "database is locked", "database table is locked", "busy",
  27. )
  28. # Lock contention by result code. SQLite keeps SQLITE_BUSY when FTS5's xConnect loses the race
  29. # on its %_config read but replaces the text with "vtable constructor failed: messages_fts",
  30. # so a phrase match read a busy store as a hard failure.
  31. _SQLITE_LOCK_CODES = (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)
  32. def _sqlite_primary_code(exc_or_str) -> int | None:
  33. """Primary result code (extended codes keep it in the low byte); None when unknown."""
  34. code = getattr(exc_or_str, "sqlite_errorcode", None)
  35. return code & 0xFF if isinstance(code, int) else None
  36. def is_sqlite_lock_error(exc_or_str) -> bool:
  37. """SQLITE_BUSY / SQLITE_LOCKED: wait and retry, never treat as damage. A known result code
  38. decides; only without one (our own re-raised messages, RPC-wrapped strings) does the text."""
  39. code = _sqlite_primary_code(exc_or_str)
  40. if code is not None:
  41. return code in _SQLITE_LOCK_CODES
  42. text = str(exc_or_str).lower()
  43. return "locked" in text or "busy" in text
  44. def _is_no_more_rows(exc: sqlite3.Error) -> bool:
  45. """Transient engine error on contended WAL appends (retries like locked/busy);
  46. message-scoped because some builds raise it as InterfaceError."""
  47. return "no more rows available" in str(exc).lower()
  48. def is_transient_sqlite_error(exc: BaseException) -> bool:
  49. """"Busy right now", not "damaged": one predicate so retry and the HTTP
  50. 503-vs-500 split cannot drift apart."""
  51. return isinstance(exc, sqlite3.OperationalError) and (
  52. is_sqlite_lock_error(exc) or any(marker in str(exc).lower() for marker in _TRANSIENT_SQLITE_MARKERS)
  53. )
  54. def is_malformed_schema_error(exc: BaseException) -> bool:
  55. """Only SQLite's explicit malformed-schema text: a generic "disk image is
  56. malformed" may be any B-tree page, so runtime repair must fail closed on it."""
  57. return isinstance(exc, sqlite3.DatabaseError) and any(
  58. marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS
  59. )
  60. # "Filesystem cannot accept another write" substrings (OSError, sqlite3, wrapped RPC strings).
  61. _DISK_FULL_MARKERS = (
  62. "no space left on device", "not enough space", "database or disk is full", # SQLITE_FULL
  63. "disk full", "full disk", "enospc",
  64. )
  65. def is_disk_full_error(exc: BaseException | str | None) -> bool:
  66. """Disk-full / ENOSPC: OSError(ENOSPC), SQLITE_FULL, or matching strings."""
  67. if exc is None:
  68. return False
  69. if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC:
  70. return True
  71. lowered = (exc if isinstance(exc, str) else str(exc)).lower()
  72. return any(marker in lowered for marker in _DISK_FULL_MARKERS)
  73. # Every classify_persistence_error bucket; consumers enumerate this tuple.
  74. PERSISTENCE_ERROR_CAUSES = (
  75. "locked", "compression", "compression_closed", "turn_lease", "corrupt", "fts_index",
  76. "replaced", "deleted_wal", "disk", "session_row_missing", "unknown",
  77. )
  78. # "Database FILE structurally damaged" substrings. "database disk image is
  79. # malformed" contains "disk", so this check MUST run before the disk bucket in
  80. # classify_persistence_error or B-tree corruption reads as "free some disk space".
  81. # Kept as plain substrings so sqlite3.DatabaseError, wrapped RPC strings, and logged message text all match
  82. # the same helper. See #77386.
  83. _DB_CORRUPTION_MARKERS = (
  84. "malformed", "file is not a database", "not a database", "database corruption",
  85. )
  86. # The module constant exists on Python 3.11+; the numeric value is stable across SQLite releases.
  87. SQLITE_CORRUPT_VTAB = getattr(sqlite3, "SQLITE_CORRUPT_VTAB", 267)
  88. # Every FTS object hangs off this prefix: the virtual tables and their _data/_idx/_content/
  89. # _docsize/_config shadow b-trees. FTS5 names the table in its own corruption reports.
  90. _FTS_OBJECT_RE = re.compile(r"\bmessages_fts\w*")
  91. def is_fts_scoped_corruption_error(exc_or_str) -> bool:
  92. """Corruption SQLite itself attributes to the FTS index layer: the ONE provenance rule
  93. shared by the write-repair gate (``SessionDB._is_fts_write_corruption_error``), the
  94. gateway transcript retry and :func:`classify_persistence_error` (#96038, #97794).
  95. A known result code outranks prose: ``SQLITE_CORRUPT_VTAB`` is FTS-scoped even with
  96. the generic malformed-image text older SQLite builds emit, while bare ``SQLITE_CORRUPT``
  97. / ``SQLITE_NOTADB`` carry no object scope and any other known code contradicts
  98. FTS-looking prose, so both fail closed. Only without a code (Python < 3.11, RPC-wrapped
  99. strings) does the text decide, and then only an ``fts5:`` corruption report or a
  100. corruption marker that names a ``messages_fts*`` object counts.
  101. """
  102. if exc_or_str is None:
  103. return False
  104. code = getattr(exc_or_str, "sqlite_errorcode", None)
  105. if code is not None:
  106. return code == SQLITE_CORRUPT_VTAB
  107. text = (exc_or_str if isinstance(exc_or_str, str) else str(exc_or_str)).lower()
  108. if not _FTS_OBJECT_RE.search(text):
  109. return False
  110. if text.startswith("fts5:") and "corrupt" in text:
  111. return True
  112. return any(marker in text for marker in _DB_CORRUPTION_MARKERS)
  113. class CompressionSessionClosedError(RuntimeError):
  114. """A durable write targeted a parent already closed by compression."""
  115. def __init__(self, session_id: str):
  116. self.session_id = session_id
  117. super().__init__(
  118. f"Session {session_id!r} is closed by compression; "
  119. "adopt its live continuation before appending messages"
  120. )
  121. class CompressionSessionBusyError(RuntimeError):
  122. """A non-owner tried to write while compression owns the session."""
  123. class SessionCompressionInProgressError(CompressionSessionBusyError):
  124. """A concurrent writer collided with a *live* compression lock — transient
  125. (the compressor publishes in seconds; ``_execute_write`` waits), unlike the
  126. parent class's other case (a compressor whose own lease is gone: permanent,
  127. fail fast). Subclassing keeps every existing handler working."""
  128. class PruneRowUnresolvedError(RuntimeError):
  129. """An in-place prune rewrite could not name the exact live row of a changed message. Raised before
  130. any write, so the caller commits through ``archive_and_compact`` instead."""
  131. class PruneRowStaleError(PruneRowUnresolvedError):
  132. """The named live row no longer holds the body the pruner is pruning: another writer already
  133. rewrote this generation. Unlike plain ambiguity this must NOT fall back to the full writer —
  134. that writer archives rows it never compared and would republish the stale held text over the
  135. newer one. Subclassing keeps callers that only know the parent failing closed-enough (they stop
  136. the in-place path); the prune caller tells the two apart and skips the pass entirely."""
  137. class HeldRowRewrittenError(RuntimeError):
  138. """A full-writer commit found one of the held rows it was about to archive already rewritten in
  139. place by another writer. Raised INSIDE the commit's write transaction, after the caller's
  140. precheck but before any row is archived: a write that lands in that window is exactly the lost
  141. race the precheck cannot see (#124102). Callers treat it like ``StaleHeldHistory`` — nothing
  142. written, the pass is a no-op and the newer generation keeps the session."""
  143. class SessionTurnLeaseLostError(RuntimeError):
  144. """A transcript write presented a turn-lease holder that no longer owns it.
  145. Fail-fast fencing (no ``_execute_write`` retry): a later writer may already
  146. be persisting a newer turn, and landing this one would interleave a stale reply."""
  147. class SessionActiveWriteGuardError(RuntimeError):
  148. """Raised when an active turn lease or compression lock rejects session deletion."""
  149. class StateDbReplacedError(RuntimeError):
  150. """The state.db path no longer names the file this SessionDB opened
  151. (out-of-band cp/mv/restore). In-place FTS repair and fail-open trigger
  152. dropping cannot fix a generation mismatch; they amplify it."""
  153. class DeletedWalGenerationError(StateDbReplacedError):
  154. """A live process holds a deleted state.db-wal / -shm generation. Opening or
  155. writing through this handle would mint a second WAL inode (split-brain ->
  156. intermittent SQLITE_CORRUPT / IOERR). Stop the writers; never unlink the WAL
  157. yourself. Subclasses StateDbReplacedError so every consumer that diverts
  158. transcripts on a replaced store handles this identically."""
  159. # SQLite header application_id (offset 68). Distinct from inode: ``cp`` onto the
  160. # same path keeps st_ino and truncates+rewrites.
  161. _STATE_DB_APPLICATION_ID_OFFSET = 68
  162. _STATE_DB_GENERATION_KEY = "db_file_generation"
  163. _STATE_DB_REPLACED_MSG = (
  164. "FATAL: state.db was replaced underneath the gateway; refusing further "
  165. "writes to this file. Divert transcripts to sessions/<id>.jsonl (and the "
  166. "gateway pending_messages spool) and restore or reopen after operator intervention."
  167. )
  168. STORAGE_RECOVERY_DOCS_URL = "https://hermes-agent.nousresearch.com/docs/user-guide/session-storage-recovery"
  169. # Two layers (#110054): the first sentence is for the person reading a chat bubble or a banner (what
  170. # happened, nothing is lost, the one thing to do); the rest is the operator detail. The phrase
  171. # "deleted state.db-wal or state.db-shm" is the classifier's RPC-wrapped fingerprint — keep it.
  172. _DELETED_WAL_GENERATION_MSG = (
  173. "FATAL: session storage stopped writing because another Hermes process still holds a deleted "
  174. "state.db-wal or state.db-shm inode (an old copy of the write-ahead log). Nothing is lost: quit "
  175. "every Hermes process on this profile (Desktop app, gateway, dashboard, cron), run `hermes doctor` "
  176. "(it names the processes still holding the log), then start Hermes again. Do not delete the WAL "
  177. "yourself and do not run `hermes doctor --fix` while they are running. "
  178. f"Guide: {STORAGE_RECOVERY_DOCS_URL} "
  179. "Detail: the path names a different (or missing) generation than the one this process holds "
  180. "open; opening or writing through it would mint a second WAL (split-brain). "
  181. "database.journal_mode: delete is operator containment, not a new default."
  182. )
  183. class StateDbCorruptError(sqlite3.DatabaseError):
  184. """A live SessionDB observed structural (non-FTS, non-replaced) corruption and
  185. is quarantined: sticky for the handle's life — writes fail fast, no reopen,
  186. no close-time checkpoint (a handle that kept writing after the first error
  187. checkpointed 15 pages under wrong page numbers and turned a readable file
  188. into "file is not a database"; SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE on 3.12+
  189. also stops SQLite's own). Subclasses sqlite3.DatabaseError so every degrade
  190. path keeps working. Recovery boundary: restart on a repaired/restored file.
  191. Stopping the writes is what prevents that; skipping the explicit checkpoint is the second line of
  192. defence. SQLite still runs its own last-connection checkpoint inside ``close()`` (and deletes the
  193. ``-wal`` sidecar) unless ``SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE`` is set — Python exposes it via
  194. ``Connection.setconfig()`` on 3.12+, so quarantine disables the close-time checkpoint there and the WAL
  195. survives on disk for forensics; on 3.11 the internal checkpoint is unavoidable (post-quarantine it can
  196. only carry pre-corruption committed frames, since no further writes are accepted). See #90837.
  197. """
  198. _STATE_DB_CORRUPT_MSG = (
  199. "FATAL: state.db reported structural corruption (database disk image is "
  200. "malformed outside the FTS shadow tables) on a live handle; refusing further "
  201. "writes, automatic reopen, and the close-time WAL checkpoint on this file. "
  202. "Stop the gateway, then run `hermes sessions recover --source <state.db> "
  203. "--inspect-only` or restore a snapshot. Unwritten transcripts are diverted to "
  204. "sessions/<id>.jsonl (and the gateway pending_messages spool)."
  205. )
  206. _PERSISTENCE_CAUSE_BY_TYPE = (
  207. (SessionTurnLeaseLostError, "turn_lease"),
  208. (CompressionSessionClosedError, "compression_closed"),
  209. (CompressionSessionBusyError, "compression"),
  210. # The WAL-generation error subclasses StateDbReplacedError so existing write diversion keeps
  211. # working; classify it first because its recovery artifact and operator action are different.
  212. (DeletedWalGenerationError, "deleted_wal"),
  213. (StateDbReplacedError, "replaced"),
  214. (StateDbCorruptError, "corrupt"),
  215. )
  216. _PERSISTENCE_CAUSE_BY_PHRASE = (
  217. (("turn lease",), "turn_lease"),
  218. (("closed by compression",), "compression_closed"),
  219. (("being compressed", "compression lease"), "compression"),
  220. # RPC-wrapped errors lose their exception type; retain the same sidecar/main-file split.
  221. (("deleted state.db-wal", "deleted state.db-shm"), "deleted_wal"),
  222. (("was replaced underneath",), "replaced"),
  223. (_DB_CORRUPTION_MARKERS, "corrupt"),
  224. (("locked", "busy"), "locked"),
  225. # A flush rejected by the session-row FK: the row was removed under a live agent (#123583).
  226. (("foreign key constraint failed",), "session_row_missing"),
  227. )
  228. def classify_persistence_error(exc_or_str) -> str:
  229. """Coarse cause bucket (PERSISTENCE_ERROR_CAUSES) so the user's guidance
  230. matches: "locked" = busy, retry; "disk" = full/read-only/permissions;
  231. "compression" = a live lease refused the write; "compression_closed" = adopt
  232. the rotated session id; "turn_lease" = fencing, not storage; "corrupt" =
  233. file damage (repair path, not disk space); "fts_index" = SQLite scoped the
  234. corruption to the FTS index (the transcript store is not damaged); "replaced" =
  235. main-file replacement; "deleted_wal" = a retired sidecar generation requiring
  236. capture inspection; "session_row_missing" = the session row was deleted under a
  237. live agent (FK rejection; the flush recreates it)."""
  238. if exc_or_str is None:
  239. return "unknown"
  240. # Lease refusals contain neither "locked" nor "busy": match by type first,
  241. # then by phrase for strings that survived RPC wrapping. Order matters:
  242. # StateDbReplacedError covers DeletedWalGenerationError; corruption comes
  243. # BEFORE the lock/disk buckets ("disk image is malformed" contains "disk").
  244. for exc_type, cause in _PERSISTENCE_CAUSE_BY_TYPE:
  245. if isinstance(exc_or_str, exc_type):
  246. return cause
  247. # Provenance before prose: an FTS-scoped result code (or, without one, an fts5 report
  248. # naming messages_fts*) is index damage, never whole-file corruption (#97794).
  249. if is_fts_scoped_corruption_error(exc_or_str):
  250. return "fts_index"
  251. if _sqlite_primary_code(exc_or_str) in _SQLITE_LOCK_CODES:
  252. return "locked"
  253. if getattr(exc_or_str, "sqlite_errorcode", None) == sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY:
  254. return "session_row_missing"
  255. text = str(exc_or_str).lower()
  256. for markers, cause in _PERSISTENCE_CAUSE_BY_PHRASE:
  257. if any(marker in text for marker in markers):
  258. return cause
  259. if is_disk_full_error(exc_or_str) or any(m in text for m in ("disk", "readonly", "read-only")):
  260. return "disk"
  261. return "unknown"