hermes_state.py 93 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734
  1. #!/usr/bin/env python3
  2. """SQLite state store for Hermes Agent: session metadata, message history, model
  3. config, FTS5 search. WAL mode (concurrent readers + one writer); compression
  4. splits sessions via parent_session_id chains; sessions are source-tagged
  5. ('cli', 'telegram', ...). Batch-runner / RL trajectories live elsewhere.
  6. """
  7. import asyncio
  8. import atexit
  9. import hashlib
  10. import json
  11. import logging
  12. import os
  13. import queue
  14. import random
  15. import re
  16. import sqlite3
  17. import stat
  18. import sys
  19. import threading
  20. import time
  21. import uuid
  22. from collections import deque
  23. from contextlib import contextmanager, suppress
  24. from pathlib import Path
  25. from hermes_dir_acl import make_user_dir
  26. from hermes_constants import get_hermes_home, mkdir_under_hermes_home
  27. from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypeVar, cast
  28. from hermes_state_common import (
  29. TITLE_SOURCE_DERIVED as _TITLE_SOURCE_DERIVED, TITLE_SOURCE_LLM as _TITLE_SOURCE_LLM,
  30. TITLE_SOURCE_USER as _TITLE_SOURCE_USER,
  31. escape_like as _escape_like, _placeholders,
  32. stat_db_file_identity as _stat_db_file_identity,
  33. )
  34. from hermes_state_holders import read_only_db_uri
  35. from hermes_state_pidns import holder_pid_checkable
  36. from hermes_state_health import (
  37. STORAGE_CORRUPT, mark_storage_corrupt, note_storage_error, storage_corrupt_reason, storage_state,
  38. )
  39. from hermes_state_errors import (
  40. _DELETED_WAL_GENERATION_MSG, _DISK_IO_ERROR_MARKER, _STATE_DB_CORRUPT_MSG, _STATE_DB_GENERATION_KEY,
  41. _STATE_DB_REPLACED_MSG, DeletedWalGenerationError, SessionCompressionInProgressError, StateDbCorruptError,
  42. StateDbReplacedError, _is_no_more_rows, classify_persistence_error, is_malformed_db_error,
  43. is_malformed_schema_error, is_sqlite_lock_error,
  44. )
  45. from hermes_state_guard import (
  46. _STATE_DB_GUARD_BYPASS_ENV, _in_test_context, _is_production_state_db, _real_platform_state_root,
  47. _register_test_instance, _set_last_init_error, get_last_init_error,
  48. )
  49. from hermes_state_readpool import _READ_POOL_MAX, _proc_fd_targets, _read_budget_for
  50. from hermes_state_sessions import SessionSessionsMixin
  51. from hermes_state_fts import SessionFtsSetupMixin, load_fts5_cjk_extension
  52. from hermes_state_portability import SessionPortabilityMixin
  53. from hermes_state_telegram import SessionTelegramTopicsMixin
  54. from hermes_state_profile_repair import SessionProfileRepairMixin
  55. from hermes_state_schema import SessionSchemaMixin
  56. import hermes_state_holders as _state_holders
  57. import hermes_state_lockguard as _lockguard
  58. from hermes_state_lockowners import log_write_lock_holders
  59. from hermes_state_dbfile import (
  60. _connect_tracked_db, _fd_is_truly_unlinked, _prepare_connection_retirement,
  61. _read_sqlite_application_id, _stat_sqlite_sidecar_identity,
  62. _watched_sqlite_sidecar_paths, has_invalid_sqlite_header_preopen, is_zeroed_state_db, quarantine_cross_process_lock,
  63. quarantine_invalid_state_db,
  64. RetiredGenerationCaptureError, capture_retired_wal_generation, refuse_deleted_wal_generation,
  65. )
  66. from hermes_state_messages import SessionMessagesMixin
  67. from hermes_state_prune_rewrite import SessionPruneRewriteMixin
  68. from hermes_state_api_content import SessionApiContentMixin
  69. from hermes_state_coverage import SessionCoverageMixin
  70. from hermes_state_rewind import SessionRewindMixin
  71. from hermes_state_wal import (
  72. _WAL_INCOMPAT_MARKERS, _on_disk_journal_mode, apply_database_pragmas, apply_wal_with_fallback,
  73. )
  74. from hermes_state_repair import _claim_repair_attempt, preflight_db_writability, repair_state_db_schema
  75. from hermes_state_titles import SessionTitlesMixin
  76. from hermes_state_prompt_refresh import SessionPromptRefreshMixin
  77. from hermes_state_usage import SessionUsageMixin
  78. from hermes_state_maintenance import SessionMaintenanceMixin
  79. from hermes_state_gateway import SessionGatewayMixin
  80. from hermes_state_compression import SessionCompressionMixin
  81. from hermes_state_search import SessionSearchMixin
  82. try: # Hard dependency, but tolerate scaffold-phase imports before pip install.
  83. import psutil
  84. except ImportError: # pragma: no cover - stripped/scaffold installs only
  85. psutil = None # type: ignore[assignment]
  86. logger = logging.getLogger(__name__)
  87. _MAX_SAFE_MESSAGES = 20_000 # resume/export guard default
  88. def _configured_transcript_limit(key: str, fallback: int = _MAX_SAFE_MESSAGES) -> int:
  89. """``sessions.<key>`` from config.yaml (lazy import: circular at load), else *fallback*; 0 disables."""
  90. try:
  91. from hermes_cli.config import load_config_readonly
  92. value = (load_config_readonly().get("sessions") or {}).get(key)
  93. if value is None:
  94. return fallback
  95. limit = int(value)
  96. return limit if limit >= 0 else fallback
  97. except Exception:
  98. return fallback
  99. def resolved_max_resume_messages() -> int:
  100. return _configured_transcript_limit("max_resume_messages")
  101. def resolved_max_export_messages() -> int:
  102. return _configured_transcript_limit("max_export_messages")
  103. class SessionResumeTooLargeError(ValueError):
  104. def __init__(
  105. self, message_count: int, limit: int = _MAX_SAFE_MESSAGES, scope: str = "across its lineage",
  106. ):
  107. self.message_count, self.limit = message_count, limit
  108. self.scope = scope
  109. super().__init__(
  110. f"This session is too long to reload safely ({message_count} messages; limit {limit}). "
  111. "Start a fresh chat and keep a copy with the dashboard Sessions page's Export action or "
  112. "`hermes sessions export --format md --session-id <id>` (neither is capped), or raise the limit "
  113. "with `hermes config set sessions.max_resume_messages 0`."
  114. )
  115. class SessionExportTooLargeError(ValueError):
  116. def __init__(self, session_id: str, message_count: int, limit: int = _MAX_SAFE_MESSAGES):
  117. self.session_id, self.message_count, self.limit = session_id, message_count, limit
  118. # User-facing refusal shared by every in-memory JSON/JSONL export (CLI and console).
  119. super().__init__(
  120. f"Session '{session_id}' has more than {limit:,} exportable messages; the JSON/JSONL "
  121. "backup is built in memory and capped per session. Use the dashboard Sessions page's streaming "
  122. "Export action, or set sessions.max_export_messages: 0 in config.yaml to disable the guard."
  123. )
  124. def _compression_lock_holder_process_is_dead(holder: str) -> bool:
  125. """True only when a ``pid=<n>`` lock holder's local PID is provably gone.
  126. Reclaim on kernel proof only: unstructured/same-process holders (another
  127. thread's live lease) and any probe doubt keep the lease until TTL expiry
  128. (PID reuse must never steal a live lease; a wrongly-kept one self-heals).
  129. Foreign/unstamped PID namespaces defer to TTL: see ``hermes_state_pidns``."""
  130. match = re.search(r"(?:^|:)pid=(\d+)(?::|$)", holder or "")
  131. pid = int(match.group(1)) if match else 0
  132. if pid <= 0 or pid == os.getpid():
  133. return False
  134. if not holder_pid_checkable(holder):
  135. return False # foreign / unknown namespace: defer to TTL
  136. if psutil is not None:
  137. try:
  138. return not psutil.pid_exists(pid) # recycled PIDs read as alive (conservative)
  139. except Exception:
  140. return False
  141. # psutil-less fallback is POSIX-only: on Windows os.kill(pid, 0) maps sig=0 to
  142. # CTRL_C_EVENT and can kill the target's console group.
  143. if os.name == "nt":
  144. return False
  145. try:
  146. os.kill(pid, 0) # windows-footgun: ok — nt early-returns just above
  147. except ProcessLookupError:
  148. return True
  149. except (OSError, OverflowError): # PermissionError is an OSError: alive but foreign
  150. return False
  151. return False
  152. # Billing buckets that aren't a routable provider identity: a session that persisted only
  153. # one of these (never ran /model) falls back to the config default. Shared by
  154. # session_gateway_runtime and tui_gateway.server so they cannot drift.
  155. _BARE_BILLING_PROVIDERS = frozenset({"auto", "custom"})
  156. T = TypeVar("T")
  157. # Import-time snapshot lets _default_db_path() detect a re-pointed DEFAULT_DB_PATH
  158. # (tests monkeypatch the constant directly).
  159. DEFAULT_DB_PATH = _IMPORT_DEFAULT_DB_PATH = get_hermes_home() / "state.db"
  160. # Back off from read-only opens after one fails: not per query, but short enough that
  161. # transient fd pressure doesn't strand the read pool.
  162. _READ_OPEN_RETRY_SECONDS = 60.0
  163. # Transient SQLITE_IOERR retry budget for READ-ONLY opens (#100436): a WAL writer's checkpoint/
  164. # reset/frame flush surfaces "disk I/O error" to a concurrent mode=ro reader for a millisecond-
  165. # wide window — the ro connection cannot perform WAL recovery because recovery writes the -shm
  166. # index, which mode=ro refuses. The writer closes the window on its own, so a few short retries
  167. # make the open succeed instead of 500-ing the whole /api/sessions poll (or any other ro opener).
  168. # Deliberately NOT for writable opens: a writer owns the transition, so an IOERR there is a real
  169. # storage/fd problem. A persistent IOERR still exhausts the budget and propagates.
  170. _READ_ONLY_IOERR_RETRY_ATTEMPTS, _READ_ONLY_IOERR_RETRY_BACKOFF_S = 3, 0.05
  171. # SQLite busy handler budget for reads. Under DELETE (rollback-journal) mode a reader needs a SHARED
  172. # lock, which every commit from another process blocks across its journal+db fsyncs; the writer
  173. # connection's 1 s timeout exists for writes (they retry at application level) and starved readers
  174. # into "database is locked" (dashboard 503s, `hermes sessions list` crashes) under a busy gateway.
  175. _READ_BUSY_TIMEOUT_S = 5.0
  176. def _default_db_path() -> Path:
  177. """Default state DB path at CALL time: a re-pointed ``DEFAULT_DB_PATH`` wins, else
  178. ``get_hermes_home()`` is resolved fresh (a runtime HERMES_HOME redirect works regardless of import)."""
  179. return DEFAULT_DB_PATH if DEFAULT_DB_PATH != _IMPORT_DEFAULT_DB_PATH else get_hermes_home() / "state.db"
  180. # Live-DB guard knobs live HERE (not in hermes_state_guard): the hermetic conftest monkeypatches
  181. # ``hermes_state._STATE_DB_GUARD_BYPASS`` (``@pytest.mark.live_system_guard_bypass`` escape hatch)
  182. # and ``_EXTRA_DENY_ROOTS`` (the pre-sandbox root, so custom-HERMES_HOME deployments are covered).
  183. _STATE_DB_GUARD_BYPASS = False
  184. _STATE_DB_GUARD_EXTRA_DENY_ROOTS: tuple[Path, ...] = ()
  185. def _ensure_test_isolation(db_path: Path) -> None:
  186. """Raise before any connection/mkdir/pragma/byte probe when a pytest-context process
  187. (env OR ancestry) resolves a production DB.
  188. Env alone is not enough: a child spawned with a rebuilt environment loses ``PYTEST_*`` and
  189. ``HERMES_HOME`` together, which is precisely the state in which it writes to production (#82770).
  190. """
  191. if _STATE_DB_GUARD_BYPASS or os.environ.get(_STATE_DB_GUARD_BYPASS_ENV) or not _in_test_context():
  192. return
  193. try:
  194. resolved = Path(db_path).expanduser().resolve()
  195. except Exception:
  196. return
  197. roots = [r for r in (_real_platform_state_root(),) if r is not None]
  198. for extra in _STATE_DB_GUARD_EXTRA_DENY_ROOTS:
  199. try:
  200. roots.append(Path(extra).expanduser().resolve())
  201. except Exception:
  202. continue
  203. for root in roots:
  204. if _is_production_state_db(resolved, root):
  205. raise RuntimeError(
  206. "live-system guard: test attempted to open production "
  207. f"state.db at {resolved} (under real Hermes root {root}). "
  208. "Tests must run against a temporary HERMES_HOME — pass an "
  209. "explicit tmp db_path or let the hermetic conftest redirect "
  210. "HERMES_HOME. If this test genuinely needs the live database, mark it with "
  211. "@pytest.mark.live_system_guard_bypass — or, for a spawned "
  212. f"child process, export {_STATE_DB_GUARD_BYPASS_ENV}=1 in "
  213. "its environment."
  214. )
  215. def _secure_state_db_files(db_path: Path, *, create_main: bool = False) -> None:
  216. """Create/tighten a writable state database and its sidecars to 0600.
  217. SQLite otherwise creates ``state.db``, ``-wal``, and ``-shm`` according to
  218. the process umask (commonly 0644 under 0022). Read-only SessionDB
  219. attachments never call this helper and remain observational.
  220. Existing files are tightened with ``chmod(2)`` on the path: opening the
  221. file and closing that descriptor would drop every POSIX ``fcntl`` lock the
  222. process holds on its inode — including the locks of an already-open SQLite
  223. connection to the same database. A lock-losing close in one process lets a
  224. sibling's connection take the shared-memory DMS exclusively at its own
  225. close, checkpoint, and unlink the sidecars while long-lived holders
  226. (gateway, desktop ``hermes serve``) keep using the deleted inodes.
  227. """
  228. if os.name == "nt":
  229. return
  230. main_path = db_path
  231. if create_main:
  232. # O_EXCL: only a brand-new inode gets a descriptor. Opening an existing
  233. # file here and closing it would drop this process's POSIX locks on it.
  234. flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
  235. if hasattr(os, "O_NOFOLLOW"):
  236. flags |= os.O_NOFOLLOW
  237. if hasattr(os, "O_CLOEXEC"):
  238. flags |= os.O_CLOEXEC
  239. try:
  240. fd = os.open(main_path, flags, 0o600)
  241. except FileExistsError:
  242. pass
  243. except IsADirectoryError:
  244. # Not a database file at all; sqlite3.connect() raises the
  245. # canonical error for this, and a directory leaks no row data.
  246. return
  247. else:
  248. os.close(fd)
  249. for path in (
  250. main_path,
  251. db_path.with_name(db_path.name + "-wal"),
  252. db_path.with_name(db_path.name + "-shm"),
  253. ):
  254. # fchmod on an fd of a pre-existing file cannot be used here: close(fd)
  255. # would release this process's POSIX locks on that inode, stripping the
  256. # locks of any live SQLite connection to the same database. chmod(2)
  257. # never opens the file, so it leaves the lock state untouched.
  258. try:
  259. st = os.lstat(path)
  260. except FileNotFoundError:
  261. continue
  262. if stat.S_ISLNK(st.st_mode):
  263. # Refuse a planted symlink exactly like O_NOFOLLOW would.
  264. continue
  265. if not stat.S_ISREG(st.st_mode):
  266. continue
  267. os.chmod(path, 0o600)
  268. # Openings of the background-review harness prompts (agent/background_review.py).
  269. _REVIEW_HARNESS_PREFIXES = (
  270. "Review the conversation above and update the skill library",
  271. "Review the conversation above and consider saving to memory",
  272. )
  273. def _is_background_review_harness_message(msg: dict[str, Any]) -> bool:
  274. """Persisted harness prompt (older builds wrote the forked curator's turns
  275. into real sessions; replaying them hijacks the session)."""
  276. if not isinstance(msg, dict) or msg.get("role") not in {"user", "system"}:
  277. return False
  278. content = msg.get("content")
  279. return isinstance(content, str) and content.lstrip().startswith(_REVIEW_HARNESS_PREFIXES)
  280. def _strip_background_review_harness(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  281. """Drop harness messages and the curator-mode assistant reply that immediately followed each."""
  282. if not messages:
  283. return messages
  284. out: list[dict[str, Any]] = []
  285. skip_next_assistant = False
  286. previous_was_harness = False
  287. for msg in messages:
  288. if _is_background_review_harness_message(msg):
  289. # A consecutive harness prompt occupies the preceding prompt's
  290. # immediate reply slot, so it must not arm another assistant skip.
  291. skip_next_assistant = not previous_was_harness
  292. previous_was_harness = True
  293. continue
  294. if skip_next_assistant:
  295. skip_next_assistant = False
  296. if isinstance(msg, dict) and msg.get("role") == "assistant":
  297. previous_was_harness = False
  298. continue # the curator-mode reply to the harness prompt
  299. previous_was_harness = False
  300. out.append(msg)
  301. return out
  302. # Matches a bare protocol/tool-name marker such as "[memory]" or "[skill_manage]".
  303. _STALE_TOOL_CALL_MARKER_RE = re.compile(r"^\[[A-Za-z_][A-Za-z0-9_.-]*\]$")
  304. def _is_stale_tool_call_marker_message(msg: dict[str, Any]) -> bool:
  305. """Assistant tool-call turn whose content is a bare ``[marker]`` (an older
  306. conversation_loop persisted a local template's marker as the final response)."""
  307. if not isinstance(msg, dict) or msg.get("role") != "assistant" or not msg.get("tool_calls"):
  308. return False
  309. content = msg.get("content")
  310. return isinstance(content, str) and bool(_STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()))
  311. def _strip_stale_tool_call_markers(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  312. """Blank stale ``[marker]`` assistant content (replaying it teaches the model
  313. to keep emitting it); tool_call/result pairing stays intact."""
  314. repaired = 0
  315. for msg in filter(_is_stale_tool_call_marker_message, messages):
  316. msg["content"] = ""
  317. repaired += 1
  318. if repaired:
  319. logger.info(
  320. "Cleared %d stale tool-call marker message(s) while restoring session (#78148)", repaired,
  321. )
  322. return messages
  323. _SESSION_DB_CONSEQUENCE = "Sessions will not be saved until this is fixed."
  324. _NETWORK_DRIVE_HINT = " If the database lives on a network drive, move it to a local disk."
  325. _NETWORK_DRIVE_GLOSS = "the session database could not be opened; it may be on a network or unsupported drive"
  326. _NETWORK_DRIVE_ACTION = (
  327. "Move it to a local disk (`hermes {profile_arg}doctor` shows where it is), then start Hermes again."
  328. )
  329. def format_session_db_unavailable(
  330. prefix: str = "Hermes can't open its session history right now",
  331. *,
  332. details: bool = False,
  333. ) -> str:
  334. """User-facing one-liner: ``<prefix>: <gloss>. <consequence> <action>[ network hint]``.
  335. The cause table lives in ``hermes_state_user_copy`` so CLI, gateway and TUI agree. Chat
  336. surfaces (gateway, TUI) get the one-liner; ``details=True`` (the CLI banner) appends a
  337. ``Details: <raw cause>`` line for the raw SQLite text. Network filesystems (NFS/SMB/FUSE/ZFS)
  338. cannot host SQLite's write-ahead log: when the raw cause carries one of those markers the
  339. message names the network-drive suspicion, because ``hermes doctor --fix`` cannot repair a
  340. mount — only moving the file can."""
  341. from hermes_constants import profile_cli_selector
  342. profile_arg = profile_cli_selector()
  343. cause = get_last_init_error()
  344. if not cause:
  345. return (
  346. f"{prefix}. {_SESSION_DB_CONSEQUENCE} Run `hermes {profile_arg}doctor` to check the "
  347. "storage location."
  348. )
  349. from hermes_state_user_copy import describe_storage_failure
  350. failure = describe_storage_failure(cause)
  351. gloss, action, hint = failure.gloss, failure.action, ""
  352. if any(m in cause.lower() for m in _WAL_INCOMPAT_MARKERS):
  353. if failure.cause == "unknown":
  354. gloss, action = _NETWORK_DRIVE_GLOSS, _NETWORK_DRIVE_ACTION.replace("{profile_arg}", profile_arg)
  355. else:
  356. hint = _NETWORK_DRIVE_HINT
  357. text = f"{prefix}: {gloss}. {_SESSION_DB_CONSEQUENCE} {action}{hint}"
  358. if details:
  359. from hermes_state_user_copy import storage_failure_details
  360. text += f"\nDetails: {storage_failure_details(cause)}"
  361. return text
  362. # Auto-repair at most once per DB path per process (no repair loops; serialises concurrent
  363. # web_server / gateway opens on the same malformed file).
  364. _repair_attempted_paths: set[str] = set()
  365. _repair_attempt_lock = threading.Lock()
  366. # Cross-process schema-surgery lock timeout (``_repair_attempt_lock`` covers one interpreter
  367. # only); sized for the slowest legitimate holder (VACUUM, multi-GB DB).
  368. _REPAIR_LOCK_TIMEOUT_SECONDS = 120.0
  369. _IS_WINDOWS = sys.platform == "win32"
  370. def _close_time_checkpoint_configurable() -> bool:
  371. """Whether this runtime can switch off SQLite's close-time checkpoint (Python 3.12+ ``setconfig``)."""
  372. return (getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None) is not None
  373. and hasattr(sqlite3.Connection, "setconfig"))
  374. def divert_session_transcript_jsonl(session_id: str, messages) -> Optional[Path]:
  375. """Append pending messages to HERMES_HOME/sessions/<id>.jsonl (state.db was replaced under a
  376. live process). Returns the path, or None if nothing to write."""
  377. sid = str(session_id or "").strip()
  378. if not sid or not messages:
  379. return None
  380. sessions_dir = get_hermes_home() / "sessions"
  381. make_user_dir(sessions_dir, parents=True, exist_ok=True)
  382. path = sessions_dir / f"{sid}.jsonl"
  383. with path.open("a", encoding="utf-8") as handle:
  384. for msg in messages:
  385. if msg is not None:
  386. record = msg if isinstance(msg, dict) else {"content": str(msg)}
  387. handle.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
  388. return path
  389. # Process-wide shared SessionDB registry: long-lived in-process callers share ONE writer
  390. # connection per resolved path via hermes_state_registry.acquire(); one-shots use SessionDB() + close().
  391. def _foreign_state_db_holders(db_path: Path) -> list[tuple[int, str]]:
  392. """Compatibility delegate to the state-holder authority."""
  393. return _state_holders.foreign_state_db_holders(db_path)
  394. # ── Process-wide shared SessionDB registry (#90837) ── lives in hermes_state_registry.py (acquire /
  395. # release / close_all / release_or_close). Long-lived in-process callers (gateway, tui_gateway, cron,
  396. # in-process tools) share ONE writer connection per resolved path via hermes_state_registry.acquire(); CLI
  397. # one-shots, recovery flows, and read-only cross-profile opens use SessionDB() directly with their own close().
  398. class SessionDB(
  399. SessionSessionsMixin, SessionFtsSetupMixin, SessionSearchMixin, SessionSchemaMixin,
  400. SessionPortabilityMixin, SessionTelegramTopicsMixin, SessionCompressionMixin,
  401. SessionGatewayMixin, SessionMaintenanceMixin, SessionUsageMixin, SessionTitlesMixin,
  402. SessionMessagesMixin, SessionPruneRewriteMixin, SessionApiContentMixin, SessionCoverageMixin, SessionRewindMixin, SessionProfileRepairMixin,
  403. SessionPromptRefreshMixin,
  404. ):
  405. """SQLite-backed session storage with FTS5 search; many reader threads, one writer (WAL)."""
  406. # Only these state-owned producers join automatic stale-open reconciliation; messaging/UI
  407. # sources have their own lifecycle owners; unknown sources fail closed.
  408. # See #60609. `recovered` = placeholders `hermes sessions recover` synthesizes for
  409. # orphaned messages (no live owner, never stamped ended_at); without it they are immortal.
  410. _AUTO_PRUNE_STALE_OPEN_SOURCES: tuple[str, ...] = (
  411. "cli", "cron", "kanban", "acp", "api_server", "subagent", "tool", "recovered",
  412. )
  413. # ── Write-contention tuning ──
  414. # SQLite's deterministic busy handler convoys under many hermes processes: keep its
  415. # timeout short (1s) and retry with random jitter. Patience is TIME-based (a sibling
  416. # legitimately holds the lock for seconds: checkpoint at close, VACUUM, recovery, FTS
  417. # optimize); attempt-counted budgets destroyed turns on a healthy store. Transcript
  418. # writes (failure aborts the turn) get the long budget; observation-only activity
  419. # writes sit on the response-critical path and get a sub-second one.
  420. _WRITE_PATIENCE_S, _TRANSCRIPT_WRITE_PATIENCE_S, _ACTIVITY_WRITE_PATIENCE_S = 20.0, 60.0, 0.5
  421. # A live compression lock gets a short wait (compression publishes in seconds), but the lease
  422. # is a correctness boundary: a writer still locked out afterwards is refused.
  423. # Observation-only activity heartbeat/label writes (#76354 review S1): these run on (or adjacent to) the
  424. # response-critical path and must never wait out the full routine patience under contention. Sub-second
  425. # budget; a skipped write is retried naturally at the next heartbeat window.
  426. # A live compression lock gets its own, much shorter budget than the write lock. Compression publishes
  427. # in a couple of seconds, so a brief wait saves the overwhelming majority of concurrent turns (#75083).
  428. # It deliberately stays short: the lease is a correctness boundary, not just a busy signal (see
  429. # test_compression_lease_blocks_non_owner_but_allows_owner_flush), so a writer that is still locked out
  430. # after this budget must still be refused rather than allowed to land a stale turn in a session whose
  431. # compression is genuinely long-running or wedged.
  432. _COMPRESSION_BUSY_WAIT_S = 5.0
  433. _WRITE_RETRY_MIN_S, _WRITE_RETRY_MAX_S = 0.020, 0.150 # fast jitter for the first _SLOW_AFTER_S
  434. _WRITE_RETRY_SLOW_AFTER_S = 2.0
  435. _WRITE_RETRY_SLOW_MIN_S, _WRITE_RETRY_SLOW_MAX_S = 0.250, 1.000
  436. # PASSIVE WAL checkpoint every N successful writes.
  437. _CHECKPOINT_EVERY_N_WRITES = 50
  438. # Bounded FTS ``'merge'`` (ms of lock each) instead of ``'optimize'`` (9-18s per index on a 10GB
  439. # DB, longer than a writer's patience); up to _COMMANDS_PER_PASS per index, stopping on no-progress.
  440. _FTS_MERGE_EVERY_N_WRITES, _FTS_MERGE_MAX_PAGES_PER_INDEX, _FTS_MERGE_COMMANDS_PER_PASS = 1000, 500, 4
  441. # Imports cap lower than exports: an import holds one BEGIN IMMEDIATE.
  442. _IMPORT_MAX_SESSIONS, _IMPORT_MAX_MESSAGES_PER_SESSION, _IMPORT_MAX_TOTAL_MESSAGES = 500, 10_000, 50_000
  443. _IMPORT_MAX_SESSION_BYTES, _IMPORT_MAX_TOTAL_BYTES = 5 * 1024 * 1024, 25 * 1024 * 1024
  444. # Accounting workers retire when idle so a bound-method target can't keep an abandoned SessionDB alive.
  445. _TOKEN_WRITER_IDLE_SECONDS = 30.0
  446. @staticmethod
  447. def _store_system_prompt(conn, system_prompt: Optional[str]) -> Optional[str]:
  448. if system_prompt is None:
  449. return None
  450. prompt_hash = hashlib.sha256(system_prompt.encode("utf-8")).hexdigest()
  451. conn.execute(
  452. "INSERT OR IGNORE INTO system_prompts (hash, prompt) VALUES (?, ?)",
  453. (prompt_hash, system_prompt),
  454. )
  455. return prompt_hash
  456. @staticmethod
  457. def _delete_unreferenced_system_prompts(conn) -> None:
  458. conn.execute(
  459. "DELETE FROM system_prompts WHERE NOT EXISTS ("
  460. "SELECT 1 FROM sessions WHERE sessions.system_prompt_hash = system_prompts.hash) AND NOT EXISTS ("
  461. "SELECT 1 FROM sessions WHERE sessions.tool_names = system_prompts.hash)"
  462. )
  463. @staticmethod
  464. def _session_row_dict(row: sqlite3.Row) -> dict[str, Any]:
  465. data = dict(row)
  466. for column in ("system_prompt", "tool_names"):
  467. if f"_{column}_resolved" in data:
  468. resolved = data.pop(f"_{column}_resolved")
  469. if column in data:
  470. data[column] = resolved
  471. # Lift /new vs /branch markers out of model_config so list payloads
  472. # (which strip that heavy field) can still tell a reset sibling from
  473. # a genuine fork. Keep this a local helper: tests sometimes replace
  474. # the SessionDB name with a factory lambda.
  475. if not (data.get("_reset_from") and data.get("_branched_from")):
  476. raw = data.get("model_config")
  477. cfg = None
  478. if isinstance(raw, str) and raw:
  479. try:
  480. cfg = json.loads(raw)
  481. except (TypeError, ValueError):
  482. cfg = None
  483. elif isinstance(raw, dict):
  484. cfg = raw
  485. if isinstance(cfg, dict):
  486. if not data.get("_reset_from"):
  487. value = cfg.get("_reset_from")
  488. if isinstance(value, str) and value.strip():
  489. data["_reset_from"] = value.strip()
  490. if not data.get("_branched_from"):
  491. value = cfg.get("_branched_from")
  492. if isinstance(value, str) and value.strip():
  493. data["_branched_from"] = value.strip()
  494. return data
  495. @staticmethod
  496. def _close_connection_quietly(conn: Optional[sqlite3.Connection]) -> None:
  497. """Close a partially initialized connection without masking its error."""
  498. if conn is None:
  499. return
  500. try:
  501. conn.close()
  502. except Exception:
  503. logger.debug("Could not close a SessionDB connection", exc_info=True)
  504. def _close_conn_logged(self, conn, label: str) -> None:
  505. """Close *conn*; a failing close leaks a tracked fd: logged at WARNING, never swallowed."""
  506. try:
  507. conn.close()
  508. except Exception as exc:
  509. logger.warning("%s close failed for %s: %s", label, self.db_path, exc)
  510. def __init__(self, db_path: Path | None = None, read_only: bool = False):
  511. self.db_path = db_path or _default_db_path()
  512. _ensure_test_isolation(self.db_path) # before any connection/pragma/mkdir
  513. self.read_only = read_only
  514. # Keep only the opening call site, never a frame (which pins caller locals).
  515. self._creation_site = "unknown"
  516. caller = None
  517. try:
  518. caller = sys._getframe(1)
  519. self._creation_site = (
  520. f"{caller.f_globals.get('__name__', '?')}.{caller.f_code.co_name}:{caller.f_lineno}"
  521. )
  522. except Exception:
  523. pass # Diagnostic metadata must not prevent opening the database.
  524. finally:
  525. del caller
  526. self._lock = threading.Lock()
  527. # Read-path split (WAL only): reads borrow from a BOUNDED read-only pool so they
  528. # never queue behind writer flushes on self._lock (see _read_ctx); unbounded
  529. # per-thread connections pinned fds for the process lifetime and hit EMFILE.
  530. self._read_pool: queue.LifoQueue[sqlite3.Connection] = queue.LifoQueue(maxsize=_READ_POOL_MAX)
  531. # Permits bound PEAK descriptors (the pool bounds only the idle set), shared per
  532. # DATABASE PATH; acquired non-blocking so a permitless reader degrades to the writer lock.
  533. # One permit per live read connection, held from before the open in _get_read_conn() until after the
  534. # close in _close_read_conn(). See _READ_POOL_MAX. Acquired non-blocking on purpose: a reader that
  535. # cannot get a permit must degrade to the writer lock, not queue here — blocking would convert fd
  536. # exhaustion into a stall, which is the same outage with a different stack trace. Permits are shared
  537. # per DATABASE PATH, not per instance: the descriptors they ration belong to the file, and one
  538. # process holds several SessionDB objects on the same state.db (#98573). See _PathReadBudget.
  539. self._read_budget = _read_budget_for(self.db_path)
  540. self._read_permits = self._read_budget.permits
  541. self._read_conns_lock = threading.Lock()
  542. # Set when close() begins; an in-flight reader then closes its own connection
  543. # instead of re-populating a pool nobody will drain again.
  544. self._read_conns_closed = False
  545. # Read-open failure backoff is a TIMESTAMP, not a sticky bool: the likeliest trigger
  546. # is transient EMFILE, and a permanent flag would demote every reader forever.
  547. self._read_open_failed_at = 0.0
  548. self._wal_active, self._write_count = False, 0
  549. # File identity of the opened state.db, compared on every write so an out-of-band
  550. # replace cannot limp through in-place surgery (inode: mv/new-file; application_id: cp).
  551. self._db_file_identity: Optional[tuple] = None
  552. self._db_file_application_id: int = 0
  553. self._db_sidecar_identity: dict[str, tuple] = {}
  554. self._db_replaced = self._db_wal_generation_lost = False
  555. # Durable capture of a lost WAL generation (see _capture_retired_generation): once per handle.
  556. self._retired_generation_capture: Optional[Path] = None
  557. self._retired_capture_lock = threading.Lock()
  558. self._retire_connection: Optional[Callable[[Any], None]] = None
  559. self._connection_pinned = False # one unmatched C reference taken at most once per handle
  560. self._wal_lock_guard: dict = {} # hermes_state_lockguard.hold() record, see _open_writer
  561. self._db_corrupt, self._db_corrupt_reason = False, "" # sticky quarantine (StateDbCorruptError)
  562. self._fts_usermerge_floor_applied = False # one-shot usermerge-floor write guard
  563. self._fts_enabled = self._fts_stale = self._trigram_available = False
  564. # _fts_cjk_loaded: tokenizer on the writer connection; _fts_cjk_available: messages_fts_cjk
  565. # is queryable AND not marked stale.
  566. self._fts_cjk_loaded = self._fts_cjk_available = self._fts_unavailable_warned = False
  567. self._conn = None
  568. # Async token accounting; distinct from self._lock so enqueue/flush never contends with writes.
  569. self._token_queue: deque = deque()
  570. self._token_queue_cond = threading.Condition(threading.Lock())
  571. self._token_writer_thread: Optional[threading.Thread] = None
  572. self._token_writer_stop = self._token_writer_busy = False
  573. self._token_atexit_hook: Optional[Callable[[], None]] = None
  574. # Opened via hermes_state_registry.acquire(): close() releases a refcount instead.
  575. # Set True when this instance is opened via hermes_state_registry.acquire(). Makes close() a no-op so the
  576. # registry (not individual callers) controls the connection lifecycle (#90837).
  577. self._shared_registry_owned = False
  578. initialization_complete = False
  579. try:
  580. if read_only:
  581. self._open_read_only()
  582. else:
  583. # Where SQLite's close-time checkpoint cannot be switched off, a lost-generation handle
  584. # is retired unclosed (see close()). Resolve that capability before opening a writer:
  585. # late cleanup must not import ctypes or look up it in a cleared module dictionary.
  586. if not _close_time_checkpoint_configurable():
  587. self._retire_connection = _prepare_connection_retirement()
  588. self._open_writer()
  589. self._record_db_file_identity()
  590. initialization_complete = True
  591. except Exception as exc:
  592. # Surface WHY via /resume and friends; callers keep their ``_session_db = None`` path.
  593. _set_last_init_error(f"{type(exc).__name__}: {exc}")
  594. raise
  595. finally:
  596. if not initialization_complete:
  597. conn, self._conn = self._conn, None
  598. self._close_connection_quietly(conn)
  599. else:
  600. # Only a successfully opened handle owns a writer connection. Failed
  601. # construction must not leave a diagnostic member behind.
  602. self._read_budget.register(self)
  603. # Test-isolation runs only (gated inside the helper): register
  604. # for the suite-level leak sweep in tests/conftest.py.
  605. _register_test_instance(self)
  606. def _open_writer(self) -> None:
  607. """Writable open: preflight, zero-byte quarantine, connect + schema (one in-place repair of a
  608. malformed sqlite_master), generation stamp."""
  609. # Never materialize a deleted/archived named profile's home: a multiplexer or Desktop backend
  610. # still holding the profile's route would otherwise re-scaffold it on the next turn (#94590).
  611. mkdir_under_hermes_home(self.db_path.parent)
  612. try:
  613. # Serialize zero-byte check, quarantine, connect and schema commit so concurrent
  614. # openers don't race the absent-path -> schema-commit window.
  615. if not self.db_path.exists() or has_invalid_sqlite_header_preopen(self.db_path):
  616. # The directory must be writable before the lock file can be created in it; the
  617. # directory never vanishes mid-quarantine, so checking it unlocked is race-free.
  618. preflight_db_writability(self.db_path, db_label="state.db", directory_only=True)
  619. with quarantine_cross_process_lock(self.db_path) as lock_acquired:
  620. if not lock_acquired:
  621. logger.warning(
  622. "startup quarantine lock for %s not acquired within 5s; proceeding",
  623. self.db_path,
  624. )
  625. # Preflight under the lock: unlocked, it can list state.db just before a
  626. # concurrent opener renames it aside, and os.access on the vanished path
  627. # then reports a writable DB as "read-only for this user".
  628. preflight_db_writability(self.db_path, db_label="state.db")
  629. self._handle_quarantine_if_invalid(already_locked=lock_acquired)
  630. self._connect_and_init_with_lock_patience()
  631. else:
  632. # Read-only file/sidecar preflight BEFORE the first connection: an actionable message
  633. # instead of an opaque "attempt to write a readonly database" from inside _init_schema.
  634. preflight_db_writability(self.db_path, db_label="state.db")
  635. self._handle_quarantine_if_invalid(already_locked=False)
  636. self._connect_and_init_with_lock_patience()
  637. except sqlite3.DatabaseError as exc:
  638. # A malformed schema fails on the very first statement (before _init_schema), so the
  639. # FTS-rebuild layer never sees it: repair sqlite_master in place (backup first), reopen once.
  640. if not is_malformed_schema_error(exc) or not _claim_repair_attempt(self.db_path):
  641. raise
  642. logger.error(
  643. "state.db schema is malformed (%s) — attempting automatic "
  644. "repair (a backup copy is made first).", exc,
  645. )
  646. self._close_connection_quietly(self._conn)
  647. if not repair_state_db_schema(self.db_path).get("repaired"):
  648. raise
  649. self._connect_and_init_with_lock_patience()
  650. # FTS optimization is OPT-IN (`hermes db optimize`); no background worker races session lifecycle.
  651. self._ensure_db_file_generation()
  652. if self._wal_active:
  653. # OFD copies of the two POSIX locks that keep a sibling's close from unlinking this WAL
  654. # generation: any in-process open()/close() of state.db or -shm cancels SQLite's own
  655. # (howtocorrupt §2.2); these survive it. Lifted in close().
  656. self._wal_lock_guard = _lockguard.hold(self.db_path)
  657. def _open_read_only(self) -> None:
  658. """Read-only attach for cross-profile aggregation: no schema init, NO write
  659. lock (sidebar polling never contends with that profile's backend); the DB
  660. must exist. FTS flags are probed with SELECTs only, and the connection is
  661. closed on ANY probe failure (malformed schema raises DatabaseError) so a
  662. leaked tracked connection cannot block the forensic backup the writable heal takes next."""
  663. for attempt in range(_READ_ONLY_IOERR_RETRY_ATTEMPTS + 1):
  664. try:
  665. self._conn = conn = self._connect_read_only(timeout=_READ_BUSY_TIMEOUT_S)
  666. try:
  667. apply_database_pragmas(conn, db_label="state.db")
  668. cursor = conn.cursor()
  669. self._fts_enabled = self._fts_table_probe(cursor, "messages_fts") is True
  670. if self._fts_enabled:
  671. self._trigram_available = (
  672. self._fts_table_probe(cursor, "messages_fts_trigram") is True
  673. )
  674. except BaseException:
  675. self._conn = None
  676. self._close_connection_quietly(conn)
  677. raise
  678. return
  679. except sqlite3.OperationalError as ioerr:
  680. # In-flight WAL checkpoint/reset/frame-flush on the writer side can surface
  681. # SQLITE_IOERR to a mode=ro reader (it can't do the -shm recovery the read
  682. # needs). Closes in milliseconds: retry a bounded number of times before
  683. # classifying the store as failed (#100436; see _READ_ONLY_IOERR_RETRY_ATTEMPTS).
  684. # A lock is NOT retried here: the connection already waited _READ_BUSY_TIMEOUT_S,
  685. # and a retry would multiply that wait on blocking callers (TUI, `hermes status`).
  686. transient = _DISK_IO_ERROR_MARKER in str(ioerr).lower()
  687. if attempt >= _READ_ONLY_IOERR_RETRY_ATTEMPTS or not transient:
  688. raise
  689. time.sleep(_READ_ONLY_IOERR_RETRY_BACKOFF_S)
  690. def _connect_read_only(self, timeout: float) -> sqlite3.Connection:
  691. """``mode=ro`` tracked connection with Row factory. check_same_thread=False: pooled connections
  692. are borrowed by whichever thread reads next; exclusive ownership is enforced by pool checkout."""
  693. conn = _connect_tracked_db(
  694. read_only_db_uri(self.db_path), tracking_path=self.db_path, uri=True,
  695. check_same_thread=False, timeout=timeout, isolation_level=None,
  696. )
  697. conn.row_factory = sqlite3.Row
  698. return conn
  699. def _handle_quarantine_if_invalid(self, already_locked: bool = False) -> None:
  700. """Quarantine a zero-byte/headerless state.db so a fresh one can open; if quarantine failed,
  701. raise the clear message instead of opening the zeroed file."""
  702. if not (self.db_path.exists() and has_invalid_sqlite_header_preopen(self.db_path)):
  703. return
  704. try:
  705. zsize = self.db_path.stat().st_size
  706. except OSError:
  707. zsize = -1
  708. qpath = quarantine_invalid_state_db(self.db_path, already_locked=already_locked)
  709. where = f"moved aside to {qpath}" if qpath else "left in place (it could not be moved aside)"
  710. msg = (
  711. f"state.db was empty or damaged ({zsize} bytes) and has been {where}; Hermes started with a "
  712. "fresh, empty session database. To bring old sessions back, run "
  713. f"`hermes sessions recover --source {qpath or self.db_path} --inspect-only`, or restore a "
  714. "snapshot with `/snapshot list` then `/snapshot restore <id>` (terminal `hermes` chat only)."
  715. )
  716. logger.error(msg)
  717. _set_last_init_error(msg)
  718. if qpath is None and self.db_path.exists() and has_invalid_sqlite_header_preopen(self.db_path):
  719. raise sqlite3.DatabaseError(msg)
  720. def _open_writer_conn(self) -> sqlite3.Connection:
  721. """Connect + WAL/pragma/tokenizer setup for a writer connection (no schema init). Short timeout:
  722. jittered application-level retry handles contention, not SQLite's busy handler;
  723. isolation_level=None: explicit BEGIN IMMEDIATE."""
  724. conn = _connect_tracked_db(
  725. str(self.db_path), check_same_thread=False, timeout=1.0, isolation_level=None,
  726. )
  727. try:
  728. conn.row_factory = sqlite3.Row
  729. mode = apply_wal_with_fallback(conn, db_label="state.db")
  730. # "wal" is also the *assumed* mode when the on-disk probe was blocked by a concurrent opener
  731. # (#86515): the lock-free mode=ro read pool needs a confirmed WAL header, so confirm it here.
  732. # Unknown -> reads queue on the writer lock (slow but correct) instead of racing SQLITE_BUSY
  733. # on a file that may really be in rollback-journal mode.
  734. self._wal_active = mode == "wal" and _on_disk_journal_mode(conn) == "wal"
  735. # Existing WAL/SHM files may predate the main-file hardening;
  736. # normalize any sidecars that became visible during WAL setup.
  737. _secure_state_db_files(self.db_path)
  738. apply_database_pragmas(conn, db_label="state.db")
  739. conn.execute("PRAGMA foreign_keys=ON")
  740. self._fts_cjk_loaded = load_fts5_cjk_extension(conn)
  741. except BaseException:
  742. self._close_connection_quietly(conn)
  743. raise
  744. return conn
  745. def _connect_and_init(self) -> None:
  746. # Refuse before sqlite3.connect (under the startup lock) so we cannot mint
  747. # a replacement WAL while a live writer still holds a deleted sidecar inode.
  748. refuse_deleted_wal_generation(self.db_path)
  749. # Create/tighten the main database before sqlite3.connect() so a
  750. # permissive process umask can never expose a fresh profile store.
  751. _secure_state_db_files(self.db_path, create_main=True)
  752. self._conn = self._open_writer_conn()
  753. self._init_schema()
  754. def _connect_and_init_with_lock_patience(self) -> None:
  755. """Open + init, waiting out a sibling's write lock with jittered patience:
  756. _init_schema's DDL runs on a 1s-timeout connection, so a sibling's VACUUM
  757. or checkpoint used to fail the ENTIRE open and callers disabled
  758. persistence for the whole run. Non-lock errors propagate immediately."""
  759. # Lock contention during open: _init_schema's DDL/reconcile statements run on a 1s-timeout
  760. # connection with no retry, so a sibling process holding the write lock (VACUUM, TRUNCATE checkpoint
  761. # at close, a long FTS pass from an older still-running install) used to fail the ENTIRE open —
  762. # callers then disable persistence for the whole run ("Failed to initialize SessionDB ... database
  763. # is locked", #74478). The store is healthy; wait it out with the same jittered patience the write
  764. # path uses.
  765. deadline = time.monotonic() + self._WRITE_PATIENCE_S
  766. while True:
  767. try:
  768. self._connect_and_init()
  769. return
  770. except sqlite3.OperationalError as exc:
  771. if not is_sqlite_lock_error(exc):
  772. raise
  773. self._close_connection_quietly(self._conn)
  774. now = time.monotonic()
  775. if now >= deadline:
  776. log_write_lock_holders(self.db_path, self._WRITE_PATIENCE_S)
  777. raise
  778. jitter = random.uniform(self._WRITE_RETRY_SLOW_MIN_S, self._WRITE_RETRY_SLOW_MAX_S)
  779. time.sleep(min(jitter, max(deadline - now, 0.001)))
  780. # ── Read-path split ──
  781. def _get_read_conn(self) -> Optional[sqlite3.Connection]:
  782. """Open a fresh read-only connection, or None when unavailable (callers
  783. return it to self._read_pool). WAL only: WAL readers never block on the
  784. writer, so reads skip self._lock; under DELETE journal mode (NFS fallback)
  785. readers hit SQLITE_BUSY storms, so the legacy locked path stays. Autocommit
  786. reads see everything committed so far (read-your-writes for flush-then-search)."""
  787. if not self._wal_active or self.read_only:
  788. return None
  789. with self._read_conns_lock:
  790. failed_at = self._read_open_failed_at
  791. backing_off = failed_at and time.monotonic() - failed_at < _READ_OPEN_RETRY_SECONDS
  792. if self._read_conns_closed or backing_off:
  793. return None
  794. # Permit BEFORE the open: openers race for permits, not descriptors.
  795. if not self._read_budget.acquire(self):
  796. logger.debug(
  797. "read pool at capacity (%d) for %s; serving this read from the "
  798. "locked writer connection", _READ_POOL_MAX, self.db_path,
  799. )
  800. return None
  801. conn = None # bound before the try so the handlers can close a half-open one
  802. try:
  803. conn = self._connect_read_only(timeout=_READ_BUSY_TIMEOUT_S)
  804. apply_database_pragmas(conn, db_label="state.db")
  805. if self._fts_cjk_loaded: # registers in the connection, not the file: ro is fine
  806. load_fts5_cjk_extension(conn)
  807. except BaseException as exc:
  808. # A half-open connection (open ok, extension load failed) is a live tracked descriptor,
  809. # the leak shape this pool exists to fix; a stranded permit would shrink the read
  810. # path by one slot forever. (Not _close_read_conn: callers release their own permit.)
  811. if conn is not None:
  812. self._close_conn_logged(conn, "partially-opened read conn")
  813. self._read_budget.release()
  814. if not isinstance(exc, sqlite3.Error):
  815. raise
  816. with self._read_conns_lock:
  817. self._read_open_failed_at = time.monotonic()
  818. logger.debug("read-only connection open failed for %s", self.db_path, exc_info=True)
  819. return None
  820. return conn
  821. def _evict_one_idle_read_conn(self) -> bool:
  822. """Close one idle pooled connection (a peer on the same file wants its permit); never a live one."""
  823. try:
  824. conn = self._read_pool.get_nowait()
  825. except queue.Empty:
  826. return False
  827. self._close_read_conn(conn)
  828. return True
  829. def _close_read_conn(self, conn) -> None:
  830. """Close a pooled read connection and release its permit even when the close fails (withholding
  831. it would narrow the read path forever). Over-releasing the BoundedSemaphore raises ValueError."""
  832. try:
  833. self._close_conn_logged(conn, "read-conn")
  834. finally:
  835. self._read_budget.release()
  836. def _checkout_read_conn(self) -> Optional[sqlite3.Connection]:
  837. """Borrow a read connection, opening on a miss; None when the read path is unavailable.
  838. A pool hit costs no permit (the connection already holds one)."""
  839. if not self._wal_active or self.read_only:
  840. return None
  841. try:
  842. return self._read_pool.get_nowait()
  843. except queue.Empty:
  844. return self._get_read_conn()
  845. @contextmanager
  846. def _read_ctx(self, *, reopen: bool = True) -> Iterator[sqlite3.Connection]:
  847. """Yield a connection for read-only statements: a pooled read-only
  848. connection with NO lock under WAL; otherwise (non-WAL, open failure,
  849. ceiling reached) the writer connection under self._lock — deliberate
  850. degradation: slower beats EMFILE, which the supervisor cannot see.
  851. ``reopen=False``: a read after ``close()`` raises instead of reopening the
  852. writer (an optional reader must not hold a home that is being deleted)."""
  853. conn = self._checkout_read_conn()
  854. if conn is not None:
  855. try:
  856. yield conn
  857. finally:
  858. returned = False
  859. with self._read_conns_lock:
  860. if not self._read_conns_closed:
  861. try:
  862. self._read_pool.put_nowait(conn)
  863. returned = True
  864. except queue.Full:
  865. pass
  866. if not returned:
  867. # close() drained the pool (or queue.Full: unreachable while
  868. # permits == maxsize, load-bearing if they drift): surplus.
  869. self._close_read_conn(conn)
  870. return
  871. with self._lock:
  872. if self._conn is None: # close() raced a still-unwinding reader
  873. if not reopen:
  874. raise sqlite3.ProgrammingError(f"SessionDB for {self.db_path} is closed")
  875. self._reopen_after_close_locked(context="read")
  876. conn = cast(sqlite3.Connection, self._conn)
  877. if self._wal_active or self.read_only: # WAL: no SHARED-lock wait; ro: opened with the read budget
  878. yield conn
  879. return
  880. # DELETE-mode writer connection: wait out a sibling's commit like a pooled reader would.
  881. previous_ms = conn.execute("PRAGMA busy_timeout").fetchone()[0]
  882. conn.execute(f"PRAGMA busy_timeout={int(_READ_BUSY_TIMEOUT_S * 1000)}")
  883. try:
  884. yield conn
  885. finally:
  886. with suppress(sqlite3.Error):
  887. conn.execute(f"PRAGMA busy_timeout={int(previous_ms)}")
  888. def _reopen_after_close_locked(self, context: str = "write") -> None:
  889. """Reopen the writer after ``close()`` raced a live caller (a teardown owner
  890. set ``_conn = None`` while a worker still had a transcript flush to land).
  891. Loud (WARNING) and bounded (only after an explicit close()). Caller holds
  892. ``self._lock``. No _init_schema: no DDL races with siblings during teardown."""
  893. if self.read_only:
  894. raise sqlite3.ProgrammingError(
  895. f"SessionDB for {self.db_path} was closed (read-only handle); "
  896. f"cannot serve a {context} after close()"
  897. )
  898. # A reopen resolves the PATH again: a replaced file would be written through stale WAL/shm
  899. # assumptions; a quarantined handle must never hand a fresh connection to a damaged file.
  900. if self._db_corrupt and not (self._db_replaced or self._db_file_was_replaced()):
  901. raise self._corrupt_error(
  902. f"state.db connection for {self.db_path} is quarantined after "
  903. f"structural corruption; refusing to reopen for a {context} "
  904. "after close(). "
  905. )
  906. self._halt_if_db_generation_changed()
  907. logger.warning(
  908. "state.db connection for %s was closed while a %s was still in "
  909. "flight — reopening (teardown/worker race, #94736)", self.db_path, context,
  910. )
  911. try:
  912. self._conn = self._open_writer_conn()
  913. except Exception as exc:
  914. raise sqlite3.OperationalError(
  915. f"state.db connection was closed while a {context} was still "
  916. f"in flight (a session-teardown path called close() before "
  917. f"this worker finished — #94736) and the automatic reopen failed: {exc}"
  918. ) from exc
  919. if self._wal_active: # a reopened writer is a live generation holder like the first open
  920. self._wal_lock_guard = _lockguard.hold(self.db_path)
  921. def _execute_write(
  922. self, fn: Callable[[sqlite3.Connection], T], patience_s: Optional[float] = None,
  923. ) -> T:
  924. """Run *fn(conn)* inside BEGIN IMMEDIATE with jittered lock retry; commit
  925. is handled here (callers must not commit). Returns *fn*'s result.
  926. BEGIN IMMEDIATE takes the WAL write lock up front so contention surfaces
  927. immediately; on locked/busy the Python lock is released, a jitter slept,
  928. and the WHOLE callback retried — *fn* must stay idempotent under retry."""
  929. if patience_s is None:
  930. patience_s = self._WRITE_PATIENCE_S
  931. deadline = time.monotonic() + patience_s
  932. compression_deadline: Optional[float] = None # set on the first compression-busy collision
  933. # One retry for SQLITE_IOERR raised by BEGIN IMMEDIATE itself (callback not run: nothing
  934. # replayed). Once fn has started, an IOERR leaves settlement unknown and must propagate.
  935. # The callback has not run at that point, so there is no durable effect to replay and the retry is
  936. # exactly-once safe (#99502's contract). Once the callback starts, an IOERR leaves the write's
  937. # settlement unknown and must propagate — this helper owns non-idempotent transcript/counter
  938. # mutations, not just idempotent UPSERTs.
  939. ioerr_begin_retried = False
  940. while True:
  941. self._raise_if_db_corrupt(storage=True)
  942. # NOTE: the replaced/generation live probe runs INSIDE the lock below,
  943. # not here. close() mutates _conn and _db_sidecar_identity under that
  944. # same lock, ending the WAL generation (SQLite unlinks the -wal/-shm
  945. # sidecars on a clean close). A lock-free probe that races close() can
  946. # observe the mid-teardown state — sidecars already unlinked while
  947. # _db_sidecar_identity is not yet cleared — and misclassify this
  948. # process's OWN clean close as an externally deleted generation,
  949. # raising a sticky DeletedWalGenerationError that permanently refuses
  950. # later writes (#105567). Inside the lock the probe only ever sees the
  951. # stable post-close state (identity cleared → adopt / reopen path).
  952. fn_started = False
  953. try:
  954. with self._lock:
  955. self._raise_if_db_replaced()
  956. if self._conn is None: # close() raced this writer
  957. self._reopen_after_close_locked(context="write")
  958. self._conn.execute("BEGIN IMMEDIATE")
  959. try:
  960. fn_started = True
  961. result = fn(self._conn)
  962. self._conn.commit()
  963. except BaseException:
  964. try:
  965. self._conn.rollback()
  966. except Exception:
  967. pass
  968. raise
  969. # Success — periodic best-effort checkpoint + FTS merge.
  970. self._write_count += 1
  971. if self._write_count % self._CHECKPOINT_EVERY_N_WRITES == 0:
  972. self._try_wal_checkpoint()
  973. if self._write_count % self._FTS_MERGE_EVERY_N_WRITES == 0:
  974. self._try_incremental_merge_fts()
  975. return result
  976. except SessionCompressionInProgressError:
  977. # Transient (see _COMPRESSION_BUSY_WAIT_S): a steer landing mid-compression must not abort.
  978. # A live foreign compression lock is transient: the compressor publishes in a couple of
  979. # seconds. Without any wait, a steer that lands mid-compression aborts the user's turn as
  980. # session_persistence_failed and sends the operator hunting disk space that was never the
  981. # problem (#75083). The budget is _COMPRESSION_BUSY_WAIT_S, not the write-lock patience: the
  982. # lease is a correctness boundary, so a writer still locked out after a short wait must be
  983. # refused rather than left to land a stale turn once a long-running or wedged compression
  984. # finally lets go.
  985. if compression_deadline is None:
  986. compression_deadline = min(time.monotonic() + self._COMPRESSION_BUSY_WAIT_S, deadline)
  987. if self._sleep_before_write_retry(
  988. compression_deadline, self._COMPRESSION_BUSY_WAIT_S
  989. ):
  990. continue
  991. raise
  992. except sqlite3.Error as exc:
  993. # 'no more rows' is a transient engine error on contended WAL appends (some builds
  994. # raise it as InterfaceError, a sibling of DatabaseError): retry like locked/busy.
  995. if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s):
  996. continue
  997. err_msg = str(exc).lower()
  998. if isinstance(exc, sqlite3.OperationalError):
  999. if is_sqlite_lock_error(exc):
  1000. if self._sleep_before_write_retry(deadline, patience_s):
  1001. continue
  1002. # Say what actually happened, not disk/permission damage. The holder goes to
  1003. # the log, not the message: classify_persistence_error() buckets by phrase and
  1004. # a holder's argv (a worktree named fix-corrupt-db) would flip the bucket.
  1005. log_write_lock_holders(self.db_path, patience_s)
  1006. raise sqlite3.OperationalError(
  1007. f"database is locked (another Hermes process held the "
  1008. f"state.db write lock for over {patience_s:.0f}s — "
  1009. "likely a long maintenance operation such as VACUUM, "
  1010. "a large WAL checkpoint, or an older pre-update "
  1011. "process; the database itself is healthy)"
  1012. ) from exc
  1013. if (
  1014. _DISK_IO_ERROR_MARKER in err_msg and not fn_started and not ioerr_begin_retried
  1015. and self._sleep_before_write_retry(deadline, patience_s)
  1016. ):
  1017. # Retry on the SAME connection: close()+reopen would cancel this process's
  1018. # POSIX locks for every sibling (howtocorrupt §2.2).
  1019. ioerr_begin_retried = True
  1020. continue
  1021. raise # non-lock error, callback already ran, or patience exhausted
  1022. if isinstance(exc, sqlite3.DatabaseError):
  1023. # An out-of-band replace surfaces as this same corruption class; in-file repair
  1024. # on a NEW generation amplifies the damage.
  1025. if (
  1026. "not a database" in err_msg or is_malformed_db_error(exc)
  1027. or self._is_fts_write_corruption_error(exc)
  1028. ):
  1029. with self._lock:
  1030. self._raise_if_db_replaced()
  1031. # Corrupt FTS shadow tables fail every write via the sync triggers while canonical
  1032. # rows are intact: detach the derived indexes atomically and retry (never rebuild here).
  1033. if self._enter_fts_fail_open(exc, deadline=deadline, patience_s=patience_s):
  1034. continue
  1035. # What survives both checks is structural damage: quarantine.
  1036. if self._is_structural_corruption_error(exc):
  1037. self._halt_db_corrupt(exc)
  1038. raise
  1039. def _write_sql(
  1040. self, sql: str, params: Any = (), *, many: bool = False, patience_s: Optional[float] = None,
  1041. ) -> None:
  1042. """Run one INSERT/UPDATE/DELETE through ``_execute_write``."""
  1043. def _do(conn):
  1044. (conn.executemany if many else conn.execute)(sql, params)
  1045. self._execute_write(_do, patience_s=patience_s)
  1046. def _write_rowcount(self, sql: str, params: Any = (), *, patience_s: Optional[float] = None) -> int:
  1047. """Run one UPDATE/DELETE through ``_execute_write``; return rows changed
  1048. (``SELECT changes()`` when the driver reports None / negative)."""
  1049. def _do(conn):
  1050. rowcount = conn.execute(sql, params).rowcount
  1051. if rowcount is None or rowcount < 0:
  1052. rowcount = conn.execute("SELECT changes()").fetchone()[0]
  1053. return rowcount
  1054. return self._execute_write(_do, patience_s=patience_s)
  1055. def _read_one(self, sql: str, params: Any = ()) -> Optional[sqlite3.Row]:
  1056. """``fetchone()`` of one read-only statement via ``_read_ctx``."""
  1057. return self._read_retrying_ioerr(lambda conn: conn.execute(sql, params).fetchone())
  1058. def _read_all(self, sql: str, params: Any = ()) -> list[sqlite3.Row]:
  1059. """``fetchall()`` of one read-only statement via ``_read_ctx``."""
  1060. return self._read_retrying_ioerr(lambda conn: conn.execute(sql, params).fetchall())
  1061. def _read_retrying_ioerr(self, fn: Callable[[sqlite3.Connection], T]) -> T:
  1062. """Run an idempotent SELECT through ``_read_ctx``, retrying a transient SQLITE_IOERR.
  1063. A warm ``mode=ro`` pooled reader can hit the same millisecond-wide WAL transition window as a
  1064. read-only OPEN (#100436) when its statement executes or steps: a sibling process's checkpoint /
  1065. WAL reset / frame flush surfaces ``disk I/O error`` because a read-only connection cannot rewrite
  1066. the -shm index (#100871, WSL2 ext4-on-vhdx, multi-process). The window closes on its own, so
  1067. the statement is replayed on the SAME connection within the read-only IOERR budget -- never
  1068. closed and reopened (close() cancels this process's POSIX locks for every sibling connection),
  1069. never quarantined (busy is not broken). A persistent IOERR exhausts the budget and propagates."""
  1070. for attempt in range(_READ_ONLY_IOERR_RETRY_ATTEMPTS + 1):
  1071. try:
  1072. with self._read_ctx() as conn:
  1073. return fn(conn)
  1074. except sqlite3.OperationalError as exc:
  1075. if attempt >= _READ_ONLY_IOERR_RETRY_ATTEMPTS or _DISK_IO_ERROR_MARKER not in str(exc).lower():
  1076. note_storage_error(self.db_path, exc)
  1077. raise
  1078. time.sleep(_READ_ONLY_IOERR_RETRY_BACKOFF_S)
  1079. except sqlite3.DatabaseError as exc:
  1080. # A reader is often the only observer (the sidebar poll on a store nobody is
  1081. # writing to): publish structural damage so the list is not read as empty.
  1082. note_storage_error(self.db_path, exc)
  1083. raise
  1084. def _ensure_db_file_generation(self) -> None:
  1085. """Mint a once-per-file generation stamp (state_meta + application_id). First opener wins (INSERT
  1086. OR IGNORE); application_id is written only while 0 so racers converge. PASSIVE checkpoint only.
  1087. See #45383.
  1088. """
  1089. if self.read_only or self._conn is None:
  1090. return
  1091. token = uuid.uuid4().hex
  1092. try:
  1093. with self._lock:
  1094. # Read first: the stamp is minted once per file, and a no-op INSERT OR IGNORE
  1095. # still takes the write lock — under a sibling's transaction it blocked for the
  1096. # busy timeout and the except below then dropped the token entirely. First
  1097. # opener still wins via INSERT OR IGNORE; racers converge on the re-read.
  1098. row = self._conn.execute(
  1099. "SELECT value FROM state_meta WHERE key = ?", (_STATE_DB_GENERATION_KEY,),
  1100. ).fetchone()
  1101. if not (row and row[0]):
  1102. self._conn.execute(
  1103. "INSERT OR IGNORE INTO state_meta (key, value) VALUES (?, ?)",
  1104. (_STATE_DB_GENERATION_KEY, token),
  1105. )
  1106. row = self._conn.execute(
  1107. "SELECT value FROM state_meta WHERE key = ?",
  1108. (_STATE_DB_GENERATION_KEY,),
  1109. ).fetchone()
  1110. if row and row[0]:
  1111. token = str(row[0])
  1112. pragma_row = self._conn.execute("PRAGMA application_id").fetchone()
  1113. current = int(pragma_row[0] or 0) if pragma_row else 0
  1114. if current == 0:
  1115. current = (int(token[:8], 16) & 0x7FFFFFFF) or 1
  1116. self._conn.execute(f"PRAGMA application_id={current}")
  1117. self._db_file_application_id = current
  1118. try:
  1119. self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
  1120. except sqlite3.Error:
  1121. pass
  1122. except sqlite3.Error as exc:
  1123. logger.debug("state.db generation stamp skipped: %s", exc)
  1124. def _record_db_file_identity(self) -> None:
  1125. """Snapshot inode plus the on-disk generation header when present."""
  1126. self._db_file_identity = _stat_db_file_identity(self.db_path)
  1127. self._db_sidecar_identity = _stat_sqlite_sidecar_identity(self.db_path)
  1128. disk_id = _read_sqlite_application_id(self.db_path)
  1129. if disk_id:
  1130. self._db_file_application_id = disk_id
  1131. elif self._conn is not None and not self._db_file_application_id:
  1132. try:
  1133. pragma_row = self._read_one("PRAGMA application_id")
  1134. except sqlite3.Error:
  1135. pragma_row = None
  1136. if pragma_row and pragma_row[0]:
  1137. self._db_file_application_id = int(pragma_row[0])
  1138. def _db_file_was_replaced(self) -> bool:
  1139. """True when the path no longer names the file this instance opened."""
  1140. recorded = self._db_file_identity
  1141. if recorded is not None and _stat_db_file_identity(self.db_path) != recorded:
  1142. return True
  1143. recorded_app = int(self._db_file_application_id or 0)
  1144. if not recorded_app:
  1145. return False
  1146. # Header 0 = WAL not yet checkpointed, not a replace; a real replacement is nonzero.
  1147. disk_app = _read_sqlite_application_id(self.db_path)
  1148. return bool(disk_app and disk_app != recorded_app)
  1149. def _wal_generation_was_lost(self) -> bool:
  1150. """True when the WAL/SHM generation this handle opened is gone. Recorded
  1151. generation: pure stat (no /proc walk on healthy writes). Empty identity
  1152. (WAL appeared after open, or cleared by a clean close()): probe
  1153. /proc/self/fd for deleted sidecars and adopt the current ones once clean."""
  1154. recorded = self._db_sidecar_identity or {}
  1155. base = os.fspath(self.db_path)
  1156. if recorded:
  1157. return any(
  1158. _stat_db_file_identity(Path(base + suffix)) != ident for suffix, ident in recorded.items()
  1159. )
  1160. if not self._wal_active: # no sidecar generation to lose; keep /proc off the hot path
  1161. return False
  1162. if sys.platform.startswith("linux"):
  1163. watched = _watched_sqlite_sidecar_paths(self.db_path)
  1164. try:
  1165. for target, fd_path in _proc_fd_targets(os.getpid()):
  1166. canonical = _state_holders.canonical_sqlite_path(target)
  1167. if (" (deleted)" in target and canonical in watched
  1168. and _fd_is_truly_unlinked(fd_path, watched[canonical])):
  1169. return True
  1170. except OSError:
  1171. return False
  1172. # Probe clean (or unavailable): adopt the current sidecar generation.
  1173. current_identity = _stat_sqlite_sidecar_identity(self.db_path)
  1174. if current_identity:
  1175. self._db_sidecar_identity = current_identity
  1176. return False
  1177. def _halt_if_db_generation_changed(self) -> None:
  1178. """Stop writes (logging once) when the file was replaced or its WAL/SHM generation
  1179. is gone: never run in-file repair on a new generation, never keep committing on a
  1180. split WAL. Both flags are sticky."""
  1181. # A reopen resolves the PATH again — if the file at that path is no longer the one this instance
  1182. # originally opened (out-of-band restore/cp/mv), reconnecting would write into the new generation
  1183. # through stale WAL/shm assumptions (#89332). Refuse instead.
  1184. if self._db_replaced or self._db_file_was_replaced():
  1185. self._db_replaced = True
  1186. self._disable_close_time_checkpoint()
  1187. logger.error(_STATE_DB_REPLACED_MSG)
  1188. raise StateDbReplacedError(_STATE_DB_REPLACED_MSG)
  1189. if self._db_wal_generation_lost or self._wal_generation_was_lost():
  1190. self._db_wal_generation_lost = True
  1191. self._disable_close_time_checkpoint()
  1192. try:
  1193. self._capture_retired_generation("halt")
  1194. except RetiredGenerationCaptureError as exc:
  1195. logger.error(
  1196. "Could not capture the retired WAL generation of %s at halt: %s. close() retries "
  1197. "the capture and refuses to settle without it.", self.db_path, exc,
  1198. )
  1199. logger.error(_DELETED_WAL_GENERATION_MSG)
  1200. raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
  1201. def _capture_retired_generation(self, trigger: str) -> Path:
  1202. """Durably capture the lost WAL generation this handle still holds open, once per handle.
  1203. The quarantine keeps the retired frames from being checkpointed under wrong page numbers,
  1204. but they live only in an unlinked inode that dies with this process's last descriptor, and
  1205. the canonical DeletedWalGenerationError remediation is to stop the writers. Capturing at the
  1206. first halt (or at close(), whichever sees the loss first) makes "preserve" outlive the
  1207. process. Raises RetiredGenerationCaptureError; nothing is mutated on failure."""
  1208. with self._retired_capture_lock:
  1209. if self._retired_generation_capture is not None:
  1210. return self._retired_generation_capture
  1211. artifact = capture_retired_wal_generation(
  1212. self.db_path, sidecar_identity=dict(self._db_sidecar_identity or {}), trigger=trigger,
  1213. )
  1214. self._retired_generation_capture = artifact
  1215. logger.warning(
  1216. "Captured the retired WAL generation of %s at %s to %s; read its manifest.json before deciding "
  1217. "whether those frames belong on top of the file now at the path.", self.db_path, trigger, artifact,
  1218. )
  1219. return artifact
  1220. def _raise_if_db_replaced(self) -> None:
  1221. """Sticky-flag fast path (no log spam on every write), then the live probe."""
  1222. if self._db_replaced:
  1223. raise StateDbReplacedError(_STATE_DB_REPLACED_MSG)
  1224. if self._db_wal_generation_lost:
  1225. raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
  1226. self._halt_if_db_generation_changed()
  1227. @classmethod
  1228. def _is_structural_corruption_error(cls, exc: BaseException) -> bool:
  1229. """Bare SQLITE_CORRUPT/NOTADB with no FTS provenance: canonical B-tree/schema/freelist damage,
  1230. never repairable from the live write path."""
  1231. return (
  1232. isinstance(exc, sqlite3.DatabaseError)
  1233. and not isinstance(exc, StateDbCorruptError)
  1234. and not cls._is_fts_write_corruption_error(exc)
  1235. and classify_persistence_error(exc) == "corrupt"
  1236. )
  1237. def _corrupt_error(self, prefix: str = "") -> StateDbCorruptError:
  1238. """Build the quarantine error for this handle (message assembled once)."""
  1239. return StateDbCorruptError(f"{prefix}{_STATE_DB_CORRUPT_MSG} (cause: {self._db_corrupt_reason})")
  1240. def _halt_db_corrupt(self, exc: BaseException) -> None:
  1241. """Quarantine this handle and raise; never run in-file repair here."""
  1242. self._db_corrupt = True
  1243. self._db_corrupt_reason = str(exc)
  1244. # Publish the profile-level state first: readiness, /api/status and the session list
  1245. # endpoints read it, and every other handle in this process refuses writes on it.
  1246. mark_storage_corrupt(self.db_path, exc)
  1247. self._disable_close_time_checkpoint()
  1248. logger.error(
  1249. "state.db %s reported structural corruption outside the FTS "
  1250. "indexes (%s); quarantining this handle: no further writes, no "
  1251. "automatic reopen, no explicit WAL checkpoint at close. Stop the "
  1252. "gateway and run `hermes sessions recover --source %s --inspect-only`.", self.db_path, exc,
  1253. self.db_path,
  1254. )
  1255. err = self._corrupt_error()
  1256. for attr in ("sqlite_errorcode", "sqlite_errorname"):
  1257. if getattr(exc, attr, None) is not None:
  1258. setattr(err, attr, getattr(exc, attr))
  1259. raise err from exc
  1260. def _disable_close_time_checkpoint(self) -> bool:
  1261. """Best-effort SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE (Python 3.12+): sqlite3's
  1262. close() otherwise runs the internal last-connection checkpoint that wrote
  1263. the incident's pages under wrong page numbers (see StateDbCorruptError and
  1264. the generation-loss halts).
  1265. <3.12 has no setconfig, so a lost-generation handle is retired unclosed
  1266. instead (see close()): closing its last descriptor could both run that
  1267. checkpoint and discard committed data present only in an unlinked WAL."""
  1268. flag = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
  1269. conn = self._conn
  1270. setconfig = getattr(conn, "setconfig", None)
  1271. if flag is None or setconfig is None:
  1272. # Same predicate as _close_time_checkpoint_configurable() plus the per-instance
  1273. # getattr: __init__ binds no retirement capability when either half is missing,
  1274. # and close() must agree with that decision or the lost handle would neither
  1275. # setconfig nor pin.
  1276. return False
  1277. try:
  1278. setconfig(flag, True)
  1279. except Exception:
  1280. # No retention capability is bound on this runtime, so close() will let SQLite run the
  1281. # checkpoint over the newer generation: say so where an operator can see it.
  1282. logger.error(
  1283. "Could not disable SQLite's close-time checkpoint on the quarantined handle for %s; "
  1284. "closing it may checkpoint retired frames over the newer generation.",
  1285. self.db_path, exc_info=True,
  1286. )
  1287. return False
  1288. return True
  1289. def _pin_connection(self, conn) -> None:
  1290. """Retain the exact quarantined connection past GC and interpreter teardown (once per handle).
  1291. Takes the connection as a parameter: callers are lock-held close paths, and the
  1292. writer-conn thread-safety audit flags self._conn in functions outside `with self._lock`."""
  1293. if not self._connection_pinned:
  1294. self._retire_connection(conn)
  1295. self._connection_pinned = True
  1296. def _settle_lost_generation_locked(self) -> bool:
  1297. """Capture the retired generation; return whether the handle must be retired unclosed.
  1298. Where SQLite's close-time checkpoint cannot be switched off (no setconfig, Python < 3.12),
  1299. sqlite3_close would write the retired frames over the newer generation, so the exact
  1300. connection is retired unclosed instead. A failed capture leaves the handle open for a retry
  1301. -- but the pin is taken FIRST on such a runtime: every production caller reaches close()
  1302. through hermes_state_registry.release_or_close, which swallows the error, so an interpreter
  1303. exit before the retry must not be able to checkpoint the stale frames either."""
  1304. self._db_wal_generation_lost = True
  1305. retire_without_close = not self._disable_close_time_checkpoint() and self._retire_connection is not None
  1306. try:
  1307. artifact = self._capture_retired_generation("close")
  1308. except RetiredGenerationCaptureError as exc:
  1309. if retire_without_close:
  1310. self._pin_connection(self._conn)
  1311. logger.error(
  1312. "Could not capture the retired WAL generation of %s at close: %s. The handle stays open "
  1313. "and close() retries the capture; those frames are NOT yet preserved.", self.db_path, exc,
  1314. )
  1315. raise
  1316. logger.warning(
  1317. "Skipping the close-time WAL checkpoint for %s: this handle's WAL/SHM generation "
  1318. "was deleted or replaced; the retired generation is captured at %s. Stop the other "
  1319. "writers before reopening and inspect the capture before deciding its disposition.",
  1320. self.db_path, artifact,
  1321. )
  1322. if retire_without_close:
  1323. logger.warning(
  1324. "Retaining the quarantined connection for %s unclosed: this runtime cannot "
  1325. "switch off SQLite's close-time checkpoint.", self.db_path,
  1326. )
  1327. return retire_without_close
  1328. def _raise_if_db_corrupt(self, *, storage: bool = False) -> None:
  1329. if self._db_corrupt:
  1330. raise self._corrupt_error()
  1331. if storage and storage_state(self.db_path) == STORAGE_CORRUPT:
  1332. # Another handle in this process already saw structural damage on this file.
  1333. # Quarantine this one before it touches SQLite; the error type is the same
  1334. # StateDbCorruptError, so every transcript-diversion owner handles it unchanged.
  1335. self._halt_db_corrupt(sqlite3.DatabaseError(
  1336. "database disk image is malformed (reported earlier in this process: "
  1337. f"{storage_corrupt_reason(self.db_path)})"))
  1338. def _sleep_before_write_retry(self, deadline: float, patience_s: float) -> bool:
  1339. """Sleep one jitter interval if the budget allows; True = retry, False = deadline passed. Small
  1340. jitter for the first _WRITE_RETRY_SLOW_AFTER_S, then slow; never overshoots the deadline."""
  1341. now = time.monotonic()
  1342. if now >= deadline:
  1343. return False
  1344. slow = now - (deadline - patience_s) >= self._WRITE_RETRY_SLOW_AFTER_S
  1345. jitter = random.uniform(*(
  1346. (self._WRITE_RETRY_SLOW_MIN_S, self._WRITE_RETRY_SLOW_MAX_S) if slow
  1347. else (self._WRITE_RETRY_MIN_S, self._WRITE_RETRY_MAX_S)
  1348. ))
  1349. time.sleep(min(jitter, max(deadline - now, 0.001)))
  1350. return True
  1351. def _foreign_state_db_holders(self) -> list[tuple[int, str]]:
  1352. """Foreign processes holding this DB or its WAL sidecars (see hermes_state_holders)."""
  1353. return _foreign_state_db_holders(self.db_path)
  1354. def _quarantine_reason(self) -> Optional[str]:
  1355. """Why this handle must not checkpoint or run in-file repair, or None. A corrupted image has
  1356. torn B-trees; a replaced file or a deleted/replaced WAL generation would checkpoint under
  1357. wrong page numbers into the main DB -- the shutdown-time cause of #105670. Precedence note:
  1358. close() evaluates generation loss BEFORE calling this (and skips it entirely when lost —
  1359. a lost generation settles through the capture path, not the quarantine advisory), while
  1360. the halt path checks replaced first."""
  1361. if self._db_corrupt:
  1362. return f"structural corruption ({self._db_corrupt_reason})"
  1363. if self._db_replaced:
  1364. return "a replaced state.db file"
  1365. if self._db_wal_generation_lost:
  1366. return "a deleted WAL generation (split-brain)"
  1367. return None
  1368. def _try_wal_checkpoint(self) -> None:
  1369. """Best-effort PASSIVE WAL checkpoint; never raises. PASSIVE never blocks writers;
  1370. TRUNCATE corrupted B-trees on 65K+ page databases under exclusive-lock I/O pressure.
  1371. Previous TRUNCATE strategy caused B-tree corruption on large databases (65K+ pages) due to the
  1372. exclusive-lock I/O pressure from checkpointing thousands of frames at once (issue #45383).
  1373. """
  1374. if self._quarantine_reason() is not None:
  1375. return
  1376. try:
  1377. with self._lock:
  1378. if self._conn is None:
  1379. return # closed underneath the timer: nothing to checkpoint, nothing to re-guard
  1380. if self._wal_lock_guard:
  1381. _lockguard.hold(self.db_path, self._wal_lock_guard) # a -shm minted after open
  1382. result = self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)").fetchone()
  1383. if result and result[1] > 0:
  1384. logger.debug("WAL checkpoint: %d/%d pages checkpointed", result[2], result[1])
  1385. except Exception as exc:
  1386. logger.warning("WAL checkpoint (PASSIVE) failed: %s", exc)
  1387. def __enter__(self) -> "SessionDB":
  1388. """``with SessionDB(path) as db:`` closes on exit; owners must release deterministically.
  1389. Ownership of a SessionDB should be released explicitly. Historically an instance with a started
  1390. token writer pinned ITSELF (bound-method writer target plus a strong ``atexit`` drain hook), so
  1391. ``__del__`` never ran for exactly the instances that leaked descriptors (#88033). The writer now
  1392. retires after an idle window and the atexit hook holds only a weak reference, so abandoned handles
  1393. are eventually collectible — but "eventually, after the idle window and a GC cycle" is not a release
  1394. policy. Call sites owning a handle are still expected to close it deterministically (see the
  1395. ownership comments in ``run_agent.py`` and ``tui_gateway/methods_session.py``).
  1396. """
  1397. return self
  1398. def __exit__(self, exc_type, exc, tb) -> bool:
  1399. self.close()
  1400. return False # never suppress the caller's exception
  1401. def close(self):
  1402. """Drain queued token deltas, then a PASSIVE checkpoint on writable handles
  1403. (NOT TRUNCATE: a full WAL reset races the gateway's live writer, tearing
  1404. B-tree pages). A registry-shared instance RELEASES one refcount instead.
  1405. Drains queued token deltas first (the background writer needs the connection). Read-only connections
  1406. never request a checkpoint. See #45383.
  1407. When this instance is shared (opened via ``hermes_state_registry.acquire``), ``close()`` RELEASES one
  1408. refcount instead of tearing down the connection: the registry owns the lifecycle and only closes on
  1409. the final release (#90837). This prevents one caller's close from tearing down the writer connection
  1410. that other callers in the same process are still using — while still letting legacy ``close()`` call
  1411. sites return their reference instead of leaking it.
  1412. """
  1413. if self._shared_registry_owned:
  1414. from hermes_state_registry import release
  1415. release(self)
  1416. return
  1417. self._stop_token_writer()
  1418. hook, self._token_atexit_hook = self._token_atexit_hook, None
  1419. if hook is not None:
  1420. atexit.unregister(hook)
  1421. # Closed flag first: an in-flight reader then closes its own connection.
  1422. with self._read_conns_lock:
  1423. self._read_conns_closed = True
  1424. while self._evict_one_idle_read_conn():
  1425. pass
  1426. with self._lock:
  1427. if self._conn:
  1428. generation_lost = not self.read_only and (
  1429. self._db_wal_generation_lost
  1430. or (bool(self._db_sidecar_identity) and self._wal_generation_was_lost())
  1431. )
  1432. # Loss is settled here, not at exit: the unlinked WAL inode dies with this process's
  1433. # last descriptor (the capture raises and the handle stays open when it fails).
  1434. retire_without_close = generation_lost and self._settle_lost_generation_locked()
  1435. quarantine_reason = None if generation_lost else self._quarantine_reason()
  1436. if quarantine_reason is not None:
  1437. logger.warning(
  1438. "Skipping the close-time WAL checkpoint for %s: this "
  1439. "handle observed %s. Take a snapshot of state.db, -wal and -shm "
  1440. "before restarting, then run `hermes sessions recover --source %s --inspect-only`.",
  1441. self.db_path, quarantine_reason, self.db_path,
  1442. )
  1443. elif not self.read_only and not generation_lost: # PASSIVE, not TRUNCATE (see docstring)
  1444. try:
  1445. # Every cron run_agent opens+closes a transient SessionDB, so a TRUNCATE here fires
  1446. # a full WAL reset many times/hour, racing the gateway's long-lived writer on large
  1447. # WAL databases and tearing hot B-tree pages -- the #45383 corruption this class's
  1448. # own periodic checkpoint was already made PASSIVE to avoid. TRUNCATE belongs only
  1449. # on a sole-opener/quiescent connection.
  1450. self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
  1451. except Exception as exc:
  1452. logger.debug("WAL checkpoint (PASSIVE) at close failed: %s", exc)
  1453. _lockguard.release(self._wal_lock_guard) # before the close: see release()
  1454. if retire_without_close:
  1455. self._pin_connection(self._conn)
  1456. self._conn = None
  1457. else:
  1458. conn, self._conn = self._conn, None
  1459. self._close_connection_quietly(conn)
  1460. # Only a clean close ends the generation; retain the recorded
  1461. # identity when retiring an unsafe handle.
  1462. self._db_sidecar_identity = {}
  1463. self._read_budget.unregister(self) # idempotent: a never-registered (failed-init) handle is a no-op
  1464. def __del__(self) -> None:
  1465. """Safety net: close() if the caller forgot. Attribute access stays
  1466. guarded: module teardown order is undefined."""
  1467. if self.__dict__.get("_conn") is not None:
  1468. try:
  1469. self.close()
  1470. except Exception:
  1471. pass
  1472. # ── Async token accounting (SessionUsageMixin) ──
  1473. # queue_token_counts() is a deque append; a single-writer thread applies deltas in
  1474. # order, coalescing consecutive deltas whose route fields are EQUAL (so the merged
  1475. # UPDATE equals applying them sequentially). Exact readers call flush_token_counts().
  1476. _TOKEN_DELTA_SUM_FIELDS = (
  1477. "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens",
  1478. "api_call_count",
  1479. )
  1480. _TOKEN_DELTA_COST_FIELDS = ("estimated_cost_usd", "actual_cost_usd")
  1481. _TOKEN_DELTA_ROUTE_FIELDS = (
  1482. "model", "cost_status", "cost_source", "pricing_version", "billing_provider", "billing_base_url",
  1483. "billing_mode", "source", "task",
  1484. )
  1485. MAX_TITLE_LENGTH = 100
  1486. # Title provenance, lowest to highest authority: auto-titling may only replace a
  1487. # strictly lower-authority title (``derived`` -> ``llm`` once; never a user-typed name).
  1488. TITLE_SOURCE_DERIVED = _TITLE_SOURCE_DERIVED
  1489. TITLE_SOURCE_LLM = _TITLE_SOURCE_LLM
  1490. TITLE_SOURCE_USER = _TITLE_SOURCE_USER
  1491. _TITLE_SOURCE_RANK = {TITLE_SOURCE_DERIVED: 0, TITLE_SOURCE_LLM: 1, TITLE_SOURCE_USER: 2}
  1492. # Bot Mode's canonical chat is resolved by exact-title lookup: the title IS the identity,
  1493. # so _set_session_title refuses renames of a hidden row holding it.
  1494. # Bot Mode's forever-chat registry: the session titled exactly this, on a bot's profile, IS the bot's
  1495. # canonical chat — resolved by exact-title lookup on every open (no session-id pointer exists). See
  1496. # #92473.
  1497. CANONICAL_BOT_CHAT_TITLE = "Bot Chat"
  1498. # ── Message storage constants (SessionMessagesMixin) ──
  1499. # Prefix marking JSON-encoded structured content; NUL cannot collide with text.
  1500. _CONTENT_JSON_PREFIX = "\x00json:"
  1501. #: Reactions live inside ``display_metadata`` so they survive row rewrites.
  1502. REACTIONS_METADATA_KEY = "reactions"
  1503. # Columns every conversation projection decodes; ``active`` rides along so a display read
  1504. # can split compaction-archived rows without a second query. Contract: must include every
  1505. # ``agent.transcript_repair._OWNED_COLUMNS`` column, because replay stamps
  1506. # ``transcript_row_snapshot(row)`` from these rows (token_count is hashed there, not decoded).
  1507. _CONVERSATION_ROW_COLUMNS = (
  1508. "id, role, content, tool_call_id, tool_calls, tool_name, effect_disposition, "
  1509. "finish_reason, reasoning, reasoning_content, reasoning_details, "
  1510. "codex_reasoning_items, codex_message_items, platform_message_id, observed, "
  1511. "_compressed_summary, timestamp, token_count, active, api_content, display_kind, display_metadata, message_uid, "
  1512. "absorbed_message_uids, tool_call_uids, tool_call_uid"
  1513. )
  1514. # ── Meta key/value (scheduler bookkeeping) ──
  1515. def get_meta(self, key: str) -> Optional[str]:
  1516. """Read state_meta[key] on self._lock (not _read_ctx): fts_rebuild_step reads progress before its
  1517. write transaction and a WAL reader would not see it."""
  1518. with self._lock:
  1519. row = self._conn.execute("SELECT value FROM state_meta WHERE key = ?", (key,)).fetchone()
  1520. return None if row is None else row[0]
  1521. def set_meta(self, key: str, value: str, *, cursor: Optional[sqlite3.Cursor] = None) -> None:
  1522. """Upsert state_meta[key]; with ``cursor`` the write is inline (the caller already holds a
  1523. transaction — nesting BEGIN IMMEDIATE would deadlock)."""
  1524. sql = (
  1525. "INSERT INTO state_meta (key, value) VALUES (?, ?) "
  1526. "ON CONFLICT(key) DO UPDATE SET value = excluded.value"
  1527. )
  1528. if cursor is not None:
  1529. cursor.execute(sql, (key, value))
  1530. else:
  1531. self._write_sql(sql, (key, value))
  1532. def retag_kanban_worker_sessions(self, workspaces_root: str) -> int:
  1533. """Retag legacy kanban worker rows from ``cli`` to ``kanban`` by cwd under the board's workspaces
  1534. root; gated once per root via state_meta. Returns rows retagged."""
  1535. prefix = str(workspaces_root).rstrip("/\\")
  1536. if not prefix:
  1537. return 0
  1538. gate = f"kanban_worker_source_retagged:{prefix}"
  1539. if self.get_meta(gate) == "1":
  1540. return 0
  1541. def _do(conn):
  1542. esc = _escape_like(prefix)
  1543. cursor = conn.execute(
  1544. "UPDATE sessions SET source = 'kanban' "
  1545. "WHERE source = 'cli' AND (cwd = ? OR cwd LIKE ? ESCAPE '\\' "
  1546. "OR cwd LIKE ? ESCAPE '\\')",
  1547. (prefix, f"{esc}/%", f"{esc}\\\\%"),
  1548. )
  1549. # rowcount BEFORE set_meta reuses this cursor for its INSERT.
  1550. retagged = cursor.rowcount or 0
  1551. self.set_meta(gate, "1", cursor=cursor)
  1552. return retagged
  1553. return self._execute_write(_do)
  1554. def is_kanban_owned_session(self, session_id: str) -> bool:
  1555. """True when this session — or any segment of the compression lineage a resume would
  1556. materialize — belongs to the Kanban dispatcher (``source``/``created_source`` =
  1557. ``'kanban'``): the transcript of a worker run, not a human conversation (#68779).
  1558. Both columns are checked: ``source`` is live routing state the dispatcher tags
  1559. (``HERMES_SESSION_SOURCE=kanban``) and the legacy retag rewrites, while immutable
  1560. ``created_source`` survives later surface flips. The lineage is the VERIFIED
  1561. compression chain (``_resume_lineage_ids``) — exactly the rows a resume loads — so a
  1562. worker transcript stays kanban-owned across rotations, while a delegate/branch child
  1563. of a worker (a DIFFERENT conversation) is not swept in. Plain classification, not a
  1564. gate: the resume-time guard (``hermes_cli/kanban_resume_guard.py``) owns the decision
  1565. and the dispatcher-owned exemption."""
  1566. lineage = self._resume_lineage_ids(session_id)
  1567. if not lineage:
  1568. return False
  1569. rows = self._read_all(
  1570. f"SELECT 1 FROM sessions WHERE id IN ({_placeholders(lineage)}) "
  1571. "AND (source = 'kanban' OR created_source = 'kanban') LIMIT 1",
  1572. tuple(lineage))
  1573. return bool(rows)
  1574. def list_meta_prefix(self, prefix: str) -> list[tuple[str, str]]:
  1575. """``[(key, value), ...]`` for state_meta keys starting with the literal
  1576. ``prefix`` (LIKE wildcards escaped) — e.g. ``loop:<session_id>`` rows."""
  1577. if not prefix:
  1578. return []
  1579. rows = self._read_all(
  1580. "SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'", (_escape_like(prefix) + "%",),
  1581. )
  1582. return [(row[0], row[1]) for row in rows]
  1583. class AsyncSessionDB:
  1584. """Async door onto SessionDB: every call runs via asyncio.to_thread so a blocking SQLite call
  1585. never freezes the event loop (no method returns a live cursor)."""
  1586. def __init__(self, db: SessionDB) -> None:
  1587. self._db = db
  1588. def __getattr__(self, name: str):
  1589. attr = getattr(self._db, name)
  1590. if not callable(attr):
  1591. return attr
  1592. async def _offloaded(*args, **kwargs):
  1593. return await asyncio.to_thread(attr, *args, **kwargs)
  1594. return _offloaded