session_transcript.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682
  1. """SessionStore transcript I/O: SQLite append with a per-session retry queue, compression-reroute
  2. following, FTS corruption recovery, rewrite/rewind/load. Mixin split out of ``gateway/session.py``;
  3. bound onto ``SessionStore`` via the MRO."""
  4. from __future__ import annotations
  5. import contextlib
  6. import json
  7. import logging
  8. import threading
  9. import time
  10. from agent.turn_context import extract_api_content_sidecar
  11. from typing import TYPE_CHECKING, Any, Dict, List, Optional
  12. if TYPE_CHECKING:
  13. from gateway.session import SessionEntry
  14. # Log-record parity with the origin module.
  15. logger = logging.getLogger("gateway.session")
  16. class TranscriptReadError(RuntimeError):
  17. """Raised when persisted history cannot be read safely."""
  18. def __init__(self, session_id: str) -> None:
  19. self.session_id = session_id
  20. super().__init__(f"transcript read failed for session {session_id}")
  21. def _spool_dropped(session_id: str, message: dict[str, Any]):
  22. """Spool one evicted/undeliverable message to disk (same machinery as the shutdown flush, so it
  23. is replayed after DB recovery); path or None."""
  24. try:
  25. from gateway.shutdown_flush import spool_dropped_transcript_message
  26. return spool_dropped_transcript_message(session_id, message)
  27. except Exception:
  28. return None
  29. # Message keys persisted only for assistant rows (None otherwise).
  30. _ASSISTANT_ONLY_KEYS = (
  31. "reasoning", "reasoning_content", "reasoning_details", "codex_reasoning_items",
  32. "codex_message_items",
  33. )
  34. # Message keys bound straight to a TEXT column. A spooled message is JSON from disk, and a dict or list
  35. # here makes sqlite refuse the whole row.
  36. _BOUND_TEXT_KEYS = ("role", "tool_name", "tool_call_id", "reasoning", "reasoning_content", "platform_message_id")
  37. def _bindable_text(value: Any) -> Any:
  38. """*value* when sqlite can bind it as is, else its JSON text (``str`` if that fails too). An int
  39. outside sqlite's signed 64-bit range raises OverflowError even for a TEXT column."""
  40. if value is None or isinstance(value, (str, float)) or (
  41. isinstance(value, int) and -2**63 <= value < 2**63):
  42. return value
  43. try:
  44. return json.dumps(value, default=str)
  45. except (TypeError, ValueError):
  46. return str(value)
  47. def transcript_append_kwargs(session_id: str, message: dict[str, Any], *, fallback_ts: Any = None) -> dict[str, Any]:
  48. """``SessionDB.append_message`` kwargs for one transcript row. The live writer and the restart
  49. replay of the transcript spool (gateway/shutdown_flush.py) both build rows here, so a replayed
  50. message lands as the row the live drain would have written. Fields are listed rather than
  51. splatted: a spooled message is arbitrary JSON from disk, and an unexpected key would raise and
  52. abort the replay. *fallback_ts* applies only to a message with no timestamp at all, because a
  53. truthiness test would rewrite epoch 0. Values bound straight to a TEXT column are made bindable
  54. (:func:`_bindable_text`), so one malformed field cannot make the row unwritable."""
  55. is_assistant = message.get("role") == "assistant"
  56. timestamp = message.get("timestamp")
  57. kwargs = {
  58. "session_id": session_id,
  59. "role": message.get("role", "unknown"),
  60. "content": message.get("content"),
  61. "tool_name": message.get("tool_name"),
  62. "tool_calls": message.get("tool_calls"),
  63. "tool_call_id": message.get("tool_call_id"),
  64. **{k: message.get(k) if is_assistant else None for k in _ASSISTANT_ONLY_KEYS},
  65. "platform_message_id": message.get("platform_message_id") or message.get("message_id"),
  66. "observed": bool(message.get("observed")),
  67. "timestamp": fallback_ts if timestamp is None else timestamp,
  68. # Exact bytes sent to the API (prompt-cache-stable replay); must survive every
  69. # persistence path or the next replay diverges.
  70. "api_content": extract_api_content_sidecar(message),
  71. # Presentation typing ("internal_notification" for self-injected async-delegation/background
  72. # notification turns, #82888). DB-only; stripped from provider-bound payloads.
  73. "display_kind": message.get("display_kind"),
  74. "display_metadata": message.get("display_metadata"),
  75. }
  76. for key in _BOUND_TEXT_KEYS:
  77. kwargs[key] = _bindable_text(kwargs[key])
  78. return kwargs
  79. class SessionTranscriptMixin:
  80. """SessionStore transcript I/O: SQLite append with a per-session retry queue,
  81. compression-reroute following, FTS corruption recovery, rewrite/rewind/load."""
  82. _MAX_PENDING_PER_SESSION = 200 # in-memory pending messages per session (DB broken)
  83. # Cooldown between FTS5 rebuild attempts (see _rebuild_fts_once); avoids permanently
  84. # disabling recovery after one failed attempt while still avoiding a rebuild storm against
  85. # a database that is corrupt on every write.
  86. _FTS_REBUILD_COOLDOWN_SECONDS = 300
  87. # Consecutive transcript-append failures for one session before escalating from WARNING to
  88. # ERROR (see _append_to_transcript_serialized); a session stalled past this many attempts is
  89. # no longer a transient blip and needs operator attention.
  90. _TRANSCRIPT_APPEND_FAILURE_ESCALATION_THRESHOLD = 3
  91. def _compression_tip_for_session_id(self, session_id: Optional[str]) -> Optional[str]:
  92. """Latest compression continuation for *session_id* (heals a mapping left pointing at a
  93. compressed parent by a restart or failed send)."""
  94. db = self._db_for_session_id(session_id) if session_id else None
  95. if db is None:
  96. return session_id
  97. try:
  98. return db.get_compression_tip(session_id) or session_id
  99. except Exception:
  100. logger.debug("Compression-tip lookup failed for session %s", session_id, exc_info=True)
  101. return session_id
  102. def _heal_compression_tip_locked(
  103. self, entry: SessionEntry, original_session_id: Optional[str],
  104. canonical_session_id: Optional[str]) -> bool:
  105. """Rewrite *entry* to the compression continuation if stale. Lock held."""
  106. if not original_session_id or not canonical_session_id:
  107. return False
  108. if entry.session_id != original_session_id or canonical_session_id == original_session_id:
  109. return False
  110. logger.info(
  111. "SessionStore healed compressed session mapping: %s -> %s", entry.session_id,
  112. canonical_session_id)
  113. entry.session_id = canonical_session_id
  114. return True
  115. def advance_compression_session(
  116. self, session_key: str, expected_session_id: str, target_session_id: str,
  117. ) -> Optional[SessionEntry]:
  118. """CAS-advance one route along an already-verified compression lineage. Unlike
  119. ``switch_session`` this never ends/reopens SQLite rows (the compression transaction owns
  120. that). ``None`` means the route moved after the caller's snapshot (e.g. /new) — caller
  121. must fail closed."""
  122. if not session_key or not expected_session_id or not target_session_id:
  123. return None
  124. with self._lock:
  125. entry = self._entry_locked(session_key)
  126. if entry is None:
  127. return None
  128. if entry.session_id == target_session_id:
  129. return entry
  130. if not self._heal_compression_tip_locked(entry, expected_session_id, target_session_id):
  131. return None # route moved (session_id != expected) or nothing to heal
  132. self._save() # bookkeeping, not user activity: leave ``updated_at`` alone
  133. return entry
  134. def _get_transcript_drain_lock(self):
  135. """Return the lock that serializes pending-queue drain boundaries."""
  136. return self._lazy("_transcript_drain_lock", threading.RLock)
  137. def append_to_transcript(self, session_id: str, message: dict[str, Any], skip_db: bool = False) -> None:
  138. """Serialize transcript draining across queue migration boundaries. A session with no usable
  139. store is NOT skipped: the write is queued and counted like any other failed append, so a
  140. dead/unopenable state.db escalates and spools instead of dropping turns silently
  141. (#114266)."""
  142. if skip_db:
  143. return
  144. with self._get_transcript_drain_lock():
  145. self._append_to_transcript_serialized(self._follow_reroutes(session_id), message)
  146. def _follow_reroutes(self, session_id: str) -> str:
  147. """Follow the compression reroute chain (cycle-guarded)."""
  148. reroutes = self._lazy("_transcript_reroutes", dict)
  149. seen = set()
  150. while session_id in reroutes and session_id not in seen:
  151. seen.add(session_id)
  152. session_id = reroutes[session_id]
  153. return session_id
  154. def _enqueue_transcript_message(self, session_id: str, message: dict[str, Any]) -> list:
  155. """Queue *message* (retry lock held); evicts + spools the oldest past the cap."""
  156. pending = self._dirty_transcripts.setdefault(session_id, [])
  157. pending.append(dict(message))
  158. # Cap pending messages per session to avoid unbounded memory growth when the DB is persistently
  159. # broken. Spool the evicted oldest message to the on-disk pending spool (same machinery
  160. # flush_pending_to_file uses at shutdown) so a runtime cap rotation does not silently discard it
  161. # (#78182); it is replayed on the next successful transcript flush.
  162. if len(pending) > self._MAX_PENDING_PER_SESSION:
  163. spool_path = _spool_dropped(session_id, pending.pop(0))
  164. if spool_path is not None:
  165. self._lazy("_spooled_drop_sessions", set).add(session_id)
  166. logger.warning(
  167. "Session DB transcript pending queue full for %s (cap=%d); spooled oldest "
  168. "message to %s for replay after DB recovery", session_id,
  169. self._MAX_PENDING_PER_SESSION, spool_path)
  170. else:
  171. logger.warning(
  172. "Session DB transcript pending queue full for %s (cap=%d); dropping oldest "
  173. "message to make room (on-disk spool unavailable)", session_id,
  174. self._MAX_PENDING_PER_SESSION)
  175. return pending
  176. def _divert_transcript_after_db_replaced(
  177. self, session_id: str, queue_session_id: str, exc: Exception
  178. ) -> None:
  179. """Stop SQLite writes on a replaced/quarantined handle and divert the backlog to the on-disk
  180. spool + JSONL fallback: retrying cannot succeed and the FTS rebuild must never run here."""
  181. logger.error(
  182. "Session DB refused further writes on this handle for %s (%s); stopping SQLite writes "
  183. "and diverting pending transcripts to the on-disk fallback: %s", session_id,
  184. type(exc).__name__, exc)
  185. with self._transcript_retry_lock:
  186. remaining = list(self._dirty_transcripts.get(queue_session_id, []))
  187. self._dirty_transcripts.pop(queue_session_id, None)
  188. self._transcript_append_failures.pop(session_id, None)
  189. for dropped in remaining:
  190. if _spool_dropped(session_id, dropped) is None:
  191. logger.warning(
  192. "pending fallback failed for replaced state.db transcript on %s", session_id,
  193. exc_info=True)
  194. try:
  195. from hermes_state import divert_session_transcript_jsonl
  196. divert_session_transcript_jsonl(session_id, remaining)
  197. except Exception:
  198. logger.warning(
  199. "JSONL divert failed for replaced state.db transcript on %s", session_id,
  200. exc_info=True)
  201. def _live_compression_child(self, session_id: str) -> str:
  202. """Transitive compression tip of *session_id* if it is a different, still-live row, else ""
  203. (a depth-1 lookup misses multi-hop lineages). Uses the PARENT's proven owner handle: the
  204. child's id is unpublished until its write succeeds, so a by-id lookup would hit the ambient
  205. store."""
  206. owner_db = self._db_for_session_id(session_id)
  207. tip = owner_db.get_compression_tip(session_id) if owner_db is not None else None
  208. if tip and tip != session_id:
  209. tip_row = owner_db.get_session(tip)
  210. if tip_row is not None and tip_row.get("ended_at") is None:
  211. return str(tip)
  212. return ""
  213. def _migrate_transcript_queue_to_child(
  214. self, session_id: str, queue_session_id: str, child_id: str, pending: list, msg
  215. ) -> list:
  216. """Move the retry queue + failure counter from parent to child and record the reroute (retry
  217. lock held); returns the child's pending list. Older parent backlog must precede messages
  218. already queued directly on the child; routing is published only AFTER the queue moved
  219. (caller), so new child writes cannot bypass older parent backlog."""
  220. if pending and pending[0] is msg:
  221. pending.pop(0)
  222. existing_child_pending = self._dirty_transcripts.get(child_id, [])
  223. if pending:
  224. pending.extend(existing_child_pending)
  225. self._dirty_transcripts[child_id] = pending
  226. elif existing_child_pending:
  227. pending = existing_child_pending
  228. self._dirty_transcripts.pop(queue_session_id, None)
  229. previous_failures = self._transcript_append_failures.pop(queue_session_id, 0)
  230. if previous_failures:
  231. self._transcript_append_failures[child_id] = max(
  232. previous_failures, self._transcript_append_failures.get(child_id, 0))
  233. self._transcript_reroutes[session_id] = child_id
  234. return pending
  235. def _publish_transcript_reroute(self, session_id: str, child_id: str) -> None:
  236. """Repoint every route at the compression child and save (index authoritative again)."""
  237. with self._lock:
  238. for entry in self._entries.values():
  239. if entry.session_id == session_id:
  240. entry.session_id = child_id
  241. self._save()
  242. (getattr(self, "_session_owner_hints", None) or {}).pop(child_id, None)
  243. def _append_to_transcript_serialized(self, session_id: str, message: dict[str, Any]) -> None:
  244. """Append a message to a session's transcript (SQLite), draining the per-session retry
  245. queue.
  246. Args: skip_db: When True, skip the SQLite write. Used when the agent already persisted messages to
  247. SQLite via its own _flush_messages_to_session_db(), preventing the duplicate-write bug (#860).
  248. """
  249. with self._transcript_retry_lock:
  250. pending = self._enqueue_transcript_message(session_id, message)
  251. msg = pending[0]
  252. queue_session_id = session_id
  253. def _ack_head() -> bool:
  254. """Pop the acknowledged head (retry lock held). True if queue drained."""
  255. if pending and pending[0] is msg:
  256. pending.pop(0)
  257. if not pending:
  258. self._dirty_transcripts.pop(queue_session_id, None)
  259. self._transcript_append_failures.pop(session_id, None)
  260. return True
  261. return False
  262. # DB write outside the retry lock so other sessions can append.
  263. while True:
  264. spool_exc = None
  265. try:
  266. # Spooled backlog (cap eviction, a stalled session, a boot hold-back) is older than
  267. # ``msg``: replay it first. While any of it stays on disk ``msg`` stays queued, since
  268. # writing it now would give it a lower row id than that backlog for good. The replay's
  269. # own error is raised so the repair/divert handling below still classifies it.
  270. spool_exc = self._drain_spooled_drops(session_id)
  271. if spool_exc is not None:
  272. raise spool_exc
  273. self._append_transcript_message(session_id, msg)
  274. except Exception as exc:
  275. from hermes_state import StateDbCorruptError, StateDbReplacedError
  276. from hermes_state_errors import CompressionSessionClosedError
  277. if isinstance(exc, (StateDbReplacedError, StateDbCorruptError)):
  278. self._divert_transcript_after_db_replaced(session_id, queue_session_id, exc)
  279. return
  280. if isinstance(exc, CompressionSessionClosedError):
  281. # Adopt only a different, still-live compression tip, else fail closed.
  282. _owner_key = self._owner_key_for_session_id(session_id)
  283. child_id = self._live_compression_child(session_id)
  284. if child_id:
  285. # Record the child's owner BEFORE writing to it (the reroute is published
  286. # only after the write succeeds — load-bearing for backlog order).
  287. if _owner_key:
  288. self._lazy("_session_owner_hints", dict)[child_id] = _owner_key
  289. try:
  290. self._append_transcript_message(child_id, msg)
  291. except Exception as reroute_exc:
  292. exc = reroute_exc
  293. else:
  294. with self._transcript_retry_lock:
  295. pending = self._migrate_transcript_queue_to_child(
  296. session_id, queue_session_id, child_id, pending, msg)
  297. queue_session_id = child_id
  298. self._publish_transcript_reroute(session_id, child_id)
  299. if not pending:
  300. return
  301. msg = pending[0]
  302. session_id = child_id
  303. continue
  304. else:
  305. # Permanent routing invariant failure, not a transient outage: drop it so it
  306. # cannot poison later writes.
  307. with self._transcript_retry_lock:
  308. _ack_head()
  309. logger.error(
  310. "Session DB transcript append rejected for compression-ended %s with "
  311. "no unique live child; not retrying", session_id)
  312. return
  313. if self._is_fts_corruption_error(exc) and self._rebuild_fts_once():
  314. if spool_exc is not None:
  315. continue # repaired: drain the older spool again before ``msg``
  316. try:
  317. self._append_transcript_message(session_id, msg)
  318. except Exception as retry_exc:
  319. exc = retry_exc
  320. else:
  321. with self._transcript_retry_lock:
  322. _ack_head()
  323. continue
  324. with self._transcript_retry_lock:
  325. failures = self._transcript_append_failures.get(session_id, 0) + 1
  326. self._transcript_append_failures[session_id] = failures
  327. if failures >= self._TRANSCRIPT_APPEND_FAILURE_ESCALATION_THRESHOLD:
  328. spooled = self._spool_stalled_backlog(session_id, queue_session_id)
  329. logger.error(
  330. "Session DB transcript append failed for %s (failure_count=%d, "
  331. "pending=%d, spooled_to_disk=%d); session is stalled and needs operator "
  332. "attention: %s", session_id, failures, len(pending), spooled, exc)
  333. else:
  334. logger.warning(
  335. "Session DB transcript append failed for %s (failure_count=%d, pending=%d); "
  336. "will retry: %s", session_id, failures, len(pending), exc)
  337. return
  338. else:
  339. with self._transcript_retry_lock:
  340. queue_empty = _ack_head()
  341. if not queue_empty:
  342. msg = pending[0]
  343. if queue_empty:
  344. return
  345. continue
  346. def _spool_stalled_backlog(self, session_id: str, queue_session_id: str) -> int:
  347. """Move a stalled session's in-memory backlog (oldest first) to the on-disk pending spool so
  348. a crash/restart during the outage no longer loses it (#114266): ``recover_pending_to_db``
  349. replays it at boot, ``_drain_spooled_drops`` before the next live write. Stops at the first
  350. spool failure so order holds; whatever stays in memory remains under the cap."""
  351. with self._transcript_retry_lock:
  352. pending = self._dirty_transcripts.get(queue_session_id, [])
  353. backlog = list(pending)
  354. spooled = 0
  355. for message in backlog:
  356. if _spool_dropped(session_id, message) is None:
  357. break
  358. spooled += 1
  359. if spooled:
  360. with self._transcript_retry_lock:
  361. self._lazy("_spooled_drop_sessions", set).add(session_id)
  362. del pending[:spooled]
  363. if not pending:
  364. self._dirty_transcripts.pop(queue_session_id, None)
  365. return spooled
  366. def _drain_spooled_drops(self, session_id: str) -> Exception | None:
  367. """Replay cap-dropped spooled transcript messages after DB recovery; return the replay
  368. failure while some of them are still on disk, else None. Best-effort: replay failures keep
  369. the spool files for the next successful flush; nothing here may raise."""
  370. spooled_sessions = getattr(self, "_spooled_drop_sessions", None)
  371. if not spooled_sessions or session_id not in spooled_sessions:
  372. return None
  373. failures: list[Exception] = []
  374. def replay(message: dict[str, Any]) -> None:
  375. try:
  376. self._append_transcript_message(session_id, message)
  377. except Exception as exc:
  378. failures.append(exc)
  379. raise
  380. try:
  381. from gateway.shutdown_flush import drain_transcript_spool
  382. # Inside an outage the append that follows logs/escalates the same failure; the
  383. # replay attempt is only the order-preserving probe, so its failure stays at DEBUG.
  384. with self._transcript_retry_lock:
  385. known_failing = bool(self._transcript_append_failures.get(session_id))
  386. _replayed, remaining = drain_transcript_spool(
  387. session_id, replay, db_known_failing=known_failing)
  388. if not remaining:
  389. with self._transcript_retry_lock:
  390. spooled_sessions.discard(session_id)
  391. return None
  392. return failures[-1] if failures else RuntimeError(
  393. f"older spooled transcript rows for {session_id} still pending")
  394. except Exception as exc:
  395. logger.warning("Failed to drain transcript spool for %s: %s", session_id, exc)
  396. return None
  397. def mark_spooled_drop_sessions(self, session_ids) -> None:
  398. """Have each session's next transcript write drain the on-disk spool first. Boot recovery marks
  399. the sessions it held back, so their next live row cannot land ahead of those files."""
  400. if not session_ids:
  401. return
  402. with self._transcript_retry_lock:
  403. self._lazy("_spooled_drop_sessions", set).update(session_ids)
  404. def _append_transcript_message(self, session_id: str, message: dict[str, Any]) -> None:
  405. """Write one transcript row. Caller handles retry queuing."""
  406. _db = self._db_for_session_id(session_id)
  407. if _db is None:
  408. # Named profile with no resolvable home yet: defer (caller queues) instead of writing
  409. # into the ambient store.
  410. raise RuntimeError(
  411. f"no owning session store for {session_id}; deferring transcript write")
  412. _db.append_message(**transcript_append_kwargs(session_id, message))
  413. @staticmethod
  414. def _is_fts_corruption_error(exc: Exception) -> bool:
  415. """True only when the failure is provably scoped to the FTS index. A bare SQLITE_CORRUPT
  416. can mean structural B-tree damage; only errors naming ``messages_fts`` or carrying FTS
  417. provenance (``SessionDB._is_fts_write_corruption_error``) may authorize the one-shot
  418. rebuild-and-retry; everything else takes the retry path.
  419. A generic ``database disk image is malformed`` (bare SQLITE_CORRUPT) can mean structural damage to
  420. canonical B-trees, not just the FTS shadow tables — treating it as FTS-only here made the store
  421. rebuild the index and retry transcript writes against a structurally corrupt database (#97940).
  422. """
  423. if "messages_fts" in str(exc).lower():
  424. return True
  425. import sqlite3
  426. from hermes_state import SessionDB
  427. return isinstance(exc, sqlite3.DatabaseError) and SessionDB._is_fts_write_corruption_error(exc)
  428. def _rebuild_fts_once(self) -> bool:
  429. """Attempt FTS5 ``rebuild``, at most once per ``_FTS_REBUILD_COOLDOWN_SECONDS`` window;
  430. True if any index was rebuilt.
  431. A permanent one-shot flag meant a single rebuild failure (e.g. a transient WAL
  432. split-brain guard hit) permanently disabled recovery for the life of the process, even
  433. though later corruption on the same store could be fixable. Retrying on a cooldown lets
  434. the store try again after the underlying condition (e.g. a foreign holder) has likely
  435. cleared, without hammering a database that is corrupt on every write.
  436. """
  437. now = time.monotonic()
  438. last_attempt = self._fts_rebuild_last_attempt_at
  439. if last_attempt is not None and (now - last_attempt) < self._FTS_REBUILD_COOLDOWN_SECONDS:
  440. return False
  441. db = self._db
  442. if db is None or not hasattr(db, "rebuild_fts"):
  443. return False
  444. self._fts_rebuild_last_attempt_at = now
  445. # WAL split-brain guard: skip when a foreign process holds state.db.
  446. foreign_holders = None
  447. if hasattr(db, "_foreign_state_db_holders"):
  448. foreign_holders = db._foreign_state_db_holders()
  449. if foreign_holders:
  450. logger.warning(
  451. "Skipping Session DB FTS rebuild while foreign processes hold the database or "
  452. "WAL sidecars (%s); canonical transcript writes remain available.", foreign_holders)
  453. return False
  454. try:
  455. rebuilt = db.rebuild_fts()
  456. except Exception as exc:
  457. logger.warning("Session DB FTS rebuild failed: %s", exc)
  458. return False
  459. if rebuilt:
  460. logger.warning("Rebuilt %d Session DB FTS index(es) after append corruption", rebuilt)
  461. return rebuilt > 0
  462. def _clear_dirty_transcript(self, session_id: str) -> None:
  463. """Drop queued pending messages so a rewrite/rewind doesn't re-insert them."""
  464. with self._transcript_retry_lock:
  465. self._dirty_transcripts.pop(session_id, None)
  466. self._transcript_append_failures.pop(session_id, None)
  467. def has_platform_message_id(self, session_id: str, platform_message_id: str) -> bool:
  468. """Whether a message with this platform_message_id is persisted (False without a DB).
  469. Thin wrapper over SessionDB.has_platform_message_id(). Returns False when no DB is available
  470. (in-memory sessions). Used by the gateway's transient-failure dedupe guard (#47237).
  471. """
  472. db = self._db_for_session_id(session_id)
  473. if not db:
  474. return False
  475. try:
  476. return db.has_platform_message_id(session_id, platform_message_id)
  477. except Exception:
  478. logger.debug("has_platform_message_id lookup failed", exc_info=True)
  479. return False
  480. def transcript_tail_role(self, session_id: str) -> Optional[str]:
  481. """Role of the newest live conversation row on the route ``load_transcript`` reads (``None``
  482. when empty, no DB, or the read fails — the boundary write would fail the same way)."""
  483. session_id = self._compression_tip_for_session_id(self._follow_reroutes(session_id))
  484. db = self._db_for_session_id(session_id)
  485. if not db:
  486. return None
  487. try:
  488. return db.latest_conversation_role(session_id)
  489. except Exception:
  490. logger.debug("transcript tail lookup failed for %s", session_id, exc_info=True)
  491. return None
  492. def rewrite_transcript(
  493. self, session_id: str, messages: list[dict[str, Any]], active_only: bool = False,
  494. reject_active_turn_lease: bool = False) -> bool:
  495. """Replace a session's transcript (/retry, /compress). DESTRUCTIVE by default:
  496. ``active_only=False`` DELETEs every row incl. soft-archived compaction history (pass
  497. ``active_only=True`` for sessions that may carry archived rows). True when the write lands
  498. or there is no DB, False on failure — callers committing a destructive change on top
  499. (/compress repointing) must check it. ``reject_active_turn_lease`` is for user-initiated
  500. rewrites that do not own the cross-process turn lease."""
  501. db = self._db_for_session_id(session_id)
  502. if not db:
  503. return True
  504. with self._get_transcript_drain_lock():
  505. try:
  506. # Even when the current agent doesn't "own" persistence, the session on disk may already
  507. # carry compaction-archived rows — e.g. after a model switch or a /restore, both of which
  508. # mint a fresh agent with _session_db_created=False (so the check above is False) yet leave
  509. # the durable archived transcript in place. A full-history replace would DELETE those
  510. # archived rows just like the owned-agent case. Guard against it by replacing ONLY the live
  511. # (active=1) set unconditionally: on a fresh create/fork every row is active=1, so
  512. # active-only replace is behaviorally identical to the full replace — and when archived rows
  513. # DO exist they survive. An existence probe here (has_archived_messages) would fail OPEN
  514. # into the destructive replace on any DB error and can race a concurrent archive_and_compact
  515. # — the same probe failure mode #80216's /retry fix (gateway/slash_commands.py) deliberately
  516. # avoids.
  517. db.replace_messages(
  518. session_id, messages, active_only=active_only,
  519. reject_active_turn_lease=reject_active_turn_lease)
  520. except Exception as e:
  521. logger.debug("Failed to rewrite transcript in DB: %s", e)
  522. return False
  523. self._clear_dirty_transcript(session_id)
  524. return True
  525. def is_published_compression_child(self, parent_session_id: str, child_session_id: str) -> bool:
  526. """True when *child_session_id* is an atomically-published compression continuation of
  527. *parent_session_id*: ``publish_compression_child()`` wrote the handoff (plus any cloned
  528. foreign-tail rows) in the SAME transaction that ended the parent with
  529. ``end_reason='compression'``, so the child transcript is already durable on disk.
  530. Rewriting such a child is redundant AND destructive: the in-memory handoff lacks the rows
  531. cloned during publication, and a failed rewrite reports a false "failed to persist
  532. compressed transcript" over a committed compression. Fails open (False) on any DB error so
  533. callers fall back to the rewrite."""
  534. if not parent_session_id or not child_session_id:
  535. return False
  536. # The child id is not routed yet; the parent is, and it was ended in the same publish transaction.
  537. db = self._db_for_session_id(parent_session_id)
  538. if db is None:
  539. return False
  540. try:
  541. child = db.get_session(child_session_id)
  542. if not child or child.get("parent_session_id") != parent_session_id:
  543. return False
  544. if not db._is_compression_child_row(child):
  545. return False
  546. return db.get_active_message_watermark(child_session_id) > 0
  547. except Exception:
  548. logger.debug("published-compression-child probe failed for %s", child_session_id, exc_info=True)
  549. return False
  550. def persist_rotated_compression_child(
  551. self, parent_session_id: str, child_session_id: str, messages: list[dict[str, Any]]) -> bool:
  552. """Make a rotated compression child durable before the live entry is repointed: a published
  553. child already is, anything else gets *messages* written by the destructive rewrite."""
  554. return (self.is_published_compression_child(parent_session_id, child_session_id)
  555. or self.rewrite_transcript(child_session_id, messages))
  556. def has_input_owner(self, session_id: str, owner: str) -> bool:
  557. """Find this accepted input on the canonical live continuation and its ancestors.
  558. Content and unrelated writers cannot establish ownership. Query only existence;
  559. compaction archives can contain many megabytes that replay never needs to load.
  560. """
  561. try:
  562. current = self._follow_reroutes(session_id)
  563. db = self._db_for_session_id(current)
  564. current = db.get_compression_tip(current) or current
  565. seen = set()
  566. while current and current not in seen:
  567. seen.add(current)
  568. if db.has_gateway_input_owner(current, owner):
  569. return True
  570. row = db.get_session(current)
  571. if not row or not db._is_compression_child_row(row):
  572. break
  573. current = row["parent_session_id"]
  574. return False
  575. except Exception as e:
  576. raise TranscriptReadError(session_id) from e
  577. def load_transcript(self, session_id: str) -> list[dict[str, Any]]:
  578. """Load all messages from a session's transcript (state.db is canonical). Reads follow the
  579. same routing writes use — the in-memory reroute map, then the durable compression tip —
  580. otherwise the transcript "vanishes" while every message sits under the child."""
  581. if not self._db_for_session_id(session_id):
  582. return []
  583. session_id = self._follow_reroutes(session_id)
  584. with contextlib.suppress(Exception):
  585. # Durable successor survives restart; the reroute map doesn't.
  586. db = self._db_for_session_id(session_id)
  587. session_id = db.get_compression_tip(session_id) or session_id
  588. try:
  589. # repair_alternation: this feeds LIVE REPLAY; heal a durable user;user wedge once here.
  590. return self._db_for_session_id(session_id).get_messages_as_conversation(
  591. session_id, repair_alternation=True)
  592. except Exception as e:
  593. # Empty history is valid data; a failed canonical read is not — live-replay callers
  594. # must fail closed, not start from [].
  595. logger.error(
  596. "Transcript read failed for session %s; refusing to treat the conversation as "
  597. "empty: %s", session_id, e, exc_info=True)
  598. raise TranscriptReadError(session_id) from e
  599. def rewind_session(
  600. self, session_id: str, n: int = 1, *, require_retryable_composite: bool = False,
  601. ) -> Optional[dict[str, Any]]:
  602. """Back up ``n`` user turns via soft-delete (``active=0``), mirroring CLI ``/undo [N]``.
  603. Returns ``{"rewound_count", "turns_undone", "target_text"}`` or ``None`` (no DB / no rewindable
  604. turn / persistence failure); ``n`` clamps to the oldest user turn. ``require_retryable_composite``
  605. is the gateway ``/retry`` guard: the selected turn must be a composite carrier whose live payload
  606. is losslessly replayable as text — that replay-policy ``ValueError`` propagates so /retry can
  607. explain why the carrier is unsafe."""
  608. db = self._db_for_session_id(session_id)
  609. if not db:
  610. return None
  611. from hermes_state_rewind import RewindTargetUnavailableError
  612. with self._get_transcript_drain_lock():
  613. try:
  614. outcome = db.rewind_user_turn(
  615. session_id, -max(n, 1), require_retryable=require_retryable_composite,
  616. require_composite=require_retryable_composite)
  617. except RewindTargetUnavailableError as e:
  618. logger.debug("rewind_session: %s", e)
  619. return None
  620. except ValueError:
  621. raise
  622. except Exception as e:
  623. logger.debug("rewind_session: rewind failed: %s", e)
  624. return None
  625. self._clear_dirty_transcript(session_id)
  626. return {"rewound_count": outcome.rewound_count, "turns_undone": outcome.turns_undone,
  627. "target_text": outcome.live_text}