session_persistence.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607
  1. """Durable transcript persistence for ``AIAgent`` (mixin; MRO-resolved from ``run_agent``): SQLite flush
  2. with intrinsic ``_DB_PERSISTED_MARKER`` dedup, ephemeral-scaffolding filtering, explicit
  3. trajectory export."""
  4. import hashlib
  5. import logging
  6. import re
  7. from contextlib import nullcontext
  8. from typing import Any, Dict, List, Optional, Tuple
  9. from agent.context_compressor import (
  10. COMPRESSED_SUMMARY_METADATA_KEY,
  11. _DB_PERSISTED_MARKER,
  12. ContextCompressor,
  13. _newest_checkpoint_carrier,
  14. _strip_persistence_markers,
  15. drop_shadowed_checkpoints,
  16. user_originated_turn_view,
  17. )
  18. from agent.lazy_forward import forward as _forward, forward_static as _forward_static
  19. from agent.memory_context import sanitize_context
  20. from agent.tool_dispatch_helpers import _is_multimodal_tool_result, _multimodal_text_summary
  21. from agent.trajectory import save_trajectory as _save_trajectory_to_file
  22. from agent.message_metadata import (
  23. DB_ROW_SNAPSHOT, MERGED_TURN_PREFIX, REPAIR_BOOKKEEPING_FIELDS, RESTORED_FROM_STORE, TOOL_CALL_UID,
  24. copy_identity_fields, message_uid_or_none, tool_call_uid_from_history)
  25. from agent.transcript_repair import sync_flushed_message_markers
  26. logger = logging.getLogger("run_agent") # origin module's name: log records / caplog filters unchanged
  27. # Flags marking ephemeral recovery scaffolding the loop pops before appending the real response.
  28. # Persistence must skip them or a resumed session replays synthetic turns / breaks prefix-cache reuse.
  29. _EPHEMERAL_SCAFFOLDING_FLAGS = (
  30. "_empty_recovery_synthetic",
  31. "_empty_terminal_sentinel",
  32. "_thinking_prefill",
  33. "_verification_stop_synthetic", # verify-on-stop nudge; the assistant candidate itself is NOT synthetic
  34. "_pre_verify_synthetic",
  35. "_kanban_stop_synthetic", # kanban worker stop-guard
  36. "_dropped_toolcall_nudge", # internal retry instruction; must not replay as user context
  37. )
  38. _IMAGE_PART_TYPES = {"image", "image_url", "input_image"}
  39. # Reasoning/codex fields are role-gated (assistant-only) inside _insert_message_rows.
  40. _ROW_REASONING_KEYS = ("reasoning", "reasoning_content", "reasoning_details", "codex_reasoning_items", "codex_message_items")
  41. _PERSIST_AFTER_ADMISSION_INTERRUPT = "_persist_after_admission_interrupt"
  42. def _is_ephemeral_scaffolding(msg: Any) -> bool:
  43. """True when ``msg`` is internal recovery scaffolding that must never reach the durable transcript."""
  44. return isinstance(msg, dict) and any(msg.get(flag) for flag in _EPHEMERAL_SCAFFOLDING_FLAGS)
  45. # `_DB_PERSISTED_MARKER` (agent.context_compressor) is the intrinsic "already written to SQLite" marker: an
  46. # id(msg) set can alias a freed dict's address onto a new message, a key on the dict cannot. The `_` prefix is
  47. # mandatory (wire sanitizers strip `_` keys). CONTRACT: the marker asserts the dict's CONTENT is durable as
  48. # written — any in-place mutation that must persist MUST pop it (turn_finalizer, context_compressor).
  49. def _safe_session_filename_component(session_id: str) -> str:
  50. """Path-safe component for a (possibly untrusted ``X-Hermes-Session-Id``) ID: non ``[A-Za-z0-9_-]`` → ``_``,
  51. capped, plus a content hash when changed so distinct IDs cannot collide."""
  52. raw = str(session_id or "").strip()
  53. sanitized = re.sub(r"[^\w-]", "_", raw).strip("._")[:96] or "session"
  54. if raw and sanitized == raw:
  55. return sanitized
  56. return f"{sanitized}_{hashlib.sha256(raw.encode('utf-8', errors='surrogatepass')).hexdigest()[:12]}"
  57. def _override_replaces_content(msg: dict, content: Any, override: Any) -> bool:
  58. """May the persist override replace ``content``? A plain-text override must not replace native image/audio
  59. blocks (a list override is the clean multimodal payload and does), nor a message MERGED with a compaction
  60. summary (overwriting would drop the summary)."""
  61. return (
  62. override is not None
  63. and not msg.get(COMPRESSED_SUMMARY_METADATA_KEY)
  64. and (not isinstance(content, list) or isinstance(override, list))
  65. )
  66. def _content_with_turn_override(msg: dict, content: Any, override: Any) -> Any:
  67. """Replace only the absorbed current turn, leaving unanswered history intact. The join matches
  68. ``_merge_consecutive_users``; a row that no longer holds the prefix takes the plain override."""
  69. prefix = msg.get(MERGED_TURN_PREFIX)
  70. if isinstance(content, str) and isinstance(override, str) and isinstance(prefix, str):
  71. if content == prefix or content.startswith(prefix + "\n\n"):
  72. return prefix + "\n\n" + override if override else prefix
  73. return override
  74. def durable_user_row_content(agent, msg: dict, content: Any, api_content: Any) -> tuple[Any, Any]:
  75. """``(content, api_content)`` as the current turn's user row is written: the persist override is the
  76. clean transcript, the live content is what the wire sent — so when they differ and nothing else was
  77. injected, the live bytes ARE the sidecar. Shared by the flush and the turn-start stamp so the stamp
  78. matches the row the flush wrote."""
  79. override = getattr(agent, "_persist_user_message_override", None)
  80. if _override_replaces_content(msg, content, override):
  81. if api_content is None and isinstance(content, str) and content != override:
  82. api_content = content
  83. content = _content_with_turn_override(msg, content, override)
  84. return content, api_content
  85. def _summary_display_kind(msg: dict) -> Any:
  86. """Standalone handoffs are hidden so they never occupy the active user slot in retry/undo dispatch;
  87. merge-into-tail carriers keep their prior visibility."""
  88. if (
  89. msg.get(COMPRESSED_SUMMARY_METADATA_KEY)
  90. and user_originated_turn_view(msg) is None
  91. and (
  92. ContextCompressor.classify_summary_content(msg.get("content")) == "standalone"
  93. or not msg.get("_compressed_summary_has_user_turn")
  94. )
  95. ):
  96. return "hidden"
  97. return msg.get("display_kind")
  98. def _durable_content(content: Any) -> Any:
  99. """Text-only DB projection: multimodal envelopes → summary; part lists keep text, images → ``[screenshot]``."""
  100. if _is_multimodal_tool_result(content):
  101. return _multimodal_text_summary(content)
  102. if not isinstance(content, list):
  103. return content
  104. txt = [
  105. str(p.get("text", "")) if p.get("type") == "text" else _image_marker(p)
  106. for p in content
  107. if isinstance(p, dict) and (p.get("type") == "text" or p.get("type") in _IMAGE_PART_TYPES)
  108. ]
  109. return "\n".join(txt) if txt else None
  110. def _image_marker(part: dict) -> str:
  111. """``[screenshot]`` at the image's position; a caller-chosen ``detail`` rides along so a replay
  112. (``agent.image_replay``) rebuilds the part the live turn sent."""
  113. url = part.get("image_url")
  114. detail = part.get("detail") or (url.get("detail") if isinstance(url, dict) else None)
  115. return f"[screenshot detail={detail}]" if detail in ("low", "high", "auto") else "[screenshot]"
  116. def _persist_lock(agent):
  117. """Close and turn-start persistence can run on separate CLI threads: one critical section.
  118. ``__init__`` always creates ``_session_persist_lock``; only ``object.__new__``-built test stubs lack it
  119. (they run unlocked, matching the historical ``if persist_lock is None`` branch).
  120. """
  121. lock = getattr(agent, "_session_persist_lock", None)
  122. return nullcontext() if lock is None else lock
  123. def adopt_unanswered_turn(history: list[dict[str, Any]], query: Any, agent: Any) -> bool:
  124. """Re-stage the transcript's unanswered tail row as THIS turn's user message; True when adopted.
  125. A dispatcher's re-run of a failed delivery turn resumes the DM its first attempt already persisted
  126. instead of appending it again. Rows loaded from the store are born durable (``_rows_to_conversation``),
  127. so handing the tail row back as ``agent._pending_cli_user_message`` makes ``_stage_turn_user_message``
  128. reuse it as this turn's user dict and the flush writes no second row. The one user is the peer-DM
  129. lane's re-run (``resume_unanswered_turn`` in ``gateway.platforms.api_server``, #115325): in process it
  130. calls this on the agent it just built for the re-run; under the session authority the admission
  131. carries ``resume_unanswered`` and ``gateway.run_turn_runner`` calls it on the admitted history
  132. when its tail is that bare user row (a tool suffix there keeps the completed work in view).
  133. The owner-side Bot Chat retry (``gateway.session_bot._maybe_retry``) does not adopt: its failed
  134. turn is already closed by the durable failed-turn boundary, which this function declines by design.
  135. The DM is not always the literal tail: a turn that died mid-way persisted its tool scaffolding — assistant
  136. ``tool_calls`` rows and their ``tool`` results — behind the DM before the failure text was built, and the
  137. dispatcher retries that too. The DM is still unanswered while nothing after it is a plain assistant reply,
  138. so it is adopted and the failed attempt's scaffolding leaves the in-memory transcript: the re-run starts
  139. the turn over from the DM (the rows stay in the DB as the record of the failed attempt; the re-run's
  140. answer lands after them as a valid continuation). Anything else declines — no user row at the tail, or a
  141. different text there — so a person's deliberate re-send of the same text is never swallowed.
  142. """
  143. idx = next((i for i in range(len(history) - 1, -1, -1)
  144. if isinstance(history[i], dict) and history[i].get("role") == "user"), None)
  145. if idx is None or history[idx].get("content") != query:
  146. return False
  147. if not all(isinstance(row, dict) and (row.get("role") == "tool" or (row.get("role") == "assistant" and row.get("tool_calls")))
  148. for row in history[idx + 1:]):
  149. return False
  150. tail = history[idx]
  151. del history[idx:]
  152. tail[_DB_PERSISTED_MARKER] = True
  153. agent._pending_cli_user_message = tail
  154. return True
  155. # --- flush phases (module-level so the flush also works bound onto duck-typed agents) ---
  156. def _db_flush_seed_ids(agent) -> set:
  157. """One-shot ``_flushed_db_message_ids`` seed (same session, after a non-empty flush); the scan translates
  158. it to markers and the flush clears it."""
  159. current_session_id = getattr(agent, "session_id", None)
  160. same_session = getattr(agent, "_flushed_db_message_session_id", None) == current_session_id
  161. seed_ids = getattr(agent, "_flushed_db_message_ids", None) if same_session and agent._last_flushed_db_idx != 0 else None
  162. agent._flushed_db_message_session_id = current_session_id
  163. return seed_ids if isinstance(seed_ids, set) else set()
  164. def _db_flush_store_uids(agent) -> set:
  165. """``message_uid``s this agent saw durable in its CURRENT session (flushed by it, or scanned as already-durable
  166. history): the provenance that lets the write path skip a marker-stripped copy of one of them (#134799)."""
  167. session_id = getattr(agent, "session_id", None)
  168. scope = getattr(agent, "_db_store_uid_scope", None)
  169. if not (isinstance(scope, tuple) and scope[0] == session_id):
  170. scope = agent._db_store_uid_scope = (session_id, set())
  171. return scope[1]
  172. def _db_flush_scan_start(agent, messages: list[dict]) -> int:
  173. """Skip the identity-matched, still-marked prefix of the previous flush's snapshot."""
  174. scan_start = 0
  175. for prev, cur in zip(getattr(agent, "_db_flush_scan_prefix", None) or (), messages):
  176. if cur is not prev or not cur.get(_DB_PERSISTED_MARKER):
  177. break
  178. scan_start += 1
  179. return scan_start
  180. def _db_flush_row(agent, msg: dict, is_current_turn_user: bool) -> dict[str, Any]:
  181. """Build the session-db row for ``msg``, applying the persist override to THIS row only."""
  182. role = msg.get("role", "unknown")
  183. content = msg.get("content")
  184. # api_content sidecar: exact bytes sent to the API when they differ from clean content (replay parity).
  185. api_content = msg.get("api_content") if isinstance(msg.get("api_content"), str) else None
  186. timestamp = msg.get("timestamp")
  187. if is_current_turn_user and role == "user":
  188. content, api_content = durable_user_row_content(agent, msg, content, api_content)
  189. ov_timestamp = getattr(agent, "_persist_user_message_timestamp", None)
  190. timestamp = timestamp if ov_timestamp is None else ov_timestamp
  191. if api_content == content:
  192. api_content = None
  193. # get_messages_as_conversation replays rows through sanitize_context().strip(); capture the sent bytes
  194. # when they would differ (compared in wire form).
  195. if (
  196. api_content is None and role in ("user", "assistant") and isinstance(content, str) and content
  197. and sanitize_context(content).strip() != content.strip()
  198. ):
  199. api_content = content
  200. # Key order is the divert-JSONL wire order (divert_session_transcript_jsonl).
  201. row = {
  202. "role": role, "content": _durable_content(content),
  203. "tool_name": msg.get("tool_name") or (msg.get("name") if role == "tool" else None),
  204. "tool_calls": msg["tool_calls"] if isinstance(msg.get("tool_calls"), list) else None,
  205. "tool_call_id": msg.get("tool_call_id"), "effect_disposition": msg.get("effect_disposition"),
  206. "token_count": msg.get("token_count"), "finish_reason": msg.get("finish_reason"),
  207. **{k: msg.get(k) for k in _ROW_REASONING_KEYS},
  208. "_compressed_summary": bool(msg.get(COMPRESSED_SUMMARY_METADATA_KEY)),
  209. "timestamp": timestamp, "api_content": api_content,
  210. "display_kind": _summary_display_kind(msg), "display_metadata": msg.get("display_metadata"),
  211. # Load-bearing for restart drain-window recovery dedup.
  212. "platform_message_id": msg.get("platform_message_id") or msg.get("message_id"),
  213. "observed": bool(msg.get("observed")),
  214. }
  215. if isinstance(msg.get("_row_id"), int):
  216. row["_row_id"] = msg["_row_id"]
  217. # The merge witness rides on the survivor's row (an owned column: a row-addressed rewrite of the
  218. # survivor writes it too), so a restart sees which rows the composite folded.
  219. copy_identity_fields(msg, row)
  220. if isinstance(msg.get(DB_ROW_SNAPSHOT), str):
  221. row[DB_ROW_SNAPSHOT] = msg[DB_ROW_SNAPSHOT]
  222. return row
  223. def sync_rewritten_tool_rows(agent, batch: list[dict], contents_before: list[Any]) -> int:
  224. """Carry an end-of-batch rewrite of already-flushed tool results (the turn budget's spill
  225. stubs) onto their durable rows; returns the rows rewritten.
  226. Each result was flushed as it landed, so its row holds the full body. Replay reads the
  227. store, so a row left stale reloads the spilled bytes next turn: the request grows by
  228. everything the budget removed and its cached prefix diverges at the first spilled row.
  229. The full body survives as the archived twin ``rewrite_pruned_rows`` keeps, beside the
  230. spill file. A store without the in-place writer, or a row another writer moved, keeps
  231. today's row.
  232. """
  233. session_db, session_id = getattr(agent, "_session_db", None), getattr(agent, "session_id", None)
  234. rewrite = getattr(session_db, "rewrite_pruned_rows", None)
  235. if getattr(agent, "_persist_disabled", False) or not session_id or not callable(rewrite):
  236. return 0
  237. changed = [
  238. (msg, before) for msg, before in zip(batch, contents_before)
  239. if msg.get(_DB_PERSISTED_MARKER) and msg.get("role") == "tool" and msg.get("content") != before]
  240. if not changed:
  241. return 0
  242. rows = [(_db_flush_row(agent, {**msg, "content": before}, False), _db_flush_row(agent, msg, False))
  243. for msg, before in changed]
  244. from hermes_state_errors import PruneRowUnresolvedError, SessionTurnLeaseLostError
  245. try:
  246. with _persist_lock(agent):
  247. written = rewrite(
  248. session_id, rows, turn_lease_holder=getattr(agent, "_active_session_turn_lease_holder", None),
  249. turn_lease_ttl_seconds=getattr(agent, "_active_session_turn_lease_ttl_seconds", 300.0) or 300.0,
  250. )
  251. except (PruneRowUnresolvedError, SessionTurnLeaseLostError) as exc: # another writer owns the row or turn
  252. logger.info("Tool-result rewrite not persisted; the stored row keeps the full body: %s", exc)
  253. return 0
  254. except Exception as exc: # health: allow BLE001 -- a store failure must not fail the turn; the row keeps its body
  255. logger.warning("Tool-result rewrite not persisted: %s", exc)
  256. return 0
  257. for (msg, _before), (_original, replacement) in zip(changed, rows):
  258. if isinstance(replacement.get(DB_ROW_SNAPSHOT), str):
  259. msg[DB_ROW_SNAPSHOT] = replacement[DB_ROW_SNAPSHOT]
  260. return int(written)
  261. def _db_flush_collect(agent, messages: list[dict], conversation_history: Optional[list[dict]],
  262. replay_history: bool = False):
  263. """Scan for un-flushed messages; returns ``(rows, msgs)`` to write in one transaction. ``replay_history``
  264. (session-row heal) writes the history prefix again instead of stamping it as already durable."""
  265. seed_ids = _db_flush_seed_ids(agent)
  266. history_ids = {id(item) for item in (conversation_history or []) if isinstance(item, dict)}
  267. ov_idx = getattr(agent, "_persist_user_message_idx", None)
  268. # Also match the staged CLI dict by identity — the close safety-net may flush a shortened snapshot whose
  269. # turn index refers to the full history.
  270. pending_cli_message = getattr(agent, "_pending_cli_user_message", None)
  271. batch_rows: list[dict[str, Any]] = []
  272. batch_msgs: list[dict] = []
  273. tool_uid_owners: dict = {} # tool_call_uid_from_history memo; the scanned dicts outlive this loop
  274. # Durable-uid re-insert guard: a row whose uid this agent already saw durable is marked RESTORED_FROM_STORE;
  275. # resolve_and_repair_transcript_batch skips it only if that uid is still ACTIVE, inside the write
  276. # transaction, and never for a snapshot-carrying dict (the CAS rewrite owns its live edit).
  277. store_uids = _db_flush_store_uids(agent)
  278. for msg_idx in range(_db_flush_scan_start(agent, messages), len(messages)):
  279. msg = messages[msg_idx]
  280. # Append-only flush: a mid-turn persist of scaffolding would commit a synthetic turn the end-of-turn
  281. # drop cannot un-write. Skip regardless of position.
  282. if not isinstance(msg, dict) or _is_ephemeral_scaffolding(msg):
  283. continue
  284. uid = message_uid_or_none(msg)
  285. if msg.get(_DB_PERSISTED_MARKER):
  286. if uid is not None:
  287. store_uids.add(uid)
  288. continue
  289. # Already durable (history copy or caller-seeded): stamp so future flushes skip it.
  290. is_history = id(msg) in history_ids
  291. if (
  292. (is_history and not replay_history) or id(msg) in seed_ids
  293. ) and not msg.get(_PERSIST_AFTER_ADMISSION_INTERRUPT):
  294. msg[_DB_PERSISTED_MARKER] = True
  295. if uid is not None:
  296. store_uids.add(uid)
  297. continue
  298. if getattr(agent, "_mute_notification_reply", False) and not is_history:
  299. # Only new rows, never the cached history prefix. Keep evidence/model
  300. # context intact while transcript pollers omit unsolicited presentation.
  301. msg["display_kind"] = "hidden"
  302. msg["display_metadata"] = {**(msg.get("display_metadata") or {}), "notification_category": "diagnostic"}
  303. if msg.get("role") == "tool" and not msg.get(TOOL_CALL_UID):
  304. # A result whose call was flushed in an earlier batch: pair it with the uid that assistant row
  305. # minted (same-batch pairing happens inside the insert).
  306. if (tool_uid := tool_call_uid_from_history(messages, msg_idx, tool_uid_owners)) is not None:
  307. msg[TOOL_CALL_UID] = tool_uid
  308. row = _db_flush_row(agent, msg, ov_idx == msg_idx or msg is pending_cli_message)
  309. if uid is not None and uid in store_uids:
  310. row[RESTORED_FROM_STORE] = True
  311. batch_rows.append(row)
  312. batch_msgs.append(msg)
  313. return batch_rows, batch_msgs
  314. def _db_flush_write(agent, batch_rows: list[dict[str, Any]], batch_msgs: list[dict], messages: list[dict]) -> None:
  315. """One transaction for the turn's new rows: on failure nothing lands and no markers are stamped."""
  316. if not batch_rows:
  317. return
  318. agent._session_db.append_messages_batch(
  319. session_id=agent.session_id, messages=batch_rows,
  320. compression_lock_holder=getattr(agent, "_active_compression_lock_holder", None),
  321. turn_lease_holder=getattr(agent, "_active_session_turn_lease_holder", None),
  322. turn_lease_ttl_seconds=getattr(agent, "_active_session_turn_lease_ttl_seconds", 300.0) or 300.0,
  323. )
  324. sync_flushed_message_markers(batch_msgs, batch_rows)
  325. _db_flush_store_uids(agent).update(uid for msg in batch_msgs if (uid := message_uid_or_none(msg)) is not None)
  326. if _newest_checkpoint_carrier(batch_msgs, "codex_reasoning_items") >= 0:
  327. # The insert already rewrote the older rows (SessionDB._drop_shadowed_checkpoint_rows); mirror it on
  328. # the live transcript so forks/compaction built from memory carry one checkpoint too. Markers stay:
  329. # the rows are durable exactly as the dicts now read.
  330. drop_shadowed_checkpoints(messages)
  331. def _db_flush_adopt_compression_tip(agent) -> bool:
  332. """Adopt the live continuation of a compression-closed session. Same-id tip = no continuation; a tip
  333. whose row is missing or already ended is not adopted either."""
  334. old_id = agent.session_id
  335. try:
  336. tip = agent._session_db.get_compression_tip(old_id)
  337. except Exception as tip_exc:
  338. logger.warning("compression tip lookup failed for %s: %s", old_id, tip_exc)
  339. return False
  340. if not tip or tip == old_id:
  341. return False
  342. try:
  343. tip_row = agent._session_db.get_session(tip)
  344. except Exception:
  345. tip_row = None
  346. if tip_row is None or tip_row.get("ended_at") is not None:
  347. return False
  348. logger.warning("Adopted live compression tip %s for closed session %s; retrying flush once", tip, old_id)
  349. agent.session_id, agent._flushed_db_message_ids, agent._last_flushed_db_idx = tip, set(), 0
  350. agent._compression_adoption_failed = False
  351. return True
  352. def _db_flush_session_row_gone(agent, session_id: Optional[str]) -> bool:
  353. """True only when ``session_id``'s row is confirmed absent; a failed lookup is not proof."""
  354. try:
  355. return agent._session_db.get_session(session_id) is None
  356. except Exception as lookup_exc:
  357. logger.warning("session row lookup failed for %s: %s", session_id, lookup_exc)
  358. return False
  359. def _db_flush_failed(agent, e: Exception, batch_rows: list[dict[str, Any]], adoption_budget: int,
  360. messages: list[dict]) -> Optional[str]:
  361. """Classify a failed flush and name the one retry the caller should take, or None to fail closed.
  362. ``"adopted"``: a compression-closed session moved onto its live tip. ``"healed"``: the session row was
  363. deleted under the live agent and has been recreated (unparented for a delegate child whose parent went
  364. with it); flush markers are reset so the caller replays the full in-memory transcript. Either retry is
  365. taken at most once (``adoption_budget``)."""
  366. agent._db_flush_scan_prefix = None # full re-scan next flush: an exception mid-loop leaves mixed dispositions
  367. # The only place the SQLite error is visible before it becomes a bare False — classify it so the turn-end
  368. # explanation names the real cause.
  369. from hermes_state import StateDbCorruptError, StateDbReplacedError, classify_persistence_error, divert_session_transcript_jsonl
  370. from hermes_state_errors import CompressionSessionClosedError
  371. agent._last_persistence_error_cause = classify_persistence_error(e)
  372. if agent._last_persistence_error_cause == "session_row_missing":
  373. # The session row was removed under this live agent (`hermes sessions delete`, the Desktop/web
  374. # delete, bulk prune, a profile-repair move, an in-place store rebuild — none visible to the
  375. # cached agent, so the cached `_session_db_created` flag is stale and every later append hits
  376. # the FK). The deletion already erased the session's message rows with it, so the durable
  377. # transcript is empty: drop the stale flag, reset the flush markers, and replay the FULL
  378. # in-memory transcript onto the recreated row — not just the current tail (#123583).
  379. # The FK class also covers the sessions table's own parent/system-prompt FKs, and create is an
  380. # upsert: only heal when the row is really gone, or the replay would duplicate a live transcript.
  381. if adoption_budget <= 0 or not _db_flush_session_row_gone(agent, agent.session_id):
  382. return None
  383. _strip_persistence_markers(messages)
  384. # The durable history prefix is gone with the row: keep a replay pending until a write succeeds, so
  385. # a failed recreate or a failed retry write can't let a later flush stamp that prefix durable.
  386. agent._session_row_replay_pending = agent.session_id
  387. agent._flushed_db_message_ids = set()
  388. agent._last_flushed_db_idx = 0
  389. agent._session_db_created = False
  390. agent._ensure_db_session()
  391. parent_id = agent._parent_session_id
  392. if not agent._session_db_created and parent_id and _db_flush_session_row_gone(agent, parent_id):
  393. # Delegate child whose parent was deleted (cascade): the row's own parent FK would reject every
  394. # recreate. Create it unparented for this call only; the relay and hooks still use the parent id.
  395. agent._parent_session_id = None
  396. try:
  397. agent._ensure_db_session()
  398. finally:
  399. agent._parent_session_id = parent_id
  400. if not agent._session_db_created:
  401. # Row creation failed too (transient store trouble): don't append into a guaranteed
  402. # rollback — keep the batch unmarked so the next flush retries the whole thing. That flush
  403. # recreates the row up front (no FK error, no heal); the pending replay covers the history prefix.
  404. logger.warning("Session DB row for %s is missing and could not be recreated; will retry next flush",
  405. getattr(agent, "session_id", None))
  406. return None
  407. logger.warning("Session DB row for %s was removed under the live agent; recreated it and replaying the transcript",
  408. getattr(agent, "session_id", None))
  409. return "healed"
  410. if isinstance(e, (StateDbReplacedError, StateDbCorruptError)):
  411. # A replaced/quarantined handle will not take this batch again — keep it on disk.
  412. try:
  413. # The CAS digest / adopted row are local repair bookkeeping, not transcript payload.
  414. divert_session_transcript_jsonl(getattr(agent, "session_id", "") or "",
  415. [{k: v for k, v in r.items() if k not in REPAIR_BOOKKEEPING_FIELDS}
  416. for r in batch_rows])
  417. except Exception:
  418. logger.warning("JSONL divert failed after state.db %s for %s",
  419. agent._last_persistence_error_cause, getattr(agent, "session_id", None), exc_info=True)
  420. if isinstance(e, CompressionSessionClosedError):
  421. # Compression race: another path rotated this session mid-write. Retry exactly once on the live tip; a
  422. # second closed-parent write fails closed.
  423. if adoption_budget > 0 and _db_flush_adopt_compression_tip(agent):
  424. return "adopted"
  425. agent._compression_adoption_failed = True # lets the turn explanation name rotation, not full-disk advice
  426. logger.warning("Session DB append_message failed: %s", e)
  427. return None
  428. class SessionPersistenceMixin:
  429. """Session DB flush and trajectory persistence (see module docstring)."""
  430. def _apply_persist_user_message_override(self, messages: list[dict]) -> None:
  431. """Rewrite the current-turn user message in place: some paths send an API-only variant that must not
  432. leak into transcripts or resumed history."""
  433. idx = getattr(self, "_persist_user_message_idx", None)
  434. override = getattr(self, "_persist_user_message_override", None)
  435. timestamp = getattr(self, "_persist_user_message_timestamp", None)
  436. platform_id = getattr(self, "_persist_user_message_platform_id", None)
  437. if idx is None or (override is None and timestamp is None and platform_id is None):
  438. return
  439. msg = messages[idx] if 0 <= idx < len(messages) else None
  440. if not (isinstance(msg, dict) and msg.get("role") == "user"):
  441. return
  442. if _override_replaces_content(msg, msg.get("content"), override):
  443. msg["content"] = _content_with_turn_override(msg, msg.get("content"), override)
  444. if timestamp is not None:
  445. msg["timestamp"] = timestamp
  446. if platform_id is not None: # load-bearing for restart drain-window recovery dedup (has_platform_message_id)
  447. msg["platform_message_id"] = platform_id
  448. def _persist_session(self, messages: list[dict], conversation_history: list[dict] | None = None):
  449. """Save to SQLite on any exit path. Trailing empty-response scaffolding is dropped from
  450. the live list; the persist override is applied to the DB row only.
  451. The persist user-message *override* is NOT applied here — it is resolved inside
  452. ``_flush_messages_to_session_db`` and written only to the DB row, never mutating the live message
  453. list used by the API call (#48677 is thus closed for every persist caller, not just this one).
  454. """
  455. from agent.agent_runtime_helpers import note_turn_persisted
  456. with _persist_lock(self):
  457. # Only the scaffolding goes here. Closing a tool tail this uncovers is the exit
  458. # owner's job (``_close_transcript_tail``, ``abort_turn_on_interrupt``): only it knows
  459. # the reason to record.
  460. self._drop_trailing_empty_response_scaffolding(messages)
  461. self._session_messages = messages
  462. self._flush_messages_to_session_db(messages, conversation_history)
  463. # Drain async token-accounting deltas at every persist point; cheap no-op when nothing queued.
  464. if self._session_db is not None:
  465. self._session_db.flush_token_counts()
  466. note_turn_persisted(self)
  467. def _drop_trailing_empty_response_scaffolding(self, messages: list[dict]) -> None:
  468. """Pop empty-response retry scaffolding from the tail. The
  469. assistant(tool_calls) / tool rows before it stay: they were saved before the tools ran, so
  470. dropping them from the live history only makes the model repeat a side effect the durable
  471. transcript already records."""
  472. def tail(*keys: str) -> bool:
  473. return bool(messages) and isinstance(messages[-1], dict) and any(messages[-1].get(k) for k in keys)
  474. while tail("_empty_recovery_synthetic", "_empty_terminal_sentinel"):
  475. messages.pop()
  476. _repair_message_sequence = _forward("agent.agent_runtime_helpers", "repair_message_sequence")
  477. def _flush_messages_to_session_db(self, messages: list[dict], conversation_history: Optional[list[dict]] = None):
  478. """Serialize direct and turn-boundary session flushes per agent."""
  479. with _persist_lock(self):
  480. return self._flush_messages_to_session_db_unlocked(messages, conversation_history)
  481. def _flush_messages_to_session_db_unlocked(
  482. self, messages: list[dict], conversation_history: Optional[list[dict]] = None, _adoption_budget: int = 1,
  483. ):
  484. """Persist un-flushed messages to SQLite. Dedup is the intrinsic ``_DB_PERSISTED_MARKER`` on each written
  485. dict — not positional slices (drift after sequence repair) nor an ``id(msg)`` set (address reuse). The
  486. persist override touches the written row only. A compression-closed session adopts its live tip and
  487. retries exactly once.
  488. Deduplicates via an intrinsic ``_DB_PERSISTED_MARKER`` stamped on each written message dict, so
  489. repeated calls (from multiple exit paths) only write truly new messages — preventing the
  490. duplicate-write bug (#860) without relying on positional slices that can drift after
  491. message-sequence repair, and without a retained ``id(msg)`` set that CPython could alias onto a
  492. freed-then-reused address (#50372). The ``_flushed_db_message_ids`` attribute is now only a one-shot
  493. seed (translated to markers, then cleared each flush), not a persisted set.
  494. """
  495. # Persistence-isolated agents (background review fork) share the parent's session_id for cache warmth;
  496. # a write here would land the curator's turn in the user's real history.
  497. if getattr(self, "_persist_disabled", False) or not self._session_db:
  498. return None
  499. batch_rows: list[dict[str, Any]] = []
  500. try:
  501. if not self._session_db_created: # retry row creation if the earlier attempt failed transiently
  502. self._ensure_db_session()
  503. # getattr: object.__new__ test agents flush without running AIAgent init.
  504. replay = getattr(self, "_session_row_replay_pending", None) == self.session_id
  505. batch_rows, batch_msgs = _db_flush_collect(self, messages, conversation_history, replay)
  506. _db_flush_write(self, batch_rows, batch_msgs, messages)
  507. self._session_row_replay_pending = None
  508. # Markers are now the sole truth; reset the one-shot seed so no id() outlives this flush.
  509. self._flushed_db_message_ids = set()
  510. self._last_flushed_db_idx = len(messages)
  511. # Snapshot for the bounded scan — only on full success, so a partial list is never treated as settled.
  512. self._db_flush_scan_prefix = messages[:]
  513. return True
  514. except Exception as e:
  515. retry = _db_flush_failed(self, e, batch_rows, _adoption_budget, messages)
  516. if retry is None:
  517. return False
  518. # After a heal the pending replay re-sends the history prefix instead of stamping it durable.
  519. return self._flush_messages_to_session_db_unlocked(messages, conversation_history, _adoption_budget=0)
  520. def _get_messages_up_to_last_assistant(self, messages: list[dict]) -> list[dict]:
  521. """Messages before the last assistant turn (rollback point for a malformed final answer); all if none."""
  522. for i in range(len(messages) - 1, -1, -1):
  523. if messages[i].get("role") == "assistant":
  524. return messages[:i]
  525. return messages.copy()
  526. _format_tools_for_system_message = _forward("agent.system_prompt", "format_tools_for_system_message")
  527. _convert_to_trajectory_format = _forward("agent.agent_runtime_helpers", "convert_to_trajectory_format")
  528. def _save_trajectory(self, messages: list[dict[str, Any]], user_query: str, completed: bool):
  529. """Save conversation trajectory to JSONL file."""
  530. if not self.save_trajectories:
  531. return
  532. trajectory = self._convert_to_trajectory_format(messages, user_query, completed)
  533. _save_trajectory_to_file(trajectory, self.model, completed)
  534. _extract_api_error_context = _forward_static("agent.agent_runtime_helpers", "extract_api_error_context")
  535. _dump_api_request_debug = _forward("agent.agent_runtime_helpers", "dump_api_request_debug")