transcript_repair.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. """Transcript repair for SessionDB batch appends: reconcile in-memory rows with committed SQLite rows
  2. (in-place sanitizer rewrites, assistant blank-row repair, concurrent-winner adoption, watermark-compaction
  3. clone lookup) and sync markers after commit."""
  4. from __future__ import annotations
  5. import hashlib
  6. import sqlite3
  7. from typing import Any, Callable, Dict, List, Mapping
  8. from agent.context_compressor import _DB_PERSISTED_MARKER
  9. from agent.message_metadata import (
  10. ABSORBED_MESSAGE_UIDS, CANONICAL_ROW, DB_ROW_SNAPSHOT, MESSAGE_UID, RESTORED_FROM_STORE, TOOL_CALL_UIDS,
  11. copy_identity_fields, message_uid_or_none)
  12. from hermes_state_common import _id_chunks, _placeholders
  13. from hermes_state_identity import _fill_missing_tool_call_uids, _restore_row_identity
  14. from hermes_state_messages import _MESSAGE_WRITE_COLUMNS, SessionMessagesMixin
  15. # Durable payload columns a row-addressed rewrite may change: every INSERT column except row identity,
  16. # role, active flag and the ones owned by the display index / timestamp / session linkage. ``message_uid``
  17. # and a result row's ``tool_call_uid`` are identity too: a rewrite changes the message's content, never
  18. # which logical message (or which call occurrence) the row is. ``tool_call_uids`` IS payload: it follows
  19. # ``tool_calls`` (an assistant merge unions both), so it is rewritten with the row.
  20. _NON_PAYLOAD_COLUMNS = frozenset(
  21. {"session_id", "role", "timestamp", "active", "display_identity", MESSAGE_UID, "tool_call_uid"})
  22. _REPAIR_COLUMNS = tuple(c for c in _MESSAGE_WRITE_COLUMNS if c not in _NON_PAYLOAD_COLUMNS)
  23. # Columns same-process writers update after our flush (reactions / display-kind stamps, api_content
  24. # backfill, codex reasoning backfill + checkpoint pruning, platform message ids). They are not part of the
  25. # ownership version: a metadata write must never make our own row look like a foreign winner.
  26. _METADATA_COLUMNS = frozenset(
  27. {"display_kind", "display_metadata", "api_content", "codex_reasoning_items", "platform_message_id"}
  28. )
  29. # The ownership version: what we last committed for the payload the sanitizer / live edits change.
  30. _OWNED_COLUMNS = tuple(c for c in _REPAIR_COLUMNS if c not in _METADATA_COLUMNS)
  31. # What a replay of the row sends: the owned payload plus the two metadata columns the provider request reads back
  32. # (``api_content`` replaces the content bytes, ``codex_reasoning_items`` is replayed reasoning). A copy that equals
  33. # the row on these carries no new information; a metadata write stays outside the ownership version above.
  34. _REPLAY_COLUMNS = _OWNED_COLUMNS + ("api_content", "codex_reasoning_items")
  35. # The replayed reasoning carriers among them; a compaction claim checks these plus ``api_content`` (it compares
  36. # the content itself, and the compressor may rewrite the rest of a carry's columns).
  37. REPLAY_SIDECARS = ("reasoning", "reasoning_content", "reasoning_details", "codex_reasoning_items", "codex_message_items")
  38. # Presentation-only metadata a matched (ours) row may hand to a live dict that lacks it.
  39. _LIVE_MISSING_METADATA = ("display_kind", "display_metadata")
  40. # ``message_id`` is identity the flush derived from the live dict (int there, TEXT in SQLite): never synced.
  41. _SYNC_FIELDS = ("role",) + _REPAIR_COLUMNS
  42. # Canonical-row markers: a matched row hands over only missing presentation metadata; a legacy (no-digest)
  43. # dict over a filled assistant row adopts only its content.
  44. _METADATA_ONLY = "_metadata_only"
  45. _CONTENT_ONLY = "_content_only"
  46. def transcript_row_snapshot(row: Mapping[str, Any]) -> str:
  47. """Fixed-size digest of the owned (non-metadata) columns (the CAS version) of a ``SELECT *`` messages row.
  48. Callers pass rows read back from SQLite, never Python bind values: column affinity rewrites values on
  49. storage (int ``platform_message_id`` -> TEXT, float ``token_count`` -> INTEGER), so hashing bind values
  50. would never match the stored version. A digest rather than a row copy: the value rides on live message
  51. dicts, so a full copy would double transcript memory and anything that prices dict bytes.
  52. """
  53. digest = hashlib.blake2b(digest_size=16)
  54. for column in _OWNED_COLUMNS:
  55. value = row[column]
  56. if value is None:
  57. digest.update(b"N")
  58. continue
  59. if isinstance(value, (bytes, bytearray, memoryview)):
  60. tag, data = b"B", bytes(value)
  61. elif isinstance(value, str):
  62. tag, data = b"S", value.encode("utf-8", "surrogatepass")
  63. else:
  64. tag, data = (b"F" if isinstance(value, float) else b"I"), repr(value).encode("ascii")
  65. digest.update(tag + len(data).to_bytes(8, "big") + data)
  66. return digest.hexdigest()
  67. def stamp_inserted_row_snapshots(conn: sqlite3.Connection, session_id: str, messages: list[dict[str, Any]]) -> None:
  68. """Stamp the stored-row digest on freshly inserted dicts (every ``_insert_message_rows`` caller; one SELECT
  69. per batch)."""
  70. by_id = {msg["_row_id"]: msg for msg in messages if isinstance(msg.get("_row_id"), int)}
  71. for chunk in _id_chunks(by_id):
  72. for row in conn.execute(
  73. f"SELECT * FROM messages WHERE session_id = ? AND id IN ({_placeholders(chunk)})", (session_id, *chunk)
  74. ).fetchall():
  75. by_id[int(row["id"])][DB_ROW_SNAPSHOT] = transcript_row_snapshot(row)
  76. def is_content_blank(content: Any) -> bool:
  77. """True when decoded message content is None, whitespace-only, or has no visible text parts."""
  78. if content is None:
  79. return True
  80. if isinstance(content, str):
  81. return not content.strip()
  82. if isinstance(content, list):
  83. return not "".join(p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text").strip()
  84. return False
  85. def _active_logical_message_row(
  86. conn: sqlite3.Connection, session_id: str, role: str, message_uid: str | None,
  87. ) -> Mapping[str, Any] | None:
  88. """Newest active physical row for one durable logical message.
  89. A ``message_uid`` names a logical message, not a physical row: compaction/copy paths deliberately
  90. keep it while re-issuing row ids. The later active row is the current version. Callers use this only
  91. when the live dict also carries a stored-row snapshot, so a fresh message that merely resembles an
  92. older one can never be adopted here.
  93. """
  94. if message_uid is None:
  95. return None
  96. return conn.execute(
  97. "SELECT * FROM messages WHERE session_id = ? AND active = 1 AND role = ? AND message_uid = ? "
  98. "ORDER BY id DESC LIMIT 1",
  99. (session_id, role, message_uid),
  100. ).fetchone()
  101. def _active_uid_dedupe(
  102. conn: sqlite3.Connection, session_id: str, messages: list[dict[str, Any]]
  103. ) -> dict[str, list[sqlite3.Row]]:
  104. """ACTIVE durable rows by uid among *messages*' restored, no-snapshot, no-``_row_id`` rows — the re-insert guard.
  105. Only a row the agent flush marked ``RESTORED_FROM_STORE`` (it saw that uid durable in this session) is a
  106. candidate: a copied uid alone is never proof, so a fresh identical message always inserts. A dict rebuilt
  107. from durable rows (crash/failed-turn history restore, session resume, a repaired sequence, any
  108. wire-crossing handoff) carries its durable ``message_uid`` — identity is restored on
  109. EVERY projection — but neither the persistence marker (underscore-prefixed, stripped before the
  110. wire) nor ``_row_id``/``DB_ROW_SNAPSHOT`` (same strip). Dicts that still carry a snapshot are
  111. excluded: the CAS path below owns them — it rewrites the active row in place, so their live edits
  112. must reach the row, never be skipped here. Runs INSIDE the caller's write transaction (the same
  113. guarantee the repair path relies on: the active set cannot change between this read and the
  114. INSERT). A probe error fails the write like any other statement in it.
  115. """
  116. candidate_uids = sorted({
  117. uid
  118. for msg in messages
  119. if isinstance(msg, dict)
  120. and msg.get(RESTORED_FROM_STORE) is True
  121. and not isinstance(msg.get("_row_id"), int)
  122. and not isinstance(msg.get(DB_ROW_SNAPSHOT), str)
  123. and (uid := message_uid_or_none(msg))
  124. })
  125. durable: dict[str, list[sqlite3.Row]] = {}
  126. for chunk in _id_chunks(candidate_uids):
  127. for row in conn.execute(
  128. f"SELECT * FROM messages WHERE session_id = ? AND active = 1 AND message_uid IN ({_placeholders(chunk)})",
  129. (session_id, *chunk),
  130. ).fetchall():
  131. durable.setdefault(row["message_uid"], []).append(row)
  132. return durable
  133. # Owned columns a restore projection may leave off the dict: ``token_count`` is never decoded, the summary
  134. # flag only on request, and the underscore identity maps are lost to any wire strip. A dict without a value
  135. # says nothing about the column, so the row's value stands (the flush row turns an absent summary flag into
  136. # False, so falsy is the absent case); a dict that carries a value is compared as usual.
  137. _RESTORE_MAY_OMIT = (
  138. ("token_count", "token_count"), ("_compressed_summary", "_compressed_summary"),
  139. ("tool_call_uids", TOOL_CALL_UIDS), ("absorbed_message_uids", ABSORBED_MESSAGE_UIDS))
  140. def _same_as_stored(
  141. msg: dict[str, Any], rows: list[sqlite3.Row],
  142. serialize_message_fn: Callable[[dict[str, Any], float], Mapping[str, Any]],
  143. encode_content_fn: Callable[[Any], Any], decode_content_fn: Callable[[Any], Any],
  144. ) -> bool:
  145. """*msg* serializes to the replayed columns of one of its uid's active *rows*: the re-handed copy, not an edit.
  146. The row is seen through the restore lens (``_loaded_view_content``: user/assistant text sanitized and
  147. stripped on load), because that is what the dict was rebuilt from; the dict's own content is never
  148. normalized, so text that differs after the lens is an edit and inserts.
  149. """
  150. for row in rows:
  151. serialized = dict(serialize_message_fn(msg, float(row["timestamp"])))
  152. for column, key in _RESTORE_MAY_OMIT:
  153. if not msg.get(key):
  154. serialized[column] = row[column]
  155. if serialized["content"] == encode_content_fn(
  156. SessionMessagesMixin._loaded_view_content(row["role"], decode_content_fn(row["content"]))
  157. ):
  158. serialized["content"] = row["content"]
  159. if all(row[column] == serialized[column] for column in _REPLAY_COLUMNS):
  160. return True
  161. return False
  162. def resolve_and_repair_transcript_batch(
  163. conn: sqlite3.Connection,
  164. session_id: str,
  165. messages: list[dict[str, Any]],
  166. encode_content_fn: Callable[[Any], Any],
  167. decode_content_fn: Callable[[Any], Any],
  168. serialize_message_fn: Callable[[dict[str, Any], float], Mapping[str, Any]],
  169. decode_row_fn: Callable[[Mapping[str, Any]], dict[str, Any]],
  170. ) -> list[dict[str, Any]]:
  171. """Resolve row-addressed rewrites without appending duplicates or replacing concurrent winners.
  172. A durable row snapshot is a compare-and-swap version for sanitizer rewrites. When a live replay loses
  173. its physical ``_row_id``, the pair (logical ``message_uid``, stored-row snapshot) recovers the newest
  174. active generation without matching mutable payload. Watermark-compaction clones are matched by their
  175. copied payload identity, not timestamp alone. Legacy blank assistant rows retain the narrow interrupted-
  176. stream content repair. A snapshot-less restored dict whose uid is already an ACTIVE durable row with the
  177. same payload is that message re-handed to the flush — stamped durable and skipped, never re-INSERTed. Returns
  178. only rows that need fresh inserts.
  179. """
  180. # Late import: agent.session_persistence imports this module at load time.
  181. from agent.session_persistence import _PERSIST_AFTER_ADMISSION_INTERRUPT
  182. inserted_rows: list[dict[str, Any]] = []
  183. durable_uids = _active_uid_dedupe(conn, session_id, messages)
  184. for msg in messages:
  185. existing_row_id = msg.get("_row_id") if isinstance(msg, dict) else None
  186. role = msg.get("role", "unknown") if isinstance(msg, dict) else "unknown"
  187. expected = msg.get(DB_ROW_SNAPSHOT) if isinstance(msg, dict) else None
  188. target_row = None
  189. if isinstance(existing_row_id, int):
  190. target_row = _active_message_row(conn, session_id, existing_row_id, role)
  191. elif isinstance(expected, str):
  192. # Logical identity + the CAS version prove this dict came from durable replay. A new message
  193. # has neither proof, even when role/content/timestamp happen to equal an older message exactly.
  194. target_row = _active_logical_message_row(conn, session_id, role, message_uid_or_none(msg))
  195. if target_row is None:
  196. if (
  197. isinstance(msg, dict)
  198. and (stored := durable_uids.get(message_uid_or_none(msg) or ""))
  199. and not msg.get(_PERSIST_AFTER_ADMISSION_INTERRUPT)
  200. and _same_as_stored(msg, stored, serialize_message_fn, encode_content_fn, decode_content_fn)
  201. ):
  202. # Restored logical message, no row address and no CAS version (the wire strip removes
  203. # every underscore-prefixed field): its uid already names an ACTIVE durable row holding
  204. # the same payload, so this is the re-insert defect, not a new message. Stamp durable; a
  205. # restored dict edited since (other content) inserts as before, and the row-addressed
  206. # repair above keeps snapshot-carrying rewrites live.
  207. msg[_DB_PERSISTED_MARKER] = True
  208. continue
  209. inserted_rows.append(msg)
  210. continue
  211. target_id = int(target_row["id"])
  212. msg["_row_id"] = target_id
  213. _fill_missing_tool_call_uids(target_row, msg) # a dict that lost its uids must not rewrite them away
  214. canonical = None
  215. adopt = wrote = False
  216. if isinstance(expected, str):
  217. # The digest covers only the columns we own, so it answers "is the row still what we last
  218. # committed?". Match: the live dict is the source of truth (the DB holds its lossy durable
  219. # projection, multimodal parts -> text, which must never flow back): write the live owned values
  220. # and leave same-process metadata writes (reactions, backfills) alone. The caller holds BEGIN
  221. # IMMEDIATE, so the row cannot change between compare and UPDATE. Mismatch: another writer changed
  222. # the payload after our flush; adopt its durable version.
  223. adopt = transcript_row_snapshot(target_row) != expected
  224. if not adopt:
  225. serialized = serialize_message_fn(msg, float(target_row["timestamp"]))
  226. if serialized["token_count"] is None:
  227. # Replays never decode token_count (the agent flush row sets it to None): None means
  228. # "unknown", not NULL.
  229. serialized = {**serialized, "token_count": target_row["token_count"]}
  230. if any(target_row[column] != serialized[column] for column in _OWNED_COLUMNS):
  231. _rewrite_row(conn, session_id, target_row, serialized)
  232. wrote = True
  233. missing = {c: target_row[c] for c in _LIVE_MISSING_METADATA if msg.get(c) is None}
  234. if any(value is not None for value in missing.values()):
  235. decoded = decode_row_fn(target_row)
  236. canonical = {c: decoded[c] for c in missing if c in decoded}
  237. canonical[_METADATA_ONLY] = True
  238. elif role == "assistant" and is_content_blank(decode_content_fn(target_row["content"])):
  239. # Legacy dict (no digest) over a blank assistant row: the interrupted-stream repair. Fill the row
  240. # from live content with a content-only CAS and never adopt the blank row onto the live dict.
  241. wrote = conn.execute(
  242. "UPDATE messages SET content = ? WHERE id = ? AND session_id = ? AND content IS ?",
  243. (encode_content_fn(msg.get("content")), target_id, session_id, target_row["content"]),
  244. ).rowcount > 0
  245. else:
  246. # Legacy dict (no digest: a row-addressed resume, a clone, or a repair_alternation=False
  247. # projection) over a non-blank assistant row: another writer already filled it. Adopt its content
  248. # only, never the whole row: the live tool_calls / reasoning* / codex_* fields may be sanitizer-fixed
  249. # while the durable JSON still holds the raw escaped surrogate, and live-only fields must survive.
  250. if role == "assistant":
  251. canonical = {"content": decode_content_fn(target_row["content"]), _CONTENT_ONLY: True}
  252. final_row = conn.execute(
  253. "SELECT * FROM messages WHERE id = ? AND session_id = ?", (target_id, session_id)
  254. ).fetchone() if wrote else target_row
  255. msg["timestamp"] = final_row["timestamp"]
  256. # A row-addressed rewrite keeps the row's identity: the stored uid wins over whatever the live dict
  257. # carried (a restored dict without one, or a dict stamped before a rolled-back insert).
  258. _restore_row_identity(final_row, msg)
  259. msg[DB_ROW_SNAPSHOT] = transcript_row_snapshot(final_row)
  260. if adopt:
  261. canonical = decode_row_fn(final_row)
  262. if canonical:
  263. msg[CANONICAL_ROW] = canonical
  264. else:
  265. msg.pop(CANONICAL_ROW, None)
  266. return inserted_rows
  267. def _rewrite_row(
  268. conn: sqlite3.Connection,
  269. session_id: str,
  270. target_row: Mapping[str, Any],
  271. serialized: Mapping[str, Any],
  272. ) -> None:
  273. """Rewrite one durable payload whose digest matched the live dict's last committed version."""
  274. old_identity = target_row["display_identity"]
  275. old_peer_ids = [
  276. int(row["id"])
  277. for row in conn.execute(
  278. "SELECT id FROM messages WHERE session_id = ? AND id != ? "
  279. "AND (active = 1 OR compacted = 1) AND display_identity IS ?",
  280. (session_id, target_row["id"], old_identity),
  281. ).fetchall()
  282. ] if old_identity is not None else []
  283. # Owned columns only: same-process metadata writes (a reaction, a backfill) stay as committed.
  284. assignments = ", ".join(f"{column} = ?" for column in _OWNED_COLUMNS)
  285. conn.execute(
  286. f"UPDATE messages SET {assignments} WHERE id = ? AND session_id = ?",
  287. [*(serialized[column] for column in _OWNED_COLUMNS), int(target_row["id"]), session_id],
  288. )
  289. _restore_display_index(conn, session_id, target_row, serialized, old_identity, old_peer_ids)
  290. def _restore_display_index(
  291. conn: sqlite3.Connection,
  292. session_id: str,
  293. target_row: Mapping[str, Any],
  294. serialized: Mapping[str, Any],
  295. old_identity: Any,
  296. old_peer_ids: list[int],
  297. ) -> None:
  298. """Restore display identities/orders invalidated by the payload-update trigger."""
  299. if old_peer_ids:
  300. placeholders = _placeholders(old_peer_ids)
  301. old_order = min(old_peer_ids)
  302. conn.execute(
  303. f"UPDATE messages SET display_identity = ?, display_order = ? "
  304. f"WHERE session_id = ? AND id IN ({placeholders})",
  305. (old_identity, old_order, session_id, *old_peer_ids),
  306. )
  307. target_id = int(target_row["id"])
  308. new_identity = serialized["display_identity"]
  309. visible = bool(target_row["active"] or target_row["compacted"])
  310. if not visible:
  311. conn.execute(
  312. "UPDATE messages SET display_identity = ?, display_order = ? WHERE id = ? AND session_id = ?",
  313. (new_identity, target_id, target_id, session_id),
  314. )
  315. return
  316. peers = conn.execute(
  317. "SELECT id, display_order FROM messages WHERE session_id = ? AND id != ? "
  318. "AND (active = 1 OR compacted = 1) AND display_identity IS ?",
  319. (session_id, target_id, new_identity),
  320. ).fetchall()
  321. new_order = min(
  322. [target_id]
  323. + [int(peer["display_order"] if peer["display_order"] is not None else peer["id"]) for peer in peers]
  324. )
  325. conn.execute(
  326. "UPDATE messages SET display_identity = ?, display_order = ? WHERE id = ? AND session_id = ?",
  327. (new_identity, new_order, target_id, session_id),
  328. )
  329. if peers:
  330. conn.executemany(
  331. "UPDATE messages SET display_order = ? WHERE id = ? AND session_id = ?",
  332. [(new_order, int(peer["id"]), session_id) for peer in peers],
  333. )
  334. def _active_message_row(conn: sqlite3.Connection, session_id: str, row_id: int, role: str):
  335. """The same-role active clone for ``row_id``, or the addressed inactive row when no clone exists."""
  336. row = conn.execute(
  337. "SELECT * FROM messages WHERE id = ? AND session_id = ?", (row_id, session_id)
  338. ).fetchone()
  339. if row is None or row["role"] != role:
  340. return None
  341. if int(row["active"] or 0) == 1:
  342. return row
  343. # Watermark compaction copies the complete durable row payload and display identity byte-for-byte.
  344. # Timestamp alone is not an identity: externally supplied event timestamps may collide across unrelated
  345. # messages. Legacy rows without an indexed identity cannot be resolved safely, so retain the addressed row.
  346. if row["display_identity"] is None:
  347. return row
  348. payload_predicates = " AND ".join(f"{column} IS ?" for column in _REPAIR_COLUMNS)
  349. clones = conn.execute(
  350. "SELECT * FROM messages WHERE session_id = ? AND active = 1 AND role = ? "
  351. "AND display_identity IS ? AND timestamp IS ? AND id != ? AND "
  352. f"{payload_predicates} ORDER BY id DESC LIMIT 2",
  353. (
  354. session_id,
  355. role,
  356. row["display_identity"],
  357. row["timestamp"],
  358. row["id"],
  359. *(row[column] for column in _REPAIR_COLUMNS),
  360. ),
  361. ).fetchall()
  362. return clones[0] if len(clones) == 1 else row
  363. def sync_flushed_message_markers(batch_msgs: list[dict[str, Any]], batch_rows: list[dict[str, Any]]) -> None:
  364. """Stamp persistence markers and sync canonical durable fields onto live dicts after commit."""
  365. for written, row in zip(batch_msgs, batch_rows):
  366. written[_DB_PERSISTED_MARKER] = True
  367. if isinstance(row.get("_row_id"), int):
  368. written["_row_id"] = row["_row_id"]
  369. copy_identity_fields(row, written)
  370. if isinstance(row.get("timestamp"), (int, float)):
  371. written["timestamp"] = row["timestamp"]
  372. if isinstance(row.get(DB_ROW_SNAPSHOT), str):
  373. written[DB_ROW_SNAPSHOT] = row[DB_ROW_SNAPSHOT]
  374. canonical = row.get(CANONICAL_ROW)
  375. if isinstance(canonical, dict) and canonical.get(_METADATA_ONLY):
  376. # Our own row: only hand over presentation metadata the live dict lacks, never payload.
  377. for key in _LIVE_MISSING_METADATA:
  378. if canonical.get(key) is not None and written.get(key) is None:
  379. written[key] = canonical[key]
  380. elif isinstance(canonical, dict) and canonical.get(_CONTENT_ONLY):
  381. written["content"] = canonical["content"]
  382. elif isinstance(canonical, dict):
  383. for key in _SYNC_FIELDS:
  384. if key in canonical and canonical[key] is not None:
  385. written[key] = canonical[key]
  386. elif key not in ("role", "content"):
  387. written.pop(key, None)