hermes_state_coverage.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483
  1. """Held-row coverage resolution for in-place compaction: which durable rows the compressor actually held.
  2. Names carried-forward dicts, id-less held dicts and the rows an alternation repair merged, dropped or
  3. folded, so ``archive_and_compact`` archives exactly those (or falls back to the watermark when a durable
  4. held row cannot be named). Mixin bound via the MRO next to SessionMessagesMixin, which owns the writes.
  5. """
  6. from __future__ import annotations
  7. from collections import Counter
  8. from typing import Any, Dict, List, Optional, Set, Tuple
  9. from agent.message_metadata import DB_ROW_SNAPSHOT, MESSAGE_UID, message_uid_or_none
  10. from hermes_cli.timefmt import coerce_epoch
  11. from hermes_state_common import _id_chunks, _placeholders
  12. from hermes_state_messages import _parse_tool_calls
  13. def _text_of(content: Any) -> str:
  14. """The text a message carries: a string, or its text parts joined; image parts carry none."""
  15. if isinstance(content, str):
  16. return content
  17. if isinstance(content, list):
  18. return "\n".join(
  19. str(part.get("text") or "") for part in content
  20. if isinstance(part, dict) and part.get("type") in {"text", "input_text", "output_text"})
  21. return ""
  22. def _holds_as_block(inner: str, outer: str) -> bool:
  23. """True when *inner* sits in *outer* as whole lines: the compressor adds only whole blocks around a carried
  24. row's text (header, markers, summary), so a hit that starts or ends mid-line is the row losing text instead.
  25. Blanks beside the hit count as line edges: the load lens strips the stored text, never the copy inside."""
  26. start = outer.find(inner)
  27. while start != -1:
  28. end = start + len(inner)
  29. if not outer[:start].rpartition("\n")[2].strip() and not outer[end:].partition("\n")[0].strip():
  30. return True
  31. start = outer.find(inner, start + 1)
  32. return False
  33. def _images_replaced(stored: Any, held: Any) -> bool:
  34. """True when *held* is *stored* with at least one image part swapped for a text part, all else equal."""
  35. from agent.context_compressor import _is_image_part
  36. if not (isinstance(stored, list) and isinstance(held, list)) or len(stored) != len(held):
  37. return False
  38. swapped = False
  39. for was, now in zip(stored, held):
  40. if was == now:
  41. continue
  42. if not (_is_image_part(was) and isinstance(now, dict) and now.get("type") == "text"):
  43. return False
  44. swapped = True
  45. return swapped
  46. class SessionCoverageMixin:
  47. """Coverage helpers for SessionDB; relies on SessionMessagesMixin's codec/identity helpers."""
  48. def _carried_tail_ids(
  49. self, conn, tail_ids: list[int], compacted_messages: list[dict[str, Any]],
  50. ) -> tuple[dict[int, int], set[int]]:
  51. """Rows of *tail_ids* that *compacted_messages* already carry -> ``({row id: carrying dict index}, weak)``.
  52. A row above the watermark is normally a concurrent append the summary never saw, so the commit
  53. clones it after the compacted set. A row the compressor ALSO held (flushed after the watermark
  54. was read, or left unnamed so coverage fell back to the watermark) is already in the compacted
  55. set as a carried copy; cloning it too stores the message twice, with a fresh id and timestamp
  56. each (#137197). A claim must prove the occurrence AND the version, so the next request sees exactly
  57. what it would have seen with the clone minus the second copy: only a dict that names the row
  58. (``_row_id``, else ``message_uid``) may claim it, never matching text, which another surface can
  59. append independently. The first pass claims a named dict holding the row's stored content and its
  60. replayed sidecars (``_replays_as``). The second lets a still-unclaimed named dict with other content
  61. claim the row only when the compressor's own rewrite of the row's CURRENT version explains the
  62. difference (see ``_rewrite_explains``), exactly one tail row carries that uid and no active row
  63. outside *tail_ids* does (there the dict's original is that row, so the tail row is another version).
  64. Anything else is cloned: a duplicate, never a stale or missing model input. Each dict claims at most
  65. one row and each row is claimed once.
  66. ``weak`` holds the rows claimed by a rewritten carry: it is not the stored text, so the caller archives
  67. those as compacted history, never rewinds them out of display.
  68. """
  69. if not tail_ids or not compacted_messages:
  70. return {}, set()
  71. by_row, by_uid = self._index_held_dicts(compacted_messages)
  72. rows = conn.execute(
  73. # Every column: the claims compare replayed sidecars and the row's version digest (DB_ROW_SNAPSHOT).
  74. f"SELECT * FROM messages WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id", tail_ids,
  75. ).fetchall()
  76. carried: dict[int, int] = {}
  77. for row in rows:
  78. index = self._claim_stored_copy(row, compacted_messages, by_row, by_uid, set(carried.values()))
  79. if index is not None:
  80. carried[int(row["id"])] = index
  81. named_uids = {*by_uid, *(named[2] for named in by_row.values() if named[2] is not None)}
  82. held_elsewhere = self._uids_active_outside(conn, rows, tail_ids, named_uids)
  83. tail_uids = Counter(row[MESSAGE_UID] for row in rows if row[MESSAGE_UID])
  84. weak: set[int] = set()
  85. for row in rows:
  86. row_id, row_uid = int(row["id"]), row[MESSAGE_UID] or None
  87. if row_id in carried or row_uid is None or tail_uids[row_uid] != 1 or row_uid in held_elsewhere:
  88. continue
  89. index = self._rewritten_carry(row, row_uid, compacted_messages, by_row, by_uid, set(carried.values()))
  90. if index is not None:
  91. carried[row_id] = index
  92. weak.add(row_id)
  93. return carried, weak
  94. def _index_held_dicts(self, compacted_messages: list[dict[str, Any]]) -> tuple[dict, dict]:
  95. """Held dicts by the name they carry: ``_row_id`` -> (index, identity, uid); uid -> [(index, identity)].
  96. A dict naming neither is never a claim: equal text is not the same occurrence."""
  97. by_row: dict[int, tuple[int, tuple[Any, ...], Optional[str]]] = {}
  98. by_uid: dict[str, list[tuple[int, tuple[Any, ...]]]] = {}
  99. for index, message in enumerate(compacted_messages):
  100. if not isinstance(message, dict):
  101. continue
  102. identity = self._row_identity(
  103. message.get("role", "unknown"), message.get("content"), message.get("tool_call_id"),
  104. _parse_tool_calls(message.get("tool_calls")))
  105. row_id = message.get("_row_id")
  106. uid = message_uid_or_none(message)
  107. if isinstance(row_id, int) and not isinstance(row_id, bool) and row_id > 0:
  108. # Authoritative only while the stored identity still matches (a stale stamp names another row).
  109. by_row.setdefault(row_id, (index, identity, uid))
  110. elif uid is not None:
  111. by_uid.setdefault(uid, []).append((index, identity))
  112. return by_row, by_uid
  113. def _claim_stored_copy(self, row, compacted_messages, by_row, by_uid, claimed) -> Optional[int]:
  114. """First pass: the unclaimed dict naming *row* (``_row_id``, else uid) that replays exactly as it does."""
  115. identity = self._row_identity(
  116. row["role"], self._decode_content(row["content"]), row["tool_call_id"], _parse_tool_calls(row["tool_calls"]))
  117. named = by_row.get(int(row["id"]))
  118. candidates = [named[0]] if named is not None and named[1] == identity else []
  119. row_uid = row[MESSAGE_UID] or None
  120. candidates += [index for index, held in by_uid.get(row_uid, ()) if held == identity] if row_uid else []
  121. return next((index for index in candidates
  122. if index not in claimed and self._replays_as(row, compacted_messages[index], True)), None)
  123. def _rewritten_carry(self, row, row_uid, compacted_messages, by_row, by_uid, claimed) -> Optional[int]:
  124. """Second pass: the one unclaimed dict naming *row* (``_row_id`` with the row's uid, else its uid alone)
  125. whose different content the compressor's rewrite of that row explains; None means clone."""
  126. named = by_row.get(int(row["id"]))
  127. if named is not None and named[2] == row_uid:
  128. candidates = [named[0]]
  129. else:
  130. candidates = [index for index, _identity in by_uid.get(row_uid, ())]
  131. candidates = [index for index in candidates if index not in claimed]
  132. if len(candidates) != 1:
  133. return None
  134. if self._rewrite_explains(row, compacted_messages[candidates[0]]):
  135. return candidates[0]
  136. return None
  137. def _replays_as(self, row, held: dict[str, Any], same_content: bool) -> bool:
  138. """Whether *held* replays the sidecars the next request would read from *row* (``api_content`` and the
  139. reasoning carriers in ``REPLAY_SIDECARS``), compared through the writer's own serializer.
  140. A sidecar the copy carries must equal the row's: a different one is an older version (a later
  141. ``set_message_api_content``, a re-stamped reasoning item). A copy WITHOUT the row's ``api_content``
  142. stands for it only when its own content is those bytes (the flush stores sanitize-changed text as the
  143. sidecar), or when the compressor rewrote the content, which drops the stale sidecar by design."""
  144. from agent.transcript_repair import REPLAY_SIDECARS
  145. serialized = self._serialized_message_row(row["session_id"], held, float(row["timestamp"] or 0))
  146. content = self._decode_content(row["content"])
  147. row_api, held_api = (None if api == text else api for api, text in (
  148. (row["api_content"], content), (serialized["api_content"], held.get("content"))))
  149. if any(serialized[column] is not None and serialized[column] != row[column] for column in REPLAY_SIDECARS):
  150. return False
  151. if held_api is not None or row_api is None:
  152. return held_api == row_api
  153. return not same_content or held.get("content") == row_api
  154. @staticmethod
  155. def _copy_of_current_row(row, held: dict[str, Any]) -> bool:
  156. """Whether *held* was copied from *row*'s current version, so extra text in it is the compressor's.
  157. Containment alone cannot tell "the compressor added text" from "the row lost text after the copy"
  158. (an in-place edit that shortens it). Only the stored-row digest a flushed dict carries settles it: a
  159. mismatch means the row was rewritten after the copy. A copy without one (a bare uid copy, or a
  160. row-addressed load, which omits the digest by design) may predate any number of rewrites, so the row
  161. is cloned: a duplicate, never an older instruction in place of the current one."""
  162. # Runtime import avoids the transcript_repair -> hermes_state_messages module cycle.
  163. from agent.transcript_repair import transcript_row_snapshot
  164. snapshot = held.get(DB_ROW_SNAPSHOT)
  165. return isinstance(snapshot, str) and snapshot == transcript_row_snapshot(row)
  166. def _rewrite_explains(self, row, held: dict[str, Any]) -> bool:
  167. """True when *held* differs from stored *row* only the way the compressor rewrites a carry it keeps:
  168. image parts replaced by text placeholders, or, for a copy of the row's current version, the row's
  169. text kept verbatim as whole lines inside a longer text (a summary merged in, prior-context header and
  170. markers around it); and its sidecars still replay the row's. The summary flag alone is not enough: a
  171. flagged copy taken before an in-place rewrite of the row would put the older text back."""
  172. role = row["role"]
  173. if (held.get("role"), held.get("tool_call_id") or None) != (role, row["tool_call_id"] or None):
  174. return False
  175. if (_parse_tool_calls(held.get("tool_calls")) or None) != (_parse_tool_calls(row["tool_calls"]) or None):
  176. return False
  177. stored = self._loaded_view_content(role, self._decode_content(row["content"]))
  178. content = self._loaded_view_content(role, held.get("content"))
  179. if not self._replays_as(row, held, stored == content):
  180. return False
  181. if _images_replaced(stored, content):
  182. return True
  183. if not self._copy_of_current_row(row, held):
  184. return False
  185. stored_text = _text_of(stored)
  186. return bool(stored_text.strip()) and _holds_as_block(stored_text, _text_of(content))
  187. @staticmethod
  188. def _uids_active_outside(conn, rows, tail_ids: list[int], named_uids) -> set[str]:
  189. """uids of tail *rows* a held dict names that an ACTIVE row outside *tail_ids* (a covered row, below the
  190. watermark) also carries: there the dict's own original is that row, so the tail row is another version."""
  191. wanted = sorted({row[MESSAGE_UID] for row in rows if row[MESSAGE_UID] in named_uids})
  192. if not wanted:
  193. return set()
  194. tail, session_id = set(tail_ids), rows[0]["session_id"]
  195. return {
  196. found[MESSAGE_UID]
  197. for chunk in _id_chunks(wanted)
  198. for found in conn.execute(
  199. f"SELECT id, {MESSAGE_UID} FROM messages WHERE session_id = ? AND active = 1 "
  200. f"AND {MESSAGE_UID} IN ({_placeholders(chunk)})", (session_id, *chunk)).fetchall()
  201. if int(found["id"]) not in tail
  202. }
  203. @staticmethod
  204. def _carried_tail_slots(carried: dict[int, int], compacted_count: int, tail_count: int) -> int:
  205. """How many of the carried tail's *tail_count* dicts name a row in *carried*: those originals sit
  206. above the watermark, so the positional rewind below it must take that many fewer rows."""
  207. if tail_count <= 0:
  208. return 0
  209. return sum(1 for index in carried.values() if index >= compacted_count - int(tail_count))
  210. def _resolve_carried_row_ids(
  211. self, conn, session_id: str, carried_messages: list[dict[str, Any]],
  212. ) -> list[int]:
  213. """Resolve byte-identical carried-forward live dicts to their ACTIVE durable originals.
  214. _row_id is authoritative when the message carries it and the stored identity still matches.
  215. Resume surfaces that intentionally omit row ids fall back to a UNIQUE
  216. (role/content/tool identity, timestamp) match. Ambiguous or timestamp-less fallbacks are left
  217. as compacted history rather than risking a false rewind classification.
  218. """
  219. if not carried_messages:
  220. return []
  221. carried: list[tuple[tuple[Any, ...], Any, Any]] = []
  222. for message in carried_messages:
  223. if not isinstance(message, dict):
  224. continue
  225. identity = self._row_identity(
  226. message.get("role", "unknown"), message.get("content"), message.get("tool_call_id"),
  227. _parse_tool_calls(message.get("tool_calls")))
  228. row_id = message.get("_row_id")
  229. if not (isinstance(row_id, int) and not isinstance(row_id, bool) and row_id > 0):
  230. row_id = None
  231. carried.append((identity, row_id, message.get("timestamp")))
  232. def _index(ids: Optional[list[int]]):
  233. by_id: dict[int, tuple[Any, ...]] = {}
  234. by_key: dict[tuple[Any, ...], list[int]] = {}
  235. narrow = f" AND id IN ({_placeholders(ids)})" if ids else ""
  236. for row in conn.execute(
  237. "SELECT id, role, content, tool_call_id, tool_calls, timestamp FROM messages "
  238. f"WHERE session_id = ? AND active = 1{narrow} ORDER BY id",
  239. (session_id, *(ids or ())),
  240. ).fetchall():
  241. rid = int(row["id"])
  242. by_id[rid] = self._row_identity(
  243. row["role"], self._decode_content(row["content"]), row["tool_call_id"],
  244. _parse_tool_calls(row["tool_calls"]))
  245. ts = coerce_epoch(row["timestamp"], field="message timestamp")
  246. if ts is not None:
  247. by_key.setdefault((*by_id[rid], ts), []).append(rid)
  248. return by_id, by_key
  249. # The common micro pass carries dicts that all hold a matching _row_id, so the identity
  250. # check only needs those rows; a full active-row scan is reserved for the fallbacks.
  251. row_ids = [row_id for _, row_id, _ in carried if row_id is not None]
  252. by_id, by_key = _index(row_ids if len(row_ids) == len(carried) else None)
  253. if len(row_ids) == len(carried) and any(by_id.get(rid) != ident for ident, rid, _ in carried):
  254. by_id, by_key = _index(None)
  255. resolved: list[int] = []
  256. for identity, row_id, raw_timestamp in carried:
  257. if row_id is not None and by_id.get(row_id) == identity:
  258. resolved.append(row_id)
  259. continue
  260. timestamp = coerce_epoch(raw_timestamp, field="message timestamp")
  261. if timestamp is None:
  262. continue
  263. matches = by_key.get((*identity, timestamp), [])
  264. if len(matches) == 1:
  265. resolved.append(matches[0])
  266. return list(dict.fromkeys(resolved))
  267. def _matching_active_ids(self, conn, session_id: str, message: dict[str, Any]) -> list[int]:
  268. """Active row ids whose stored role and content equal *message*. Empty when it was never persisted."""
  269. content = message.get("content")
  270. if not isinstance(content, str):
  271. return []
  272. stored = self._encode_content(self._loaded_view_content(message.get("role", "unknown"), content))
  273. return [int(row["id"]) for row in conn.execute(
  274. "SELECT id FROM messages WHERE session_id = ? AND active = 1 AND role = ? AND content = ?",
  275. (session_id, message.get("role"), stored)).fetchall()]
  276. def _matching_retired_ids(
  277. self, conn, session_id: str, retired: dict[str, Any], watermark: Optional[int] = None,
  278. ) -> list[int]:
  279. """Active rows equal to a row the alternation repair retired without an id.
  280. Matched on role, loaded-view content (an empty assistant stores ``None`` or ``""``), tool_call_id
  281. and the ids of its tool calls. Only rows at or below *watermark*: the repair ran on a load taken
  282. before the snapshot, so a later equal row is another surface's append.
  283. """
  284. role = str(retired.get("role") or "")
  285. bound = watermark is not None
  286. rows = conn.execute(
  287. "SELECT id, content, tool_call_id, tool_calls FROM messages WHERE session_id = ? AND active = 1 "
  288. f"AND role = ?{' AND id <= ?' if bound else ''} ORDER BY id",
  289. (session_id, role, *((int(watermark),) if bound else ()))).fetchall()
  290. want_calls = list(retired.get("tool_call_ids") or ())
  291. matches = []
  292. for row in rows:
  293. content = self._loaded_view_content(role, self._decode_content(row["content"]))
  294. if (content or None) != (retired.get("content") or None):
  295. continue
  296. if (row["tool_call_id"] or None) != (retired.get("tool_call_id") or None):
  297. continue
  298. calls = _parse_tool_calls(row["tool_calls"]) or []
  299. ids = [call.get("id") for call in calls if isinstance(call, dict)] if isinstance(calls, list) else []
  300. if ids != want_calls:
  301. continue
  302. matches.append(int(row["id"]))
  303. return matches
  304. def _merged_user_run(
  305. self, conn, session_id: str, message: dict[str, Any], watermark: Optional[int] = None,
  306. ) -> Optional[list[int]]:
  307. """Active rows an alternation repair merged into *message*: ``[]`` for none, None when ambiguous.
  308. A reload without row ids turns a durable ``user;user`` pair (a prompt that never got its reply)
  309. into one dict equal to neither row. Read as an unpersisted turn, its rows would be re-sequenced
  310. after the compacted set like concurrent appends, behind the turn that is running.
  311. Only a dict the repair stamped: a prompt that was never persisted can carry the same text as
  312. rows another surface appended, and those were never held.
  313. Only rows at or below *watermark*: the repair ran on a load taken before the snapshot, so a
  314. run appended later that joins to the same text is another surface's, not the merged one.
  315. """
  316. from agent.conversation_compression_archive import MERGED_DURABLE_ROWS, OWN_ROW, RETIRED_DURABLE_ROWS
  317. content, width = message.get("content"), message.get(MERGED_DURABLE_ROWS)
  318. if message.get("role") != "user" or not isinstance(content, str) or type(width) is not int or width < 2:
  319. return []
  320. from gateway.message_timestamps import strip_leading_message_timestamps
  321. content, _ = strip_leading_message_timestamps(content)
  322. bound = watermark is not None
  323. rows = [
  324. (int(row["id"]), row["role"], self._loaded_view_content(row["role"], self._decode_content(row["content"])))
  325. for row in conn.execute(
  326. f"SELECT id, role, content FROM messages WHERE session_id = ? AND active = 1"
  327. f"{' AND id <= ?' if bound else ''} ORDER BY id",
  328. (session_id, *((int(watermark),) if bound else ()))).fetchall()]
  329. # Dropping an orphan can make user rows adjacent only in the repaired view.
  330. # Skip only uniquely proved retired originals, never arbitrary intervening rows.
  331. retired_ids = set()
  332. for retired in message.get(RETIRED_DURABLE_ROWS) or ():
  333. if retired.get(OWN_ROW):
  334. continue
  335. matches = self._matching_retired_ids(conn, session_id, retired, watermark)
  336. if len(matches) != 1:
  337. return None
  338. retired_ids.update(matches)
  339. rows = [row for row in rows if row[0] not in retired_ids]
  340. runs: list[list[int]] = []
  341. for start in range(len(rows)):
  342. merged = ""
  343. for end in range(start, len(rows)):
  344. _row_id, role, part = rows[end]
  345. if role != "user" or not isinstance(part, str):
  346. break
  347. merged = f"{merged}\n\n{part}" if merged and part else (merged or part)
  348. if not content.startswith(merged):
  349. break
  350. if end - start + 1 == width:
  351. if merged == content:
  352. runs.append([rows[i][0] for i in range(start, end + 1)])
  353. break
  354. if len(runs) > 1:
  355. return None
  356. return runs[0] if runs else []
  357. def _proved_coverage(
  358. self, conn, session_id: str, covered_ids: Optional[list[int]],
  359. unresolved_held: Optional[list[dict[str, Any]]], watermark: Optional[int] = None,
  360. ) -> Optional[tuple[list[int], set[int]]]:
  361. """``(ids safe to archive as summarized, ids merged into another held dict)``, or None when
  362. a durable held row cannot be named.
  363. An unresolved dict that still carries the persist marker was loaded from the DB.
  364. Failing to name it means the watermark path, which archives the rows the compressor
  365. saw, including ones whose ids were stripped. A marker-less miss is an unpersisted
  366. turn: it names nothing, and it is not a reason to abandon the ids we do have.
  367. Several active rows with the same content are ambiguous, so that also abandons,
  368. and so does a merged dict whose run cannot be named.
  369. The second set holds only merges the caller could not count: a dict that lists its
  370. own ``_absorbed_row_ids`` is already counted in ``tail_count``.
  371. """
  372. if covered_ids is None:
  373. return None
  374. from agent.context_compressor import _DB_PERSISTED_MARKER
  375. from agent.conversation_compression_archive import (
  376. ABSORBED_ROW_IDS, MERGED_DURABLE_ROWS, OWN_ROW, RETIRED_DURABLE_ROWS, RETIRED_ROW)
  377. proved = [int(row_id) for row_id in covered_ids if isinstance(row_id, int) and row_id > 0]
  378. merged_away: set[int] = set()
  379. for message in unresolved_held or ():
  380. if not isinstance(message, dict):
  381. continue
  382. if message.get(RETIRED_ROW):
  383. # A row the repair dropped behind another held dict: that dict stands for it in the tail.
  384. # The dict's own row, recorded before a fold rewrote its text, is its own tail slot.
  385. matches = self._matching_retired_ids(conn, session_id, message, watermark)
  386. if len(matches) != 1:
  387. return None
  388. proved.extend(matches)
  389. if not message.get(OWN_ROW):
  390. merged_away.update(matches)
  391. continue
  392. # A folded assistant's own row is named by its OWN_ROW record above. Its combined text never
  393. # existed in storage, so matching it by content could only claim another surface's row.
  394. if any(isinstance(row, dict) and row.get(OWN_ROW) for row in message.get(RETIRED_DURABLE_ROWS) or ()):
  395. continue
  396. # The stamp is provenance and text is not: a surface may have re-rendered the dict since the
  397. # repair (gateway timestamps), and a later row can equal the merged text. So the run is
  398. # resolved first, and a stamped dict that names none is still durable, never an unpersisted turn.
  399. run = self._merged_user_run(conn, session_id, message, watermark)
  400. if message.get(MERGED_DURABLE_ROWS) and not run:
  401. return None
  402. matches = [] if run else self._matching_active_ids(conn, session_id, message)
  403. if len(matches) > 1 or (
  404. message.get(_DB_PERSISTED_MARKER) and len(matches) != 1 and not run):
  405. return None
  406. proved.extend(matches or run)
  407. merged_away.update(set(run[1:]) - set(message.get(ABSORBED_ROW_IDS) or ()))
  408. return list(dict.fromkeys(proved)), merged_away
  409. @staticmethod
  410. def _uncounted_merged_rows(tail: list[dict[str, Any]]) -> int:
  411. """Durable rows behind the carried *tail* beyond one per dict, for the positional rewind.
  412. The watermark path cannot name a merged dict's run, so it widens by the stamp, less the
  413. rows the dict lists in ``_absorbed_row_ids``: those are already counted in ``tail_count``.
  414. Rows the repair dropped behind a dict, or folded into an assistant turn, on a reload
  415. without ids are in neither, so their count is added. A dict that holds a ``_row_id`` was
  416. written back as a single row by an earlier compaction.
  417. """
  418. from agent.conversation_compression_archive import ABSORBED_ROW_IDS, MERGED_DURABLE_ROWS, UNNAMED_DURABLE_ROWS
  419. def behind(message: dict[str, Any]) -> int:
  420. merged, unnamed = message.get(MERGED_DURABLE_ROWS), message.get(UNNAMED_DURABLE_ROWS)
  421. unlisted = merged - 1 - len(message.get(ABSORBED_ROW_IDS) or ()) if type(merged) is int else 0
  422. return max(0, unlisted) + (unnamed if type(unnamed) is int else 0)
  423. return sum(
  424. behind(message) for message in tail
  425. if isinstance(message, dict) and not isinstance(message.get("_row_id"), int))
  426. @staticmethod
  427. def _tail_originals(covered_active: list[int], tail_count: int, merged_away: set[int]) -> list[int]:
  428. """Newest rows behind *tail_count* carried dicts; a dict that merged rows stands for each of them."""
  429. width = int(tail_count)
  430. while True:
  431. window = covered_active[-width:]
  432. need = int(tail_count) + sum(1 for row_id in window if row_id in merged_away)
  433. if need <= width or width >= len(covered_active):
  434. return window
  435. width = need