| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004 |
- """Transcript persistence for SessionDB: message append/replace/rewind, reactions, resume assembly,
- replayed-user dedupe. Mixin bound via the MRO, built on SessionDB's _read_ctx/_execute_write/_read_* primitives."""
- from __future__ import annotations
- import hashlib
- import json
- import logging
- import re
- import time
- from typing import Any, Dict, List, Optional, Set, Tuple
- from agent.context_compressor import (
- _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY, MODEL_ONLY_DISPLAY_METADATA_KEY, _is_checkpoint_item,
- _newest_checkpoint_carrier, split_user_originated_turn)
- from agent.memory_context import sanitize_context
- from agent.message_metadata import (
- CANONICAL_ROW, DATA_USE, DB_ROW_SNAPSHOT, lift_data_use, MESSAGE_UID, TOOL_CALL_UID, TOOL_CALL_UIDS,
- index_tool_call_uids, message_uid_or_none, resolve_tool_call_uid, mint_uid, stamp_message_uid)
- from agent.message_sanitization import _sanitize_surrogates, coalesce_tool_call_id
- from hermes_cli.timefmt import coerce_epoch
- from hermes_state_common import (
- _COMPRESSION_LOCK_ROW_SQL, _ENDED_ROW_SQL, _RESET_END_REASONS, _RESET_END_REASONS_SQL, _ended_by_compression,
- _json_or, _legacy_reset_child_sql, _placeholders, _sql_json_extract)
- from hermes_state_compaction_display import later_tool_copy_ids, not_a_later_tool_copy_sql
- from hermes_state_detached_results import SessionDetachedResultsMixin
- from hermes_state_todo import SessionTodoResultMixin
- from hermes_state_identity import (
- _absorbed_uids_json, _restore_identity_columns, _stable_tool_key, _tool_call_uid_map, _tool_call_uid_or_none, _tool_call_uids_json, _uid_list)
- logger = logging.getLogger("hermes_state") # caplog tests pin the origin module's name
- # One INSERT shape for every message writer (append, batch, replace, compact, import).
- _INSERT_MESSAGE_SQL = """INSERT INTO messages (session_id, role, content, tool_call_id,
- tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason,
- reasoning, reasoning_content, reasoning_details, codex_reasoning_items,
- codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind,
- display_metadata, display_identity, message_uid, absorbed_message_uids, tool_call_uids,
- tool_call_uid)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
- # Every column this module knows how to read: the ones it writes plus the three SQLite/compaction
- # owns. `_row_to_message_dict` drops raw bytes ONLY outside this set — a schema column keeps its
- # key (and its typed decoder) even when a row holds a BLOB, so no reader ever loses msg["content"].
- _MESSAGE_WRITE_COLUMNS = tuple(
- re.findall(r"\w+", _INSERT_MESSAGE_SQL.split("(", 1)[1].split(")", 1)[0])
- )
- _MESSAGE_SCHEMA_KEYS = frozenset(_MESSAGE_WRITE_COLUMNS) | {"id", "compacted", "display_order"}
- _BUMP_GENERATION_SQL = """
- INSERT INTO conversation_generations (source, session_key, generation)
- VALUES (?, ?, 1)
- ON CONFLICT(source, session_key) DO UPDATE
- SET generation = conversation_generations.generation + 1
- """
- _TURN_LEASE_ROW_SQL = "SELECT holder, expires_at FROM session_turn_leases WHERE conversation_id = ?"
- _DELETE_COMPRESSION_LOCK_SQL = "DELETE FROM compression_locks WHERE session_id = ? AND holder = ?"
- _DISPLAY_ACTIVE_CLAUSE = " AND (active = 1 OR compacted = 1)"
- # Model-only rows (see MODEL_ONLY_DISPLAY_METADATA_KEY) never enter a display projection. Unqualified on
- # purpose: inside a correlated subquery it binds to the innermost ``messages`` alias.
- DISPLAY_VISIBLE_SQL = (
- f" AND COALESCE({_sql_json_extract('display_metadata', '$.' + MODEL_ONLY_DISPLAY_METADATA_KEY)}, 0) = 0")
- _DISPLAY_META_ROW_SQL = "SELECT display_metadata FROM messages WHERE id = ? AND session_id IN ({ids})" + _DISPLAY_ACTIVE_CLAUSE
- # A display row is indexed only when both halves are set; the read path backfills before projecting, so
- # the in-transaction delete fence must refuse (not project) any session this probe still matches.
- _DISPLAY_INDEX_MISSING_SQL = ("SELECT 1 FROM messages WHERE session_id = ?" + _DISPLAY_ACTIVE_CLAUSE
- + " AND (display_order IS NULL OR display_identity IS NULL) LIMIT 1")
- _ACTIVE_IDS_SQL = "SELECT id FROM messages WHERE session_id = ? AND active = 1 ORDER BY id"
- _LIVE_IDENTITY_SQL = ("SELECT id, role, content, tool_call_id, tool_calls, message_uid FROM messages "
- "WHERE session_id = ? AND active = 1 ORDER BY id LIMIT ?")
- _SET_COUNTERS_SQL = "UPDATE sessions SET message_count = ?, tool_call_count = ?"
- _RESET_COUNTERS_SQL = "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?"
- _SET_DISPLAY_META_SQL = "UPDATE messages SET display_metadata = ? WHERE id = ?"
- _ARCHIVE_ACTIVE_SQL = "UPDATE messages SET active = 0, compacted = 1 WHERE session_id = ? AND active = 1"
- _SHADOWED_CHECKPOINT_ROWS_SQL = ("SELECT id, codex_reasoning_items FROM messages WHERE session_id = ? AND active = 1 "
- "AND role = 'assistant' AND id < ? AND codex_reasoning_items LIKE '%\"compaction\"%'")
- _SET_CODEX_REASONING_SQL = "UPDATE messages SET codex_reasoning_items = ? WHERE id = ?"
- _INVALID = object() # _json_or sentinel where the fallback must be distinguishable from JSON null
- def _coerce_timestamp(value: Any, default: float) -> float:
- """Explicit message timestamp (datetime or number) or *default* when invalid or outside the sane
- epoch window — the write-side twin of the readers' ``coerce_epoch``: a bad row is never persisted."""
- result = coerce_epoch(value, field="message timestamp")
- if result is None:
- return default
- # SQLite reads both signed zero spellings back as 0.0. Hash the value
- # in that stored form so -0.0 and 0.0 retain the historical SQL/Python
- # identity equality.
- return 0.0 if result == 0.0 else result
- def _parse_tool_calls(tool_calls: Any) -> Any:
- """tool_calls is a list (live agent) or JSON string (import/export); parse so json.dumps never double-encodes."""
- if not isinstance(tool_calls, str):
- return tool_calls
- try:
- return json.loads(tool_calls)
- except (json.JSONDecodeError, TypeError):
- return []
- def _tool_calls_count(tool_calls: Any) -> int:
- return 0 if tool_calls is None else (len(tool_calls) if isinstance(tool_calls, list) else 1)
- def _tool_calls_len(raw: Any, scalar: int = 0) -> int:
- """Count of a stored ``tool_calls`` column: list length, *scalar* for a truthy non-list, else 0."""
- parsed = _parse_tool_calls(raw)
- return len(parsed) if isinstance(parsed, list) else (scalar if parsed else 0)
- def _scrub_surrogates(value: Any) -> Any:
- """Lone surrogates make sqlite3 raise UnicodeEncodeError and abort the whole write."""
- return _sanitize_surrogates(value) if isinstance(value, str) else value
- def _stale_holder(row, now: float) -> bool:
- """A lock/lease row whose holder is expired or a provably dead local process."""
- from hermes_state import _compression_lock_holder_process_is_dead
- return float(row["expires_at"]) <= now or _compression_lock_holder_process_is_dead(row["holder"])
- class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):
- """Message append/replace/rewind, reactions, resume conversations, replay dedupe (detached results: the base)."""
- def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None:
- """Advance the peer's conversation generation past a boundary, in the txn that writes it. Only
- ``_RESET_END_REASONS`` count (compression continues one conversation). Never derived from session
- rows (deletes/prunes could re-emit a retired affinity identity); it only ever increments."""
- if end_reason not in _RESET_END_REASONS:
- return
- row = conn.execute("SELECT source, session_key FROM sessions WHERE id = ?", (session_id,)).fetchone()
- if row is None:
- return
- source, session_key = (str(row[k] or "").strip() for k in ("source", "session_key"))
- if source and session_key:
- conn.execute(_BUMP_GENERATION_SQL, (source, session_key))
- @classmethod
- def _encode_content(cls, content: Any) -> Any:
- """Serialize list/dict content (multimodal parts) as a sentinel-prefixed JSON string (sqlite3 binds
- only scalars). Lone UTF-16 surrogates (unsanitized web-scraped tool results) are scrubbed here: left
- raw, sqlite3 raises UnicodeEncodeError and the session silently stops persisting. Pairs with
- :meth:`_decode_content`.
- A literal string carrying the reserved ``\\x00json:`` prefix is indistinguishable from
- stored structured content, so it is JSON-string-wrapped (decoded back verbatim on read)
- instead of written raw — a crafted import can't smuggle the marker in and have the read
- path decode attacker-chosen structure."""
- if isinstance(content, str):
- content = _sanitize_surrogates(content)
- if not content.startswith(cls._CONTENT_JSON_PREFIX):
- return content
- # Fall through: escape the reserved prefix by double-encoding the literal.
- elif content is None or isinstance(content, (bytes, int, float)):
- return content
- try:
- return cls._CONTENT_JSON_PREFIX + json.dumps(content) # ensure_ascii escapes surrogates: bindable
- except (TypeError, ValueError):
- return _sanitize_surrogates(str(content))
- @classmethod
- def _decode_content(cls, content: Any) -> Any:
- """Reverse :meth:`_encode_content`; returns scalars unchanged. Fail closed: a reserved-prefix
- string that isn't valid JSON (a legacy/malformed marker row) returns the raw string."""
- if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX):
- return _json_or(content[len(cls._CONTENT_JSON_PREFIX):], content,
- "Failed to decode JSON-encoded message content; returning raw string")
- return content
- @staticmethod
- def _encode_display_metadata(display_metadata: Any) -> Optional[str]:
- """Serialize ``display_metadata`` for its TEXT column; an already-serialized JSON string
- (import/replace paths) is not double-encoded."""
- if not display_metadata:
- return None
- if isinstance(display_metadata, str):
- display_metadata = _json_or(display_metadata, _INVALID, "Ignoring non-JSON display metadata on write")
- if display_metadata is _INVALID:
- return None
- if not isinstance(display_metadata, dict):
- logger.warning("Ignoring non-object display metadata on write")
- return None
- elif not isinstance(display_metadata, dict):
- logger.warning("Ignoring unexpected display metadata type on write: %s", type(display_metadata).__name__)
- return None
- return json.dumps(display_metadata)
- @staticmethod
- def _decode_display_metadata(raw: Any) -> Optional[dict[str, Any]]:
- """Decode a ``display_metadata`` column to a dict (never raw TEXT: the desktop does ``'task_count'
- in meta``). Pre-guard rows are double-encoded, so a second string layer is unwrapped."""
- if raw is None:
- return None
- meta = raw
- for _ in range(2): # pre-guard rows carry a second string layer
- if isinstance(meta, str):
- meta = _json_or(meta, _INVALID, "Ignoring invalid display metadata on message row")
- if meta is _INVALID:
- return None
- if not isinstance(meta, dict):
- logger.warning("Ignoring non-object display metadata on message row")
- return None
- return meta
- @staticmethod
- def _reasoning_json_text(value: Any) -> Optional[str]:
- """Serialize a structured reasoning field for its TEXT column. Strings are stored as-is: round-trips
- (get_messages -> replace_messages) hand back raw TEXT; re-dumping would double-encode it and
- reasoning-replay consumers (``isinstance(..., list)``) would drop it."""
- return None if not value else (value if isinstance(value, str) else json.dumps(value))
- def _check_transcript_write_guards(self, conn, session_id: str, compression_lock_holder: Optional[str],
- turn_lease_holder: Optional[str] = None, turn_lease_ttl_seconds: float = 300.0,
- reject_active_turn_lease: bool = False, reject_active_compression_lock: bool = False,
- allow_closed_compression_parent: bool = False) -> None:
- """Transcript-write admission checks, run INSIDE the write txn by every writer. Ordinary appends do
- NOT check compression_locks: the lock only stops two COMPRESSIONS colliding and archive_and_compact()
- commits against a watermark, so concurrent appends are safe (blocking them killed turns during slow
- summaries). Destructive user mutations opt in via ``reject_active_*`` so a compressor that captured
- its watermark cannot resurrect the removed turn.
- Shared by :meth:`append_message` and :meth:`append_messages_batch` so the two writers can never
- diverge on these correctness invariants (this guard has already needed targeted fixes — see the
- #74478 patience note below). User-initiated transcript mutations may opt in to rejecting an active
- unowned turn lease in that same transaction.
- """
- from hermes_state import SessionCompressionInProgressError
- from hermes_state_errors import CompressionSessionClosedError, SessionTurnLeaseLostError
- # NOTE (#75316 redesign): appends do NOT check compression_locks. The lock's job is to stop two
- # COMPRESSIONS colliding, not to fence ordinary transcript writes. Concurrent appends during a
- # compression are safe by construction: archive_and_compact() commits against a watermark captured
- # at compression start and clones every row that arrived after it back into the live transcript, in
- # the same write transaction. Blocking appends here was the root cause of a whole symptom family —
- # turns dying as session_persistence_failed while a slow provider summary held the lease (#74568,
- # #77386), including stale locks from dead PIDs blocking writes for the full TTL. Keep that narrow
- # fence opt-in so ordinary appends retain the watermark behavior.
- if reject_active_compression_lock:
- active_lock = conn.execute(_COMPRESSION_LOCK_ROW_SQL, (session_id,)).fetchone()
- if active_lock is not None:
- if _stale_holder(active_lock, time.time()):
- conn.execute(_DELETE_COMPRESSION_LOCK_SQL, (session_id, active_lock["holder"]))
- elif active_lock["holder"] != compression_lock_holder:
- raise SessionCompressionInProgressError(
- f"Session {session_id!r} is being compressed by another writer")
- if turn_lease_holder or reject_active_turn_lease:
- conversation_id = self._session_turn_lease_key_on_conn(conn, session_id)
- lease = conn.execute(_TURN_LEASE_ROW_SQL, (conversation_id,)).fetchone()
- now = time.time()
- if turn_lease_holder:
- if lease is None or lease["holder"] != turn_lease_holder:
- raise SessionTurnLeaseLostError(
- f"Session turn lease lost; refusing transcript write for {session_id!r}")
- if float(lease["expires_at"]) <= now:
- # Expiry makes the row reclaimable, not taken over; BEGIN IMMEDIATE serializes this
- # renewal with acquisition, so a still-matching owner recovers from a starved refresher.
- conn.execute("UPDATE session_turn_leases SET expires_at = ? "
- "WHERE conversation_id = ? AND holder = ?",
- (now + max(0.1, float(turn_lease_ttl_seconds)), conversation_id, turn_lease_holder))
- elif lease is not None:
- if not _stale_holder(lease, now):
- raise SessionTurnLeaseLostError(
- f"Session has an active turn lease; refusing transcript mutation for {session_id!r}")
- # Same reclaim rule as acquisition; deleting also fences a stale late flush after the mutation.
- conn.execute("DELETE FROM session_turn_leases WHERE conversation_id = ? AND holder = ?",
- (conversation_id, lease["holder"]))
- if _ended_by_compression(conn.execute(_ENDED_ROW_SQL, (session_id,)).fetchone()) and not allow_closed_compression_parent:
- raise CompressionSessionClosedError(session_id)
- def _message_row_params(self, session_id: str, role: str, msg: dict[str, Any], tool_calls: Any,
- message_timestamp: float, *, keep_reasoning: bool) -> tuple:
- """Bind values for ``_INSERT_MESSAGE_SQL`` from one message dict (*tool_calls* already parsed;
- *keep_reasoning* False NULLs every reasoning column). ``platform_message_id`` falls back to
- ``message_id`` (yuanbao's message-dict convention)."""
- _str_or_none = lambda v: _scrub_surrogates(v) if isinstance(v, str) else None
- _reasoning = lambda key: msg.get(key) if keep_reasoning else None
- encoded_content = self._encode_content(msg.get("content"))
- encoded_tool_calls = json.dumps(tool_calls) if tool_calls else None
- encoded_tool_name = _scrub_surrogates(msg.get("tool_name"))
- display_metadata = msg.get("display_metadata")
- if msg.get(DATA_USE) and isinstance(display_metadata, (dict, type(None))):
- # Writers that copy a fixed field list (/branch) carry the key, not display_metadata.
- display_metadata = {**(display_metadata or {}), DATA_USE: list(msg[DATA_USE])}
- display_metadata = self._encode_display_metadata(display_metadata)
- identity_row = {
- "role": role, "content": encoded_content, "timestamp": message_timestamp,
- "tool_call_id": msg.get("tool_call_id"), "tool_calls": encoded_tool_calls,
- "tool_name": encoded_tool_name, "display_kind": msg.get("display_kind"),
- "display_metadata": display_metadata,
- }
- return (session_id, role, encoded_content, msg.get("tool_call_id"),
- encoded_tool_calls, encoded_tool_name,
- msg.get("effect_disposition"), message_timestamp, msg.get("token_count"), msg.get("finish_reason"),
- _scrub_surrogates(_reasoning("reasoning")), _scrub_surrogates(_reasoning("reasoning_content")),
- *(self._reasoning_json_text(_reasoning(k))
- for k in ("reasoning_details", "codex_reasoning_items", "codex_message_items")),
- msg.get("platform_message_id") or msg.get("message_id"),
- 1 if msg.get("observed") else 0, 1 if msg.get("_compressed_summary") else 0, 1,
- _str_or_none(msg.get("api_content")), _str_or_none(msg.get("display_kind")),
- display_metadata, self._display_identity(self._display_dedupe_key(identity_row)),
- message_uid_or_none(msg), _absorbed_uids_json(msg),
- _tool_call_uids_json(msg), _tool_call_uid_or_none(msg))
- @staticmethod
- def _stamp_tool_call_uids(msg: dict[str, Any], tool_calls: Any, batch_index: dict[str, str]) -> None:
- """Per-occurrence tool-call identity at insert: an assistant row gets a uid for every tool call that
- has none yet (``_tool_call_uids``, keyed by the effective provider id, which stays untouched); a
- tool-result row that has none is resolved through *batch_index*, the calls the same batch's
- assistant rows named. A result flushed in a later batch is resolved from the live list by the flush
- (``tool_call_uid_from_history``) or, on restore, from the preceding assistant row."""
- role = msg.get("role")
- if role == "assistant" and tool_calls:
- uids = _tool_call_uid_map(msg)
- for tc in tool_calls:
- call_id = coalesce_tool_call_id(tc)
- if call_id and call_id not in uids:
- uids[call_id] = mint_uid() # one per provider id: a repeat in one response shares it
- if uids:
- msg[TOOL_CALL_UIDS] = uids
- index_tool_call_uids(batch_index, msg) # this row's ids shadow any earlier occurrence's
- elif role == "user":
- batch_index.clear() # a result never pairs across a user turn
- elif role == "tool" and _tool_call_uid_or_none(msg) is None:
- uid = resolve_tool_call_uid(batch_index, msg.get("tool_call_id"))
- if uid:
- msg[TOOL_CALL_UID] = uid
- def _serialized_message_row(
- self, session_id: str, msg: dict[str, Any], message_timestamp: float
- ) -> dict[str, Any]:
- """Serialize one live message with the exact durable shape used by INSERT."""
- role = msg.get("role", "unknown")
- params = self._message_row_params(
- session_id,
- role,
- msg,
- _parse_tool_calls(msg.get("tool_calls")),
- message_timestamp,
- keep_reasoning=role == "assistant",
- )
- return dict(zip(_MESSAGE_WRITE_COLUMNS, params))
- def _decoded_repair_row(self, row) -> dict[str, Any]:
- """Decode one durable row without replay dedupe, alternation repair, or content stripping."""
- msg: dict[str, Any] = {
- "role": row["role"],
- "content": self._decode_content(row["content"]),
- }
- for column in ("tool_call_id", "tool_name", "effect_disposition", "token_count", "finish_reason"):
- if row[column] is not None:
- msg[column] = row[column]
- if row["tool_calls"]:
- msg["tool_calls"] = _json_or(
- row["tool_calls"], [], "Failed to deserialize repaired tool_calls, falling back to []"
- )
- if row["platform_message_id"] is not None:
- msg["message_id"] = row["platform_message_id"]
- msg["platform_message_id"] = row["platform_message_id"]
- if row["observed"]:
- msg["observed"] = True
- if row["_compressed_summary"]:
- msg["_compressed_summary"] = True
- _restore_identity_columns(row, msg)
- if row["api_content"] is not None:
- msg["api_content"] = row["api_content"]
- if row["display_kind"] is not None:
- msg["display_kind"] = row["display_kind"]
- if row["display_metadata"] is not None:
- metadata = self._decode_display_metadata(row["display_metadata"])
- if metadata is not None:
- msg["display_metadata"] = metadata
- if row["role"] == "assistant":
- for column in ("reasoning", "reasoning_content"):
- if row[column] is not None:
- msg[column] = row[column]
- for column in ("reasoning_details", "codex_reasoning_items", "codex_message_items"):
- if row[column]:
- msg[column] = _json_or(
- row[column], None, f"Failed to deserialize repaired {column}, falling back to None"
- )
- return msg
- @staticmethod
- def _bump_session_counters(conn, session_id: str, inserted: int, tool_calls: int, *, unit: bool) -> None:
- """Bump sessions.* counters after an insert; *unit* bakes the ``+ 1`` literal into the SQL."""
- inc, params = ("1", ()) if unit else ("?", (inserted,))
- if tool_calls > 0:
- conn.execute(
- f"""UPDATE sessions SET message_count = message_count + {inc},
- tool_call_count = tool_call_count + ? WHERE id = ?""",
- (*params, tool_calls, session_id))
- elif inserted > 0:
- conn.execute(
- f"UPDATE sessions SET message_count = message_count + {inc} WHERE id = ?", (*params, session_id))
- def append_message(
- self, session_id: str, role: str, content: str | None = None, tool_name: str | None = None, tool_calls: Any = None,
- tool_call_id: str | None = None, token_count: int | None = None, finish_reason: str | None = None, reasoning: str | None = None,
- reasoning_content: str | None = None, reasoning_details: Any = None, codex_reasoning_items: Any = None,
- codex_message_items: Any = None, platform_message_id: str | None = None, observed: bool = False,
- effect_disposition: Optional[str] = None, _compressed_summary: bool = False, timestamp: Any = None,
- api_content: Optional[str] = None, display_kind: Optional[str] = None,
- display_metadata: Optional[dict[str, Any]] = None, compression_lock_holder: Optional[str] = None,
- turn_lease_holder: Optional[str] = None, turn_lease_ttl_seconds: float = 300.0,
- message_uid: Optional[str] = None) -> int:
- """Append one message; returns the row id and bumps the session counters. ``platform_message_id``:
- the platform's own id. ``api_content``: byte-fidelity sidecar, the exact string sent to the API when
- it differed from ``content``, stored as sent except lone surrogates. ``message_uid``: the id a caller
- that keeps the message as a live dict already stamped on it (``stamp_message_uid``); minted when
- absent, but then only the row carries it."""
- msg = dict(locals()) # every keyword above is a message-dict field of the same name
- # Encode outside the write txn (display metadata first: log-order parity).
- msg["display_metadata"] = self._encode_display_metadata(display_metadata)
- tool_calls = _parse_tool_calls(tool_calls)
- message_timestamp = _coerce_timestamp(timestamp, time.time())
- stamp_message_uid(msg) # a fresh occurrence: mint its durable id
- self._stamp_tool_call_uids(msg, tool_calls, {})
- params = self._message_row_params(
- session_id, role, msg, tool_calls, message_timestamp, keep_reasoning=True)
- def _do(conn):
- self._check_transcript_write_guards(conn, session_id, compression_lock_holder,
- turn_lease_holder=turn_lease_holder, turn_lease_ttl_seconds=turn_lease_ttl_seconds)
- msg_id = conn.execute(_INSERT_MESSAGE_SQL, params).lastrowid
- self._bump_session_counters(conn, session_id, 1, _tool_calls_count(tool_calls), unit=True)
- return msg_id
- # THE critical write (failure aborts the turn): long patience so a sibling legitimately
- # holding the lock for seconds (VACUUM, checkpoint) can't kill it.
- return self._execute_write(_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S)
- def append_messages_batch(
- self, session_id: str, messages: list[dict[str, Any]], compression_lock_holder: Optional[str] = None,
- turn_lease_holder: Optional[str] = None, chunk_rows: Optional[int] = None,
- turn_lease_ttl_seconds: float = 300.0) -> int:
- """Append *messages* in ONE write txn (all rows land or none, guards run once); returns the inserted
- count. ``chunk_rows`` bounds txn size for LARGE copies (branch seeds; FTS triggers run per row)."""
- if not messages:
- return 0
- if chunk_rows is not None and len(messages) > chunk_rows:
- tool_uid_index: dict[str, str] = {}
- for msg in messages: # each chunk indexes only its own rows: pair results across chunk boundaries
- self._stamp_tool_call_uids(msg, _parse_tool_calls(msg.get("tool_calls")), tool_uid_index)
- return sum(self.append_messages_batch(session_id, messages[start:start + chunk_rows],
- compression_lock_holder=compression_lock_holder, turn_lease_holder=turn_lease_holder,
- turn_lease_ttl_seconds=turn_lease_ttl_seconds)
- for start in range(0, len(messages), chunk_rows))
- return self._execute_transcript_write(lambda conn: self._append_messages_in_transaction(
- conn, session_id, messages, compression_lock_holder=compression_lock_holder,
- turn_lease_holder=turn_lease_holder, turn_lease_ttl_seconds=turn_lease_ttl_seconds),
- messages, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S)
- def _append_messages_in_transaction(self, conn, session_id, messages, *,
- compression_lock_holder=None, turn_lease_holder=None, turn_lease_ttl_seconds=300.0):
- self._check_transcript_write_guards(conn, session_id, compression_lock_holder,
- turn_lease_holder=turn_lease_holder, turn_lease_ttl_seconds=turn_lease_ttl_seconds)
- from agent.transcript_repair import resolve_and_repair_transcript_batch
- inserted_rows = resolve_and_repair_transcript_batch(
- conn,
- session_id,
- messages,
- encode_content_fn=self._encode_content,
- decode_content_fn=self._decode_content,
- serialize_message_fn=lambda msg, timestamp: self._serialized_message_row(
- session_id, msg, timestamp
- ),
- decode_row_fn=self._decoded_repair_row,
- )
- inserted, tool_calls_total = self._insert_message_rows(conn, session_id, inserted_rows)
- self._bump_session_counters(conn, session_id, inserted, tool_calls_total, unit=False)
- return inserted
- _ROW_STATE_KEYS = ("_row_id", DB_ROW_SNAPSHOT, "timestamp")
- def _execute_transcript_write(self, fn, messages: list[dict[str, Any]], **kwargs):
- """``_execute_write(fn)`` for callbacks that stamp row state onto the caller's *messages* (every
- :meth:`_insert_message_rows` caller that passes caller-owned dicts; rewind and import insert fresh copies). Each attempt, and a final failure, restores the caller's
- ``_row_id`` / digest / timestamp: a rolled-back insert's id is reused by SQLite, so a stale stamp
- would make a later flush adopt another writer's row and drop this message."""
- _absent = object()
- pre_state = [tuple(m.get(k, _absent) for k in self._ROW_STATE_KEYS) for m in messages]
- def _restore() -> None:
- for msg, state in zip(messages, pre_state):
- msg.pop(CANONICAL_ROW, None)
- for key, value in zip(self._ROW_STATE_KEYS, state):
- if value is _absent:
- msg.pop(key, None)
- else:
- msg[key] = value
- def _attempt(conn):
- _restore()
- return fn(conn)
- try:
- return self._execute_write(_attempt, **kwargs)
- except BaseException:
- _restore()
- raise
- def set_latest_matching_message_display_kind(self, session_id: str, *, role: str, content: str,
- display_kind: str,
- display_metadata: Optional[dict[str, Any]] = None) -> bool:
- """Stamp presentation metadata on this turn's freshly persisted row (newest active row by content,
- right after the serial turn flushed); the model still sees ``role``/``content`` unchanged, so
- producer provenance survives without classifying by content at render time."""
- if not session_id or not content or not display_kind:
- return False
- def _do(conn):
- row = conn.execute("SELECT id FROM messages WHERE session_id = ? AND role = ? "
- "AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1",
- (session_id, role, self._encode_content(content))).fetchone()
- if row is None:
- return False
- conn.execute("UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?",
- (_scrub_surrogates(display_kind), self._encode_display_metadata(display_metadata), row[0]))
- return True
- return self._execute_write(_do)
- def _reaction_list(self, meta: Optional[dict[str, Any]]) -> list[dict[str, Any]]:
- """Well-formed (dict) reactions stored under ``REACTIONS_METADATA_KEY``."""
- reactions = (meta or {}).get(self.REACTIONS_METADATA_KEY)
- return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else []
- def set_message_reaction(self, session_id: str, message_row_id: int, emoji: Optional[str], *,
- author: str = "user") -> Optional[list[dict[str, Any]]]:
- """Set (``emoji=None``: clear) *author*'s reaction. Tapback semantics: one per author per message;
- the same emoji again clears, a different one replaces. Returns the list after the write, or
- ``None`` for a row outside the session's visible resume lineage (see ``_reaction_row_query``)."""
- if not session_id or message_row_id is None:
- return None
- sql, params = self._reaction_row_query(session_id, message_row_id)
- def _do(conn):
- row = conn.execute(sql, params).fetchone()
- if row is None:
- return None
- meta = self._decode_display_metadata(row[0]) or {}
- existing = self._reaction_list(meta)
- reactions = [r for r in existing if r.get("author") != author]
- previous = next((r for r in existing if r.get("author") == author), None)
- if emoji and (previous is None or previous.get("emoji") != emoji):
- reactions.append({"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()})
- if reactions:
- meta[self.REACTIONS_METADATA_KEY] = reactions
- else:
- meta.pop(self.REACTIONS_METADATA_KEY, None)
- conn.execute(_SET_DISPLAY_META_SQL, (self._encode_display_metadata(meta) if meta else None, message_row_id))
- return reactions
- return self._execute_write(_do)
- def get_message_reactions(self, session_id: str, message_row_id: int) -> list[dict[str, Any]]:
- """Reaction list persisted on one message row (never ``None``)."""
- if not session_id or message_row_id is None:
- return []
- row = self._read_one(*self._reaction_row_query(session_id, message_row_id))
- return self._reaction_list(self._decode_display_metadata(row[0])) if row is not None else []
- def _reaction_row_query(self, session_id: str, message_row_id: int) -> tuple[str, tuple]:
- """A reaction addresses a row the client can SEE, and a display resume materializes the whole
- compression lineage (active + compacted rows, with row ids) — so a row is "in this session" when
- its owner is any lineage segment, not only the tip, and a rewound row is not. Explicit ``/branch``
- copies keep their own rows (``_resume_lineage_ids``)."""
- lineage = self._resume_lineage_ids(session_id)
- return _DISPLAY_META_ROW_SQL.format(ids=_placeholders(lineage)), (message_row_id, *lineage)
- def take_unseen_reactions(self, session_id: str, *, author: str = "user") -> list[dict[str, Any]]:
- """Return *author*'s not-yet-surfaced reactions and mark them seen. Reactions are announced on the
- NEXT user turn (never by rewriting the reacted message: cache-safe); ``seen`` makes it exactly once.
- Include compaction-archived history that remains visible, but exclude rewound/superseded rows."""
- if not session_id:
- return []
- lineage = self._resume_lineage_ids(session_id)
- def _do(conn):
- pending = []
- # Only reaction-bearing rows cross into Python: display_metadata also carries delivery /
- # attachment markers on most rows, and the lineage scan grows with the session's age.
- for row in conn.execute("SELECT id, role, content, display_metadata FROM messages "
- f"WHERE session_id IN ({_placeholders(lineage)}){_DISPLAY_ACTIVE_CLAUSE} "
- f"AND {_sql_json_extract('display_metadata', '$.' + self.REACTIONS_METADATA_KEY)} IS NOT NULL "
- "ORDER BY id", tuple(lineage)).fetchall():
- meta = self._decode_display_metadata(row["display_metadata"])
- reactions = meta.get(self.REACTIONS_METADATA_KEY) if meta else None
- if not isinstance(reactions, list):
- continue
- changed = False
- for reaction in reactions:
- if not isinstance(reaction, dict) or reaction.get("author") != author or reaction.get("seen"):
- continue
- reaction["seen"] = True
- changed = True
- content = self._decode_content(row["content"])
- pending.append({
- "row_id": row["id"], "role": row["role"], "emoji": reaction.get("emoji") or "",
- "text": content if isinstance(content, str) else ""})
- if changed:
- conn.execute(_SET_DISPLAY_META_SQL, (self._encode_display_metadata(meta), row["id"]))
- return pending
- return self._execute_write(_do)
- def latest_message_row_id(self, session_id: str, *, role: str = "user", offset: int = 0,
- require_text: bool = True) -> Optional[int]:
- """Row id of the most recent active *role* message, or ``None``. ``offset`` steps back; ``require_text``
- skips rows without plain-text content so "the latest message" never resolves to an invisible bubble."""
- if not session_id or role not in {"user", "assistant"} or offset < 0:
- return None
- text_filter = "AND content IS NOT NULL AND TRIM(content) != '' " if require_text else ""
- row = self._read_one("SELECT id FROM messages WHERE session_id = ? AND role = ? "
- f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?",
- (session_id, role, int(offset)))
- return row[0] if row else None
- def latest_conversation_role(self, session_id: str) -> Optional[str]:
- """Role of the newest active user/assistant/tool row, or ``None``. ``session_meta`` /
- ``system`` rows are transcript bookkeeping stripped before the model sees history, so
- they must not hide an open user tail from the failed-turn boundary check."""
- row = self._read_one(
- "SELECT role FROM messages WHERE session_id = ? AND active = 1 "
- "AND role NOT IN ('session_meta', 'system') ORDER BY id DESC LIMIT 1", (session_id,))
- return row[0] if row else None
- def get_message_role(self, session_id: str, row_id: int) -> Optional[str]:
- """Role of the active message at *row_id* in *session_id*, or ``None``."""
- if not session_id:
- return None
- row = self._read_one("SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1", (int(row_id), session_id))
- return row[0] if row else None
- def _carry_parent_timestamps(self, conn, parent_session_id: str, messages: list[dict[str, Any]]) -> None:
- """Adopt the durable parent row's timestamp onto carried handoff rows so the re-inserted child row keeps
- the original's display identity (see _display_dedupe_key). Idempotent; never overwrites a timestamp the
- dict already carries; rows with no match keep their caller-supplied/now_ts value.
- Content identity = _display_dedupe_key's content key minus the timestamp (role, encoded content,
- tool_call_id, encoded tool_calls, tool_name). Donors are the parent's ACTIVE rows, consumed first-match-wins in id
- order so two identical-content turns cannot both grab the same parent row. Best-effort: on any error
- the insert falls back to today's behavior (publish-time stamp) and never breaks publish.
- """
- try:
- candidates = [i for i, msg in enumerate(messages)
- if isinstance(msg, dict) and coerce_epoch(msg.get("timestamp"), field="message timestamp") is None]
- if not candidates:
- return
- donors: dict[tuple[Any, ...], list[Any]] = {}
- for row in conn.execute(
- "SELECT id, role, content, timestamp, tool_call_id, tool_calls, tool_name FROM messages "
- "WHERE session_id = ? AND active = 1 ORDER BY id", (parent_session_id,)).fetchall():
- donors.setdefault((row["role"], row["content"], row["tool_call_id"], row["tool_calls"],
- row["tool_name"]), []).append(row["timestamp"])
- for i in candidates:
- msg = messages[i]
- tool_calls = _parse_tool_calls(msg.get("tool_calls"))
- key = (msg.get("role", "unknown"), self._encode_content(msg.get("content")),
- msg.get("tool_call_id"), json.dumps(tool_calls) if tool_calls else None,
- _scrub_surrogates(msg.get("tool_name")))
- queue = donors.get(key)
- if queue:
- msg["timestamp"] = queue.pop(0)
- except Exception:
- return # Best-effort: a failed carry falls back to the publish-time stamp, never breaks publish.
- def _insert_message_rows(self, conn, session_id: str, messages: list[dict[str, Any]], *,
- prune_checkpoints: bool = True) -> tuple[int, int]:
- """Insert *messages* as fresh active rows in the caller's txn -> ``(inserted, tool_call_count)``.
- Never touches sessions.* counters (callers reconcile differently); reasoning kept for assistant rows.
- A caller that re-archives some rows afterwards passes ``prune_checkpoints=False`` and prunes once they
- are archived again (:meth:`_prune_shadowed_checkpoints`)."""
- now_ts = time.time()
- inserted = tool_calls_total = 0
- batch_tool_index: dict[str, str] = {}
- for msg in messages:
- role = msg.get("role", "unknown")
- tool_calls = _parse_tool_calls(msg.get("tool_calls"))
- message_timestamp = _coerce_timestamp(msg.get("timestamp"), now_ts)
- # The durable per-message id: kept when the dict carries one (a copy of an already-stored
- # message), minted and stamped on the dict otherwise. Stamped BEFORE the bind so the row and
- # the live dict never disagree. Tool calls get their per-occurrence ids the same way.
- stamp_message_uid(msg)
- self._stamp_tool_call_uids(msg, tool_calls, batch_tool_index)
- cur = conn.execute(_INSERT_MESSAGE_SQL, self._message_row_params(
- session_id, role, msg, tool_calls, message_timestamp, keep_reasoning=role == "assistant"))
- # Keep the caller's live row aligned with the durable identity. Rows created without an explicit
- # timestamp (notably mid-turn steers) may be carried through several compaction generations; if
- # the generated timestamp exists only in SQLite, every copy receives a new identity and renders
- # as another logical message.
- msg["timestamp"] = message_timestamp
- if cur.lastrowid is not None:
- msg["_row_id"] = cur.lastrowid
- inserted += 1
- tool_calls_total += _tool_calls_count(tool_calls)
- now_ts = max(now_ts, message_timestamp) + 1e-6
- if prune_checkpoints:
- self._prune_shadowed_checkpoints(conn, session_id, messages)
- # Every inserter (flush, compaction clone, replace, rotation handoff, import) gets the new rows' own
- # stored-row digest, so a later re-flush of these dicts takes the versioned rewrite path instead of the
- # legacy one. One batched SELECT; hash the STORED rows (column affinity rewrites bind values).
- from agent.transcript_repair import stamp_inserted_row_snapshots
- stamp_inserted_row_snapshots(conn, session_id, messages)
- return inserted, tool_calls_total
- def _prune_shadowed_checkpoints(self, conn, session_id: str, live_messages: list[dict[str, Any]]) -> None:
- """Keep only the newest checkpoint among *live_messages* (inserted rows carrying ``_row_id``)."""
- carrier = _newest_checkpoint_carrier(live_messages, "codex_reasoning_items")
- if carrier >= 0 and isinstance(live_messages[carrier].get("_row_id"), int):
- self._drop_shadowed_checkpoint_rows(conn, session_id, live_messages[carrier]["_row_id"])
- def _drop_shadowed_checkpoint_rows(self, conn, session_id: str, carrier_row_id: int) -> int:
- """Rewrite older active assistant rows so only the row *carrier_row_id* keeps a ``type: "compaction"``
- checkpoint (durable twin of ``context_compressor.drop_shadowed_checkpoints``). Under native compaction
- every assistant response re-persists a ~120 KB checkpoint the wire builder will never replay once a
- newer one lands, and local compaction — the only other prune site — rarely fires (#102374).
- Non-checkpoint items stay; returns rows rewritten."""
- rewritten = 0
- for row_id, raw in conn.execute(_SHADOWED_CHECKPOINT_ROWS_SQL, (session_id, carrier_row_id)).fetchall():
- items = _json_or(raw, None, "Ignoring malformed codex_reasoning_items on message row")
- if not isinstance(items, list) or not any(_is_checkpoint_item(item) for item in items):
- continue
- kept = [item for item in items if not _is_checkpoint_item(item)]
- conn.execute(_SET_CODEX_REASONING_SQL, (self._reasoning_json_text(kept), row_id))
- rewritten += 1
- return rewritten
- def replace_messages(self, session_id: str, messages: list[dict[str, Any]], active_only: bool = False,
- archive_dropped: bool = False, reject_active_turn_lease: bool = False) -> None:
- """Atomically replace a session's messages (/retry, /undo, /compress). DESTRUCTIVE by default (rows
- DELETEd, leave FTS). ``active_only`` spares soft-archived rows (needed with in-place compaction).
- ``archive_dropped`` SOFT-archives live rows rewind-style: what rewind/edit/regenerate must use, since
- DELETE leaves nothing to recover. ``reject_active_turn_lease``: in-txn lease check for user rewrites.
- Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of DELETEing them: the replaced
- turns stay on disk with ``active = 0``, ``compacted = 0`` — the same "the user took it back" marking
- :meth:`rewind_to_message` applies — and stay readable via :meth:`get_messages` with
- ``include_inactive=True``. This is the mode a rewind/edit/regenerate must use: those flows overwrite
- a transcript the user may not have meant to drop, and a plain DELETE also evicts the rows from the
- FTS index, leaving nothing to recover from (#82756). It implies active-only handling —
- already-archived rows are never touched — so ``active_only`` is redundant with it. Live rows that
- *messages* keeps in place (the common in-order prefix, matched on role/content/tool identity) are
- left untouched with their ids; only the divergent live suffix is archived and only the new suffix
- of *messages* is inserted (#82956: archiving and re-inserting the kept prefix grew the archive by
- the whole transcript on every rewind). The live view is identical either way; only the durability
- of the dropped turns differs.
- """
- def _do(conn):
- self.replace_messages_in_transaction(
- conn, session_id, messages, active_only=active_only, archive_dropped=archive_dropped,
- reject_active_turn_lease=reject_active_turn_lease)
- self._execute_transcript_write(_do, messages)
- def replace_messages_in_transaction(self, conn, session_id: str, messages: list[dict[str, Any]], *,
- active_only: bool = False, archive_dropped: bool = False,
- reject_active_turn_lease: bool = False,
- expected_active_ids: Optional[list[int]] = None) -> None:
- """:meth:`replace_messages` inside the caller's write transaction (the gateway ``/retry``
- rewrite commits in its re-send's own admission). ``expected_active_ids`` pins the active set
- the replacement was computed from: a row committed since refuses (``RuntimeError``)."""
- from hermes_state_errors import CompressionSessionClosedError
- if reject_active_turn_lease:
- self._check_transcript_write_guards(
- conn, session_id, None, reject_active_turn_lease=True, reject_active_compression_lock=True)
- elif _ended_by_compression(conn.execute(_ENDED_ROW_SQL, (session_id,)).fetchone()):
- raise CompressionSessionClosedError(session_id)
- if expected_active_ids is not None and [
- int(row[0]) for row in conn.execute(_ACTIVE_IDS_SQL, (session_id,))] != expected_active_ids:
- raise RuntimeError("active transcript changed before the rewrite could be persisted")
- kept = kept_tool_calls = 0
- if archive_dropped:
- # Only the first len(messages)+1 live rows matter: the prefix to match plus the row whose
- # id anchors the archive UPDATE (which itself covers every later row via `id >= ?`).
- live = conn.execute(_LIVE_IDENTITY_SQL, (session_id, len(messages) + 1)).fetchall()
- kept = self._stamp_kept_live_prefix(live, messages)
- kept_tool_calls = sum(_tool_calls_len(row[4], scalar=1) for row in live[:kept])
- if kept < len(live):
- # FTS triggers don't fire on `active`: replaced turns stay searchable (include_inactive=True).
- conn.execute("UPDATE messages SET active = 0 WHERE session_id = ? AND active = 1 AND id >= ?",
- (session_id, live[kept][0]))
- else:
- conn.execute(f"DELETE FROM messages WHERE session_id = ?{' AND active = 1' if active_only else ''}", (session_id,))
- inserted, inserted_tool_calls = self._insert_message_rows(conn, session_id, messages[kept:])
- conn.execute(f"{_SET_COUNTERS_SQL} WHERE id = ?",
- (kept + inserted, kept_tool_calls + inserted_tool_calls, session_id))
- @classmethod
- def _row_identity(cls, role: str, content: Any, tool_call_id: Any, tool_calls: Any) -> tuple:
- """The (role, content, tool_call_id, tool_calls) columns a message writes — the identity the
- kept-prefix match compares. *content* is passed through the loader's lens first so a message
- read back from the DB matches the row it came from."""
- return (role, cls._encode_content(cls._loaded_view_content(role, content)), tool_call_id,
- json.dumps(tool_calls) if tool_calls else None)
- def _stamp_kept_live_prefix(self, live: list, messages: list[dict[str, Any]]) -> int:
- """Length of the in-order prefix of *messages* already present as the leading live rows; each
- matched message gets its existing ``_row_id`` (and ``message_uid``) stamped, as a fresh insert would set them."""
- kept = 0
- for row, msg in zip(live, messages):
- role = msg.get("role", "unknown")
- # Cheap scalars first; the content compare pays decode/sanitize/encode on both sides.
- if row[1] != role or row[3] != msg.get("tool_call_id"):
- break
- identity = self._row_identity(role, msg.get("content"), msg.get("tool_call_id"),
- _parse_tool_calls(msg.get("tool_calls")))
- if identity != self._row_identity(row[1], self._decode_content(row[2]), row[3], _parse_tool_calls(row[4])):
- break
- msg["_row_id"] = row[0]
- if row[5]:
- msg[MESSAGE_UID] = row[5]
- kept += 1
- return kept
- @staticmethod
- def _loaded_view_content(role: str, content: Any) -> Any:
- """Content as ``_rows_to_messages`` hands it to callers: user/assistant strings are sanitized and
- stripped on load, so a reloaded session's messages must be compared to rows through the same lens."""
- if role in {"user", "assistant"} and isinstance(content, str):
- return sanitize_context(content).strip()
- return content
- def has_archived_messages(self, session_id: str) -> bool:
- """True if the session has any soft-archived (``active = 0``) rows (tests/diagnostics).
- Cheap existence probe — does not load rows. NOTE: production rewrite paths no longer branch on this
- (they pass ``active_only=True`` unconditionally — a probe can fail open or race a concurrent
- ``archive_and_compact``, #80216); kept for tests and diagnostics.
- """
- return self._read_one(
- "SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1", (session_id,)) is not None
- def get_active_message_watermark(self, session_id: str) -> int:
- """MAX(id) of the active rows (0 if none), captured at compression START: every active row above it
- arrived concurrently and must survive compaction verbatim."""
- if not session_id:
- return 0
- return int(self._read_one(
- "SELECT COALESCE(MAX(id), 0) FROM messages WHERE session_id = ? AND active = 1", (session_id,))[0])
- def _tail_rows_after_watermark(self, conn, sql: str, params) -> tuple[list[int], int]:
- """``(ids, tool_call_count)`` of the concurrent-tail rows selected by *sql* (``SELECT id, tool_calls``)."""
- rows = conn.execute(sql, params).fetchall()
- return [int(r["id"]) for r in rows], sum(_tool_calls_len(r["tool_calls"]) for r in rows)
- def _clone_message_rows(self, conn, tail_ids: list[int], *, session_id: Optional[str] = None) -> None:
- """Pure-SQL clone of *tail_ids* as fresh live rows (new id/display order, active=1, compacted=0;
- message payload columns stay byte-exact and FTS triggers index the clones), into *session_id* when given."""
- retarget = session_id is not None
- # A clone is a newly positioned display generation. Copy its indexed
- # identity, but let the insert trigger assign order from rows that are
- # still display-visible (the source may just have become rewind-only).
- skip = ("id", "active", "compacted", "display_order") + (("session_id",) if retarget else ())
- col_list = ", ".join(c for c in self._message_column_names(conn) if c not in skip)
- conn.execute(
- f"INSERT INTO messages ({col_list}, {'session_id, ' if retarget else ''}active, compacted) "
- f"SELECT {col_list}, {'?, ' if retarget else ''}1, 0 FROM messages "
- f"WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id",
- [session_id, *tail_ids] if retarget else tail_ids)
- def _archive_named_rows(
- self, conn, session_id: str, compacted_messages: list[dict[str, Any]], covered: list[int], *,
- tail_count: int, carried_messages: Optional[list[dict[str, Any]]], patched_model_config: Any,
- patch: bool, merged_away: set[int],
- ) -> int:
- """Archive *covered* as summarized and clone every other active row after the new set.
- A gap below the newest held id, and rows appended after an unpersisted turn, are not
- in *covered*. They take the concurrent-append path: rewind the original, insert the
- compacted transcript, then clone them so they stay live and searchable once each.
- """
- active_ids = [int(row["id"]) for row in conn.execute(_ACTIVE_IDS_SQL, (session_id,)).fetchall()]
- covered_set = set(covered)
- carried_ids = self._resolve_carried_row_ids(conn, session_id, carried_messages or [])
- covered_set.update(carried_ids)
- unseen = [row_id for row_id in active_ids if row_id not in covered_set]
- covered_active = [row_id for row_id in active_ids if row_id in covered_set]
- # An unseen row the compacted set already carries is superseded by that copy, not cloned (#137197).
- carried_unseen, weak_claims = self._carried_tail_ids(conn, unseen, compacted_messages)
- clone_ids = [row_id for row_id in unseen if row_id not in carried_unseen]
- tail_below = tail_count - self._carried_tail_slots(carried_unseen, len(compacted_messages), tail_count)
- rewind_ids = list(carried_ids)
- if tail_below > 0:
- rewind_ids += self._tail_originals(covered_active, tail_below, merged_away)
- # A weak claim (a rewritten carry naming the row) is archived as compacted history, never rewound out of display.
- rewind_ids += [row_id for row_id in unseen if row_id not in weak_claims]
- rewind_ids = list(dict.fromkeys(rewind_ids))
- if rewind_ids:
- placeholders = _placeholders(rewind_ids)
- conn.execute(
- "UPDATE messages SET active = 0, compacted = 0 "
- f"WHERE session_id = ? AND id IN ({placeholders})",
- [session_id, *rewind_ids])
- conn.execute(_ARCHIVE_ACTIVE_SQL, (session_id,))
- # Same identity carry as publish_compression_child (#59661): carried tail copies keep the
- # durable row's timestamp so display dedupe still collapses generations. Donors are active-only;
- # the rewound/archived originals above never donate, so this is a no-op for a clean compaction.
- self._carry_parent_timestamps(conn, session_id, compacted_messages)
- inserted, tool_calls_total = self._insert_message_rows(conn, session_id, compacted_messages)
- if clone_ids:
- _ids, unseen_tool_calls = self._tail_rows_after_watermark(
- conn,
- f"SELECT id, tool_calls FROM messages WHERE id IN ({_placeholders(clone_ids)}) ORDER BY id",
- tuple(clone_ids))
- self._clone_message_rows(conn, clone_ids)
- inserted += len(clone_ids)
- tool_calls_total += unseen_tool_calls
- # A carried copy whose stored identity was computed differently lands in its own
- # display_order group and would project twice; re-fold before publishing (#122167).
- self._reconcile_display_orders(conn, session_id)
- conn.execute(
- f"{_SET_COUNTERS_SQL}{', model_config = ?' if patch else ''} WHERE id = ?",
- (inserted, tool_calls_total, *((patched_model_config,) if patch else ()), session_id))
- return inserted
- def archive_and_compact(self, session_id: str, compacted_messages: list[dict[str, Any]],
- model_config_patch: Optional[dict[str, Any]] = None, watermark: Optional[int] = None,
- lock_holder: Optional[str] = None, tail_count: int = 0,
- carried_messages: Optional[list[dict[str, Any]]] = None,
- covered_ids: Optional[list[int]] = None,
- unresolved_held: Optional[list[dict[str, Any]]] = None,
- held_for_cas: Optional[list[dict[str, Any]]] = None) -> int:
- """Non-destructive in-place compaction under ONE session id: soft-archive the active rows (``active=0,
- compacted=1``: summarized away, still searchable) and insert *compacted_messages* as fresh active
- rows, atomically; returns the new ACTIVE count (= ``message_count``). *watermark* (compression
- START): rows ``id > watermark`` arrived during the slow summary and are re-sequenced after the
- compacted set by a pure-SQL clone (fresh ids); ``None`` archives everything. *covered_ids*: the
- rows the compressor actually held. When proved, only those are summarized; every other active
- row is cloned after the new set, so a gap below the newest held id is not archived unseen.
- ``None`` keeps the watermark path. *unresolved_held*: held dicts with no row id, matched inside
- the transaction. *lock_holder*: verified
- in-txn so a reclaimed lease fails instead of clobbering the winner. *tail_count*: the LAST N compacted
- rows are the verbatim carried tail; *carried_messages* names exact durable originals carried forward
- verbatim when they are not a contiguous suffix (micro-compaction's prefix + marker + suffix shape).
- They are resolved inside this transaction by row id when present, else by unique durable identity +
- timestamp. Those originals and the clones' originals are superseded duplicates and get rewind flags
- (``active=0, compacted=0``) so search doesn't return each carried message once per compaction.
- ``held_for_cas``: the caller's held message dicts for the lease-less staleness gate — re-verified
- against the live rows INSIDE this transaction (see :meth:`held_row_mismatches`), so a rewrite that
- lands after the caller's precheck aborts the commit as :class:`HeldRowRewrittenError` before any
- row is archived (#124102). The lease path leaves it ``None``: its lease rules out an overlapping
- rewrite, and the turn's own writes are the ones a CAS here would misread.
- ``model_config_patch`` merges in the same txn (``None`` removes a key).
- Concurrent-append safety (#75316): when *watermark* is provided (the value of
- :meth:`get_active_message_watermark` captured at compression START), rows that arrived during the
- slow provider summary call (``id > watermark``) are NOT summarized away. They are re-sequenced after
- the compacted set by a pure-SQL column clone (every column except ``id`` — content, api_content,
- platform_message_id, token counts, reasoning sidecars all survive byte-exact, and the FTS triggers
- index the clones naturally), and the originals are archived. NOTE: re-sequencing assigns the tail
- rows fresh ids; consumers that reference durable row ids re-resolve by content (see 3e8ab0610).
- A tail row that *compacted_messages* already carries (the compressor held it, and it was flushed
- above the watermark or left unnamed) is not cloned: the carried copy is its one live row and the
- original takes rewind flags (#137197).
- """
- return self._execute_transcript_write(lambda conn: self._archive_and_compact_on_conn(
- conn, session_id, compacted_messages, model_config_patch=model_config_patch,
- watermark=watermark, lock_holder=lock_holder, tail_count=tail_count,
- carried_messages=carried_messages, covered_ids=covered_ids,
- unresolved_held=unresolved_held, held_for_cas=held_for_cas), compacted_messages)
- def _archive_and_compact_on_conn(
- self, conn, session_id: str, compacted_messages: list[dict[str, Any]], *,
- model_config_patch: Optional[dict[str, Any]] = None, watermark: Optional[int] = None,
- lock_holder: Optional[str] = None, tail_count: int = 0,
- carried_messages: Optional[list[dict[str, Any]]] = None,
- covered_ids: Optional[list[int]] = None,
- unresolved_held: Optional[list[dict[str, Any]]] = None,
- held_for_cas: Optional[list[dict[str, Any]]] = None) -> int:
- """:meth:`archive_and_compact` on the caller's write transaction (never commits). The owner
- and the worker/canonical-mutation verbs share this one body so they cannot drift."""
- from hermes_state import SessionCompressionInProgressError
- if lock_holder is not None:
- lock_row = conn.execute(_COMPRESSION_LOCK_ROW_SQL, (session_id,)).fetchone()
- if lock_row is None or lock_row["holder"] != lock_holder or float(lock_row["expires_at"]) <= time.time():
- raise SessionCompressionInProgressError(
- f"Compression lease for {session_id!r} lost before commit; refusing to publish a stale compaction")
- patch = model_config_patch is not None
- # on_missing="raise": never commit against a vanished session row (caller keeps the original).
- patched_model_config = self._merge_model_config_json(
- conn, session_id, model_config_patch, on_missing="raise") if patch else None
- # The in-transaction half of the lease-less staleness gate (#124102): the caller's
- # rewritten_held_row_ids precheck runs outside the write lock, so a concurrent in-place
- # rewrite landing between precheck and BEGIN IMMEDIATE was archived and its stale held
- # body republished. Re-verify on the connection this commit holds, before any row moves.
- if held_for_cas:
- mismatches = getattr(self, "held_row_mismatches", None)
- if callable(mismatches) and mismatches(conn, session_id, held_for_cas):
- from hermes_state_errors import HeldRowRewrittenError
- raise HeldRowRewrittenError(
- f"a held row of session {session_id} was rewritten in place before the commit")
- proved = self._proved_coverage(conn, session_id, covered_ids, unresolved_held, watermark)
- if proved is not None:
- return self._archive_named_rows(
- conn, session_id, compacted_messages, proved[0], tail_count=tail_count,
- carried_messages=carried_messages, patched_model_config=patched_model_config, patch=patch,
- merged_away=proved[1])
- tail_ids: list[int] = [] if watermark is None else [int(row["id"]) for row in conn.execute(
- "SELECT id FROM messages WHERE session_id = ? AND active = 1 AND id > ? ORDER BY id",
- (session_id, int(watermark))).fetchall()]
- # A concurrent-tail row the compacted set already carries is superseded by that copy, not
- # cloned beside it (#137197); its carried dict then has no original below the watermark.
- carried_tail, weak_claims = self._carried_tail_ids(conn, tail_ids, compacted_messages)
- clone_ids = [row_id for row_id in tail_ids if row_id not in carried_tail]
- _ids, tail_tool_calls = ([], 0) if not clone_ids else self._tail_rows_after_watermark(
- conn, f"SELECT id, tool_calls FROM messages WHERE id IN ({_placeholders(clone_ids)}) ORDER BY id",
- tuple(clone_ids))
- tail_below = tail_count - self._carried_tail_slots(carried_tail, len(compacted_messages), tail_count)
- # Rewind targets sit AT/BELOW the watermark (all the compressor saw); unbounded, a
- # concurrent append would steal a LIMIT slot.
- rewind_ids: list[int] = self._resolve_carried_row_ids(
- conn, session_id, carried_messages or [])
- if tail_below > 0:
- bound = watermark is not None
- rewind_ids += [int(row["id"]) for row in conn.execute(
- f"SELECT id FROM messages WHERE session_id = ? AND active = 1{' AND id <= ?' if bound else ''} "
- "ORDER BY id DESC LIMIT ?",
- (session_id, *((int(watermark),) if bound else ()),
- int(tail_below) + self._uncounted_merged_rows(compacted_messages[-int(tail_count):]))).fetchall()]
- # A weak claim (a rewritten carry naming the row) is archived as compacted history, never rewound out of display.
- rewind_ids += [row_id for row_id in tail_ids if row_id not in weak_claims]
- rewind_ids = list(dict.fromkeys(rewind_ids))
- if rewind_ids:
- placeholders = _placeholders(rewind_ids)
- conn.execute("UPDATE messages SET active = 0, compacted = 0 "
- f"WHERE session_id = ? AND id IN ({placeholders})", [session_id, *rewind_ids])
- conn.execute(f"{_ARCHIVE_ACTIVE_SQL} AND id NOT IN ({placeholders})", [session_id, *rewind_ids])
- else:
- conn.execute(_ARCHIVE_ACTIVE_SQL, (session_id,))
- # Same identity carry as publish_compression_child (#59661): carried tail copies keep the
- # durable row's timestamp so display dedupe still collapses generations. Donors are active-only;
- # the rewound/archived originals above never donate, so this is a no-op for a clean compaction.
- self._carry_parent_timestamps(conn, session_id, compacted_messages)
- inserted, tool_calls_total = self._insert_message_rows(conn, session_id, compacted_messages)
- if clone_ids:
- self._clone_message_rows(conn, clone_ids)
- inserted += len(clone_ids)
- tool_calls_total += tail_tool_calls
- # A carried copy whose stored identity was computed differently lands in its own
- # display_order group and would project twice; re-fold before publishing (#122167).
- self._reconcile_display_orders(conn, session_id)
- conn.execute(f"{_SET_COUNTERS_SQL}{', model_config = ?' if patch else ''} WHERE id = ?",
- (inserted, tool_calls_total, *((patched_model_config,) if patch else ()), session_id))
- return inserted
- def _message_column_names(self, conn) -> list[str]:
- """Column names of the messages table, cached per-connection era."""
- if not getattr(self, "_message_columns_cache", None):
- self._message_columns_cache = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()]
- return self._message_columns_cache
- def clear_user_api_content_with_prefix(self, session_id: str, prefix: str) -> int:
- """Drop user-row ``api_content`` sidecars that begin with ``prefix`` across the resume lineage.
- Yaovi sends its per-turn application state only with the live request; later turns must
- replay the clean ``content`` rather than that state. Sidecars without the prefix
- (cache-fidelity stamps, sanitize divergences) and rows written before the prefix existed
- are left untouched.
- """
- ids = [sid for sid in dict.fromkeys(self._resume_lineage_ids(session_id)) if sid]
- if not ids or not prefix:
- return 0
- marks = ",".join("?" * len(ids))
- return self._write_rowcount(
- f"UPDATE messages SET api_content = NULL WHERE session_id IN ({marks}) AND role = 'user' "
- "AND api_content IS NOT NULL AND substr(api_content, 1, ?) = ?",
- (*ids, len(prefix), prefix))
- def set_user_message_content(self, session_id: str, row_id: int, content: Any,
- display_metadata: Optional[dict[str, Any]] = None) -> int:
- """Rewrite the content of ONE known active user row. Used when a user turn was written at submit
- time (before the agent ran) and the turn prologue then rewrote the prompt it persists (@-file
- expansion, native image parts, a ``pre_llm_call`` veto's placeholder): the early row must show what
- the transcript will replay, not the raw keystrokes, and the turn must not append a second row for
- the same input. ``display_metadata``, when given, replaces the row's (and clears ``api_content``)."""
- if not session_id or isinstance(row_id, bool) or not isinstance(row_id, int) or row_id <= 0:
- return 0
- if display_metadata is None:
- return self._write_rowcount(
- "UPDATE messages SET content = ? WHERE id = ? AND session_id = ? AND role = 'user' AND active = 1",
- (self._encode_content(content), row_id, session_id))
- return self._write_rowcount(
- "UPDATE messages SET content = ?, api_content = NULL, display_metadata = ? "
- "WHERE id = ? AND session_id = ? AND role = 'user' AND active = 1",
- (self._encode_content(content), self._encode_display_metadata(display_metadata), row_id, session_id))
- def deactivate_message(self, session_id: str, row_id: int) -> int:
- """Deactivate ONE known row (id-addressed, idempotent; returns the affected row count). Used by
- the queued-prompt drain: the row written at accept time sits ahead of the in-flight turn's
- assistant reply, and the drain re-appends an identical row at the transcript end — leaving the
- early row active would put two user rows before that reply and the alternation repair would
- glue the two turns into one. The durable row is preserved (inactive), never deleted."""
- if not session_id or isinstance(row_id, bool) or not isinstance(row_id, int) or row_id <= 0:
- return 0
- return self._write_rowcount(
- "UPDATE messages SET active = 0 WHERE id = ? AND session_id = ?",
- (row_id, session_id))
- def resolve_active_row_id(self, session_id: str, row_id: int) -> Optional[int]:
- """The active row that still carries *row_id*'s message: *row_id* itself while active, else the one
- row an in-place compaction re-sequenced it into (``_clone_message_rows`` copies role, content and
- timestamp byte-exact to a higher id). ``None`` when neither exists or the clone is ambiguous.
- A caller holding a row id across a compaction (the queued-prompt envelope) re-resolves it here
- before deactivating or rewriting the row (#123675)."""
- if not session_id or isinstance(row_id, bool) or not isinstance(row_id, int) or row_id <= 0:
- return None
- origin = self._read_one("SELECT active FROM messages WHERE id = ? AND session_id = ?", (row_id, session_id))
- if origin is None:
- return None
- if origin[0]:
- return row_id
- clones = self._read_all(
- "SELECT c.id FROM messages c JOIN messages o ON o.id = ? "
- "WHERE c.session_id = ? AND c.active = 1 AND c.id > o.id AND c.role = o.role "
- "AND c.content IS o.content AND c.timestamp = o.timestamp",
- (row_id, session_id))
- return int(clones[0][0]) if len(clones) == 1 else None
- def deactivate_messages_by_display_kind(self, session_id: str, display_kind: str) -> int:
- """Deactivate every live row of one ``display_kind`` (idempotent; returns the affected row count).
- The durable counterpart to the in-memory strip a self-replacing pivot performs: the in-memory path
- drops the prior entry so N pivots leave one, but a durable append has no such step, so every switch
- left another active row and all of them replayed on resume. Rows are preserved (inactive), never
- deleted — the same contract as :meth:`deactivate_message`, keyed by class instead of by id.
- """
- if not session_id or not display_kind:
- return 0
- return self._write_rowcount(
- "UPDATE messages SET active = 0 WHERE session_id = ? AND display_kind = ? AND active = 1",
- (session_id, _scrub_surrogates(display_kind)))
- def _display_dedupe_key(self, row) -> tuple[Any, ...]:
- """Historical display identity, including normalized live content from user handoff carriers."""
- dedupe_content = row["content"]
- if row["role"] == "user":
- handoff, live_view = split_user_originated_turn({
- "role": "user", "content": self._decode_content(row["content"]),
- "display_kind": row["display_kind"],
- "display_metadata": self._decode_display_metadata(row["display_metadata"])})
- if handoff is not None and live_view is not None:
- dedupe_content = self._encode_content(live_view.get("content"))
- return _stable_tool_key(row) or (
- row["role"], dedupe_content, row["timestamp"], row["tool_call_id"], row["tool_calls"], row["tool_name"])
- def _is_model_only_row(self, row) -> bool:
- """Python twin of :data:`DISPLAY_VISIBLE_SQL`."""
- return bool((self._decode_display_metadata(row["display_metadata"]) or {}).get(MODEL_ONLY_DISPLAY_METADATA_KEY))
- @staticmethod
- def _display_identity(key: tuple[Any, ...]) -> bytes:
- """Fixed-width durable identity for indexed display-generation lookup."""
- return hashlib.sha256(repr(key).encode("utf-8", "surrogatepass")).digest()
- @staticmethod
- def _fold_summary_carrier(row, key: Any, first_by_uid: dict) -> Any:
- """The display identity both display reads give *row*, scanning rows in ``id`` order.
- A compaction carrier keeps its message's ``message_uid`` but not its content (the prior tail
- wrapped around the summary), so it is a copy of the first row with that uid: shown once."""
- uid = row["message_uid"]
- if not uid:
- return key
- if row["_compressed_summary"]:
- return first_by_uid.get(uid, key)
- first_by_uid.setdefault(uid, key)
- return key
- def _dedupe_display_generations(self, rows, *, drop_removed: bool = False):
- """Collapse compaction generations so each logical message appears once (the protected tail is copied
- into each generation: same role/content/timestamp, different ``active``/id); prefer the live row, then
- the newest. The ONE definition every display projection shares. *rows* must be ordered by ``id``.
- *drop_removed*: *rows* also hold the removed rows (``active=0, compacted=0``: rewound, or replaced by a
- newer copy) and ``compacted``. A message whose newest copy is removed is not shown, so a rewind in a
- compression child never brings back the parent's copy. Copies share the display key or ``message_uid``
- (a micro-compaction merge also stands for the rows it absorbed)."""
- seen: dict[tuple[Any, ...], Any] = {}
- first_id: dict[tuple[Any, ...], int] = {}
- uid_key: dict[Any, tuple[Any, ...]] = {} # message_uid -> the display key of its first copy
- removed: dict[Any, bool] = {} # copy identity -> its newest copy is removed (rows come in id order)
- identities: dict[tuple[Any, ...], list[Any]] = {}
- removed_ts: set[Any] = set()
- rows = rows if isinstance(rows, list) else list(rows)
- later_copies = later_tool_copy_ids(self, rows) # a compacted tool result paints once, as its original
- if drop_removed:
- # Every display key carries the row's timestamp third, so a live row can share a removed row's identity
- # only at one of the removed rows' timestamps; hash just those (a sha256 per live row cost a warm
- # 30k-row resume ~0.17 s, RT-39). No removed row at all: nothing can hide a copy.
- removed_ts = {row["timestamp"] for row in rows if not row["active"] and not row["compacted"]}
- drop_removed = bool(removed_ts)
- for row in rows:
- copies, key = None, None
- if drop_removed:
- # A removed row is read without its body (``_removed_identity_rows``): its stored display identity.
- gone = not row["active"] and not row["compacted"]
- stored = row["display_identity"] if gone else None
- key = None if stored else self._display_dedupe_key(row)
- hashed = stored or (self._display_identity(key) if key is not None and (
- gone or row["timestamp"] in removed_ts) else None)
- copies = [*((hashed,) if hashed else ()), *(
- ("uid", uid) for uid in (row["message_uid"], *_uid_list(row["absorbed_message_uids"])) if uid)]
- removed.update(dict.fromkeys(copies, gone))
- if gone:
- continue
- if self._is_model_only_row(row) or row["id"] in later_copies:
- continue
- key = key or self._display_dedupe_key(row)
- # Only the summary carrier folds by uid: other rewritten copies (a pruned tool stub) must keep
- # their archived original visible.
- key = self._fold_summary_carrier(row, key, uid_key)
- cur = seen.get(key)
- if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]):
- seen[key] = row
- identities[key] = copies
- first_id[key] = min(first_id.get(key, row["id"]), row["id"])
- # Order by the logical message's FIRST row, not the chosen representative's: a protected-tail
- # copy in a newer generation has a higher id than messages emitted after the original.
- keys = sorted(seen, key=first_id.__getitem__)
- if drop_removed:
- keys = [key for key in keys if not any(removed[copy] for copy in identities[key])]
- return [seen[key] for key in keys]
- def _reconcile_display_orders(self, conn, session_id: str) -> None:
- """Re-fold split display generations by the recomputed display key.
- A compaction generation that computed the display identity differently (or cloned
- without inheriting it) leaves one logical message in two ``display_order`` groups,
- and ``GROUP BY display_order`` then projects it twice (#122167). Folding by the same
- recomputed key :meth:`_dedupe_display_generations` uses keeps every display projection
- on one definition of a logical message; the live copy wins its group via the read
- path's ``ORDER BY candidate.active DESC, candidate.id DESC``. Writes only on drift."""
- first_id: dict[bytes, int] = {}
- uid_identity: dict[str, bytes] = {} # message_uid -> its first copy's identity (the carrier rule)
- last_id = 0
- while True:
- rows = conn.execute(
- "SELECT id, role, content, timestamp, tool_call_id, tool_calls, tool_name, "
- "display_kind, display_metadata, display_order, display_identity, message_uid, _compressed_summary "
- "FROM messages INDEXED BY idx_messages_session_id "
- "WHERE session_id = ? AND id > ? AND (active = 1 OR compacted = 1) "
- "ORDER BY id LIMIT 1000",
- (session_id, last_id))
- batch_start = last_id
- updates = []
- for row in rows:
- last_id = row["id"]
- identity = self._fold_summary_carrier(
- row, self._display_identity(self._display_dedupe_key(row)), uid_identity)
- # A row already holding this identity keeps its stored slot (an in-place prune changes a
- # row's content, not its place); otherwise the slot is the identity's first row.
- kept = row["display_order"] if row["display_identity"] == identity else None
- order = first_id.setdefault(identity, last_id if kept is None else kept)
- if order != row["display_order"] or identity != row["display_identity"]:
- updates.append((order, identity, last_id))
- rows.close()
- if last_id == batch_start:
- break
- conn.executemany(
- "UPDATE messages SET display_order = ?, display_identity = ? WHERE id = ?", updates)
- def _ensure_display_order(self, session_id: str) -> bool:
- """Backfill one legacy session once, preserving the pre-index display identity exactly."""
- with self._read_ctx() as conn:
- columns = set(self._message_column_names(conn))
- if not {"display_order", "display_identity"} <= columns:
- return False
- if self._read_one(_DISPLAY_INDEX_MISSING_SQL, (session_id,)) is None:
- return True
- if getattr(self, "read_only", False):
- return False
- def _do(conn):
- missing = conn.execute(_DISPLAY_INDEX_MISSING_SQL, (session_id,)).fetchone()
- if missing is None:
- return True
- self._reconcile_display_orders(conn, session_id)
- return True
- return bool(self._execute_write(_do))
- def _legacy_display_page(self, session_id: str, *, active_clause: str, limit: Optional[int], offset: int,
- latest: bool) -> list[Any]:
- """Project a legacy read-only display page without retaining transcript payloads."""
- representatives: dict[bytes, tuple[int, int]] = {}
- with self._read_ctx() as conn:
- conn.execute("BEGIN")
- try:
- has_session_index = conn.execute(
- "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?",
- ("idx_messages_session_id",),
- ).fetchone() is not None
- index_hint = "INDEXED BY idx_messages_session_id" if has_session_index else "NOT INDEXED"
- rows = conn.execute(
- "SELECT id, role, content, timestamp, tool_call_id, tool_calls, tool_name, active, "
- f"display_kind, display_metadata FROM messages {index_hint} "
- f"WHERE session_id = ?{active_clause} ORDER BY id ASC",
- (session_id,))
- for row in rows:
- if self._is_model_only_row(row):
- continue
- identity = self._display_identity(self._display_dedupe_key(row))
- current = representatives.get(identity)
- candidate = (row["active"], row["id"])
- if current is None or candidate > current:
- representatives[identity] = candidate
- rows.close()
- identities = list(representatives)
- identities = identities[::-1][offset:][:limit][::-1] if latest else identities[offset:][:limit]
- selected_ids = [representatives[identity][1] for identity in identities]
- selected = {}
- for start in range(0, len(selected_ids), 900):
- chunk = selected_ids[start:start + 900]
- selected.update({row["id"]: row for row in conn.execute(
- f"SELECT * FROM messages WHERE session_id = ?{active_clause} "
- f"AND id IN ({_placeholders(chunk)})",
- (session_id, *chunk))})
- return [selected[row_id] for row_id in selected_ids if row_id in selected]
- finally:
- if conn.in_transaction:
- conn.execute("ROLLBACK")
- def _row_to_message_dict(self, row, *, warn_context: str, summary_flag: bool) -> dict[str, Any]:
- """``dict(row)`` with content/tool_calls/display_metadata decoded; *summary_flag* keeps
- ``_compressed_summary`` only as ``True``."""
- msg = dict(row)
- msg.pop("display_identity", None)
- msg.pop("display_order", None)
- if summary_flag and msg.pop("_compressed_summary", 0):
- msg["_compressed_summary"] = True
- msg["content"] = self._decode_content(msg["content"])
- if msg.get("tool_calls"):
- msg["tool_calls"] = _json_or(
- msg["tool_calls"], [], f"Failed to deserialize tool_calls in {warn_context}, falling back to []")
- if msg.get("display_metadata") is not None:
- msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"])
- # A `SELECT *` picks up every column, including any BLOB added to the schema later; the
- # JSON encoder that serves these dicts over HTTP fails outright on raw bytes. Drop them
- # here, once, rather than needing a new named pop for each future binary column. Known
- # columns are exempt: popping `content` because a row holds bytes turns a decode problem
- # into a KeyError for every msg["content"] reader downstream.
- for key, value in list(msg.items()):
- if key not in _MESSAGE_SCHEMA_KEYS and isinstance(value, (bytes, bytearray)):
- msg.pop(key)
- return msg
- @staticmethod
- def _display_rows_from_conn(conn, session_id: str, *, limit: Optional[int] = None,
- offset: int = 0, latest: bool = False):
- """One display-history projection for normal reads and transactional verification."""
- direction = "DESC" if latest else "ASC"
- return conn.execute(
- f"""WITH page AS (
- SELECT display_order FROM messages
- WHERE session_id = ? AND (active = 1 OR compacted = 1){DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql("messages")}
- GROUP BY display_order ORDER BY display_order {direction}
- LIMIT ? OFFSET ?
- )
- SELECT chosen.* FROM page
- JOIN messages AS chosen ON chosen.id = (
- SELECT candidate.id FROM messages AS candidate
- WHERE candidate.session_id = ?
- AND candidate.display_order = page.display_order
- AND (candidate.active = 1 OR candidate.compacted = 1){DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql("candidate")}
- ORDER BY candidate.active DESC, candidate.id DESC LIMIT 1
- )
- ORDER BY page.display_order ASC""",
- (session_id, -1 if limit is None else limit, offset, session_id),
- ).fetchall()
- def display_message_count(self, session_id: str) -> int:
- """Rows a display read of this segment paints: one per ``display_order`` group of
- ``_display_rows_from_conn``'s set. Unindexed legacy rows count as one group (the read
- backfills them), so the count is zero exactly when the read paints nothing."""
- row = self._read_one(
- "SELECT COUNT(*) FROM (SELECT DISTINCT display_order FROM messages WHERE session_id = ?"
- f"{_DISPLAY_ACTIVE_CLAUSE}{DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql('messages')})", (session_id,))
- return int(row[0])
- def _display_messages_from_conn(self, conn, session_id: str) -> Optional[list[dict[str, Any]]]:
- """Exact display snapshot on an already-held transaction; None means fail closed."""
- if conn.execute("SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)).fetchone() is None:
- return None
- if conn.execute(_DISPLAY_INDEX_MISSING_SQL, (session_id,)).fetchone():
- return None
- return [
- self._row_to_message_dict(row, warn_context="verified delete", summary_flag=True)
- for row in self._display_rows_from_conn(conn, session_id)
- ]
- @staticmethod
- def _active_clause(include_inactive: bool, include_compacted: bool) -> str:
- """Audit: every row; display: active plus compaction-archived (never Undo/Rewind rows); default: live."""
- return "" if include_inactive else (_DISPLAY_ACTIVE_CLAUSE if include_compacted else " AND active = 1")
- def get_messages(self, session_id: str, include_inactive: bool = False, include_compacted: bool = False,
- limit: Optional[int] = None, offset: int = 0, latest: bool = False,
- after_id: Optional[int] = None, include_ancestors: bool = False) -> list[dict[str, Any]]:
- """Load messages in insertion order (id, never timestamp: clocks regress). ``include_inactive``:
- rewind rows too; ``include_compacted``: compaction-archived display history (not rewind rows).
- ``latest`` pages back from the newest but returns chronological order; ``after_id``: keyset paging.
- ``include_ancestors``: also load the compression lineage (root → tip, branch sessions exempt),
- mirroring ``get_messages_as_conversation(include_ancestors=True)`` — after a compression rotation
- the full transcript spans parent sessions, not just the child continuation (#51058)."""
- if after_id is not None and (latest or offset):
- raise ValueError("after_id is incompatible with latest/offset paging")
- if after_id is not None and include_compacted:
- raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)")
- if after_id is not None and include_ancestors:
- raise ValueError("after_id is incompatible with include_ancestors (merged-lineage reads use offset paging)")
- active_clause = self._active_clause(include_inactive, include_compacted)
- # Ancestor expansion uses the resume lineage (explicit /branch copies stay single-session).
- session_ids = self._resume_lineage_ids(session_id) if include_ancestors else [session_id]
- if len(session_ids) > 1:
- # Multi-segment lineage: dedupe-then-page on the merged display set, matching the
- # canonical multi-segment projection in get_messages_as_conversation(include_ancestors=True).
- # (display_order is per-segment and not comparable across segments, so the per-segment
- # display_order paging below does not apply across a lineage.)
- display = include_compacted and not include_inactive # removed rows too: see drop_removed
- rows = self._read_all(f"SELECT * FROM messages WHERE session_id IN ({_placeholders(session_ids)})"
- + active_clause + " ORDER BY id ASC", session_ids)
- rows = self._dedupe_display_generations(
- self._with_removed_identities(rows, session_ids) if display else rows, drop_removed=display)
- rows = rows[::-1][offset:][:limit][::-1] if latest else rows[offset:][:limit]
- elif include_compacted and not include_inactive and self._ensure_display_order(session_id):
- # _read_retrying_ioerr: mode=ro pooled readers see a transient IOERR mid-checkpoint (#100871).
- rows = self._read_retrying_ioerr(
- lambda conn: self._display_rows_from_conn(
- conn, session_id, limit=limit, offset=offset, latest=latest))
- elif include_compacted:
- # Read-only legacy stores cannot persist display identities; keep only fixed-width
- # identities and representative ids while scanning, then fetch the selected payloads.
- rows = self._legacy_display_page(
- session_id, active_clause=active_clause, limit=limit, offset=offset, latest=latest)
- else:
- sql = (f"SELECT * FROM messages WHERE session_id = ?{active_clause}"
- f"{' AND id > ?' if after_id is not None else ''} ORDER BY id {'DESC' if latest else 'ASC'}")
- params: list = [session_id] if after_id is None else [session_id, after_id]
- if limit is not None or offset:
- # SQLite's OFFSET requires LIMIT; -1 means "no limit".
- sql += " LIMIT ? OFFSET ?"
- params.extend([-1 if limit is None else limit, offset])
- rows = self._read_all(sql, params)
- if latest:
- rows.reverse()
- return [self._row_to_message_dict(row, warn_context="get_messages", summary_flag=True) for row in rows]
- def find_pr_url_messages(self, session_ids: list[str]) -> list[dict[str, Any]]:
- """Tool results containing ``/pull/``: a deliberately loose scan, oldest-first so the caller takes the last."""
- ids = [s for s in session_ids if s]
- chunks = (ids[start : start + 900] for start in range(0, len(ids), 900)) # SQLite's bound-variable ceiling.
- return [{"session_id": row[0], "content": row[1]} for chunk in chunks for row in self._read_all(
- f"""SELECT session_id, content FROM messages
- WHERE session_id IN ({_placeholders(chunk)})
- AND role = 'tool' AND content LIKE '%/pull/%'
- ORDER BY id ASC""",
- chunk)]
- def get_messages_around(self, session_id: str, around_message_id: int, window: int = 5) -> dict[str, Any]:
- """Up to *window* messages either side of an anchor id (ascending). ``messages_before``/``_after`` count
- strictly around the anchor (fewer than *window* = session boundary). Empty for a foreign anchor."""
- window = max(window, 0)
- with self._read_ctx() as conn:
- if not conn.execute("SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1",
- (around_message_id, session_id)).fetchone():
- return {"window": [], "messages_before": 0, "messages_after": 0}
- before_rows = conn.execute(
- "SELECT * FROM messages WHERE session_id = ? AND id <= ? ORDER BY id DESC LIMIT ?",
- (session_id, around_message_id, window + 1)).fetchall()
- after_rows = conn.execute(
- "SELECT * FROM messages WHERE session_id = ? AND id > ? ORDER BY id ASC LIMIT ?",
- (session_id, around_message_id, window)).fetchall()
- window_msgs = [self._row_to_message_dict(r, warn_context="get_messages_around", summary_flag=False)
- for r in (*reversed(before_rows), *after_rows)]
- # before_rows includes the anchor itself.
- return {"window": window_msgs, "messages_before": max(0, len(before_rows) - 1), "messages_after": len(after_rows)}
- def resolve_resume_session_id(self, session_id: str) -> str:
- """Redirect a resume target to the descendant holding the messages: follow the compression chain to
- the live tip (lineage-aware, so delegate/branch children never hijack it), then walk
- ``parent_session_id`` forward to the DEEPEST node with messages (a continuation may hold newer
- turns), skipping branch/delegate/reset/tool children. Unchanged when nothing has messages; depth cap 32.
- Context compression ends the current session and forks a new child session (linked via
- ``parent_session_id``). The flush cursor is reset, so the child is where new messages actually land
- — the parent ends up with ``message_count = 0`` rows unless messages had already been flushed to it
- before compression. See #15000.
- """
- if not session_id:
- return session_id
- try:
- session_id = self.get_compression_tip(session_id) or session_id
- except Exception:
- pass
- with self._read_ctx() as conn:
- current = session_id
- seen = {current}
- best = None # deepest node with messages
- for _ in range(32):
- try:
- if conn.execute("SELECT 1 FROM messages WHERE session_id = ? LIMIT 1", (current,)).fetchone() is not None:
- best = current
- child_row = conn.execute(
- "SELECT id FROM sessions AS child WHERE child.parent_session_id = ? "
- f" AND {_sql_json_extract('child.model_config', '$._branched_from')} IS NULL "
- f" AND {_sql_json_extract('child.model_config', '$._delegate_from')} IS NULL "
- f" AND {_sql_json_extract('child.model_config', '$._reset_from')} IS NULL "
- f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} "
- " AND COALESCE(child.source, '') != 'tool' "
- "ORDER BY child.started_at DESC, child.id DESC LIMIT 1", (current,)).fetchone()
- except Exception:
- return session_id
- if child_row is None or not child_row["id"] or child_row["id"] in seen:
- break
- current = child_row["id"]
- seen.add(current)
- return best if best is not None else session_id
- def _fetch_conversation_rows(self, session_ids: list[str], active_clause: str, *, with_session_id: bool,
- with_compacted: bool = False):
- """``_CONVERSATION_ROW_COLUMNS`` rows for *session_ids* ORDER BY id (timestamps are not monotonic
- and would break tool-call adjacency). ``with_compacted``: the display set (``active_clause`` is
- ignored) with the ``compacted`` column, plus the removed rows' identities for ``drop_removed``."""
- rows = self._read_all(
- f"SELECT {'session_id, ' if with_session_id or with_compacted else ''}{'compacted, display_order, ' if with_compacted else ''}"
- f"{self._CONVERSATION_ROW_COLUMNS} "
- f"FROM messages WHERE session_id IN ({_placeholders(session_ids)})"
- f"{_DISPLAY_ACTIVE_CLAUSE if with_compacted else active_clause} ORDER BY id", tuple(session_ids))
- return self._with_removed_identities(rows, session_ids) if with_compacted else rows
- def _with_removed_identities(self, rows, session_ids: list[str]) -> list[Any]:
- """*rows* (ordered by id) merged with the identities of the lineage's removed rows (``active=0,
- compacted=0``), which ``_dedupe_display_generations(drop_removed=True)`` needs only to hide a copy, never
- to show: no body (a legacy row without a stored display identity keeps its content for the key)."""
- removed = self._read_all(
- "SELECT session_id, id, role, timestamp, tool_call_id, tool_calls, tool_name, display_kind, display_metadata, "
- "message_uid, absorbed_message_uids, display_identity, active, compacted, "
- "CASE WHEN display_identity IS NULL THEN content END AS content "
- f"FROM messages WHERE session_id IN ({_placeholders(session_ids)}) AND active = 0 AND compacted = 0 "
- "ORDER BY id", tuple(session_ids))
- return sorted([*rows, *removed], key=lambda row: row["id"]) if removed else list(rows)
- def get_messages_as_conversation(self, session_id: str, include_ancestors: bool = False,
- include_inactive: bool = False, repair_alternation: bool = False,
- include_row_ids: bool = False,
- include_compacted: bool = False) -> list[dict[str, Any]]:
- """Load messages in OpenAI format. ``include_compacted`` (deduped display history) is for DISPLAY reads
- only: the model-fed restore must not regrow what compaction summarized away. ``repair_alternation``
- repairs the loaded list for LIVE REPLAY callers (a durable ``user;user`` pair would re-trigger the
- per-request repair forever), preserving summary markers before repair so derivative context
- cannot merge with an original user turn; the stored transcript is never mutated."""
- display = include_compacted and not include_inactive # removed rows too: see drop_removed
- rows = self._fetch_conversation_rows(
- self._resume_lineage_ids(session_id) if include_ancestors else [session_id],
- self._active_clause(include_inactive, include_compacted), with_session_id=False, with_compacted=display)
- if include_compacted:
- rows = self._dedupe_display_generations(rows, drop_removed=display)
- return self._rows_to_conversation(rows, session_id=session_id, include_ancestors=include_ancestors,
- repair_alternation=repair_alternation, include_row_ids=include_row_ids,
- include_summary_markers=repair_alternation)
- def conversation_revision(self, session_id: str, include_ancestors: bool = False) -> tuple:
- """A cheap fingerprint of what ``get_messages_as_conversation`` (live rows) would load: the
- lineage ids, the active rows' count and highest id, and the sessions' write-side
- ``transcript_revision`` (bumped by a trigger on every in-place row rewrite). Any append,
- rewind, undo, compaction or rewrite moves it, so a payload derived from the transcript can be
- reused while it holds."""
- ids = self._resume_lineage_ids(session_id) if include_ancestors else [session_id]
- marks = _placeholders(ids)
- row = self._read_one(
- f"SELECT COUNT(*), COALESCE(MAX(id), 0), (SELECT COALESCE(SUM(transcript_revision), 0) FROM sessions"
- f" WHERE id IN ({marks})) FROM messages WHERE session_id IN ({marks}) AND active = 1", (*ids, *ids))
- return (tuple(ids), int(row[0]), int(row[1]), int(row[2]))
- def _dedupe_replayed_user(self, messages, msg, exact_user_clones) -> tuple[bool, Any]:
- """Ancestor-lineage dedupe of one decoded user *msg* -> ``(skip, exact_clone_key)``. Rotation
- column-clones the concurrent tail into the child, so copies need not be adjacent: the exact
- ``(timestamp, canonical content)`` clone index is checked first, then the adjacent heuristic. A
- rotated child carrier wins over the ancestor copy."""
- canonical_content = self._canonical_replayed_user_content(msg)[0]
- exact_clone_key = self._exact_replayed_user_clone_key(msg.get("timestamp"), canonical_content)
- previous_exact = exact_user_clones.get(exact_clone_key) if exact_clone_key is not None else None
- duplicate = None
- if previous_exact is not None:
- previous_index = next((i for i, candidate in enumerate(messages) if candidate is previous_exact), None)
- if previous_index is not None:
- duplicate = (previous_index, True)
- if duplicate is None:
- duplicate = self._find_duplicate_replayed_user_message(messages, msg)
- if duplicate is None:
- return False, exact_clone_key
- duplicate_index, prefer_current = duplicate
- if prefer_current:
- messages.pop(duplicate_index)
- return not prefer_current, exact_clone_key
- def _rows_to_conversation(self, rows, *, session_id: str, include_ancestors: bool, repair_alternation: bool,
- include_row_ids: bool = False,
- include_summary_markers: bool = False) -> list[dict[str, Any]]:
- """Decode fetched rows (ordered by id, pre-filtered) into OpenAI format, stable key order. Every dict is
- stamped ``_DB_PERSISTED_MARKER_KEY`` (born durable) so an identity-losing handoff never re-appends the
- transcript on flush. Unaddressed live-replay projections also carry the stored-row CAS digest: if a later rewrite
- loses its physical ``_row_id``, logical ``message_uid`` can recover the row without guessing by
- mutable payload while the digest still fences a concurrent winner. ``_row_id`` is opt-in (gateway
- reactions); reasoning restored on assistant rows only; ``api_content`` VERBATIM (no sanitize/strip)
- so replay keeps the provider prompt cache byte-stable."""
- from hermes_state import _strip_background_review_harness, _strip_stale_tool_call_markers
- # Runtime import avoids the transcript_repair -> hermes_state_messages module cycle.
- from agent.transcript_repair import transcript_row_snapshot
- # Only the unaddressed live replay gets the digest: row-addressed loaders (include_row_ids) keep the
- # legacy resumed-dict path, whose rewrite never re-writes columns the projection does not decode
- # (a CAS-match rewrite of a resumed row would otherwise null token_count).
- stamp_snapshot = repair_alternation and not include_row_ids
- messages = []
- exact_user_clones: dict[tuple[Any, str], dict[str, Any]] = {}
- tool_uid_index: dict[str, str] = {} # pairing-id variant -> uid, from the assistant rows indexed so far
- # Assistant rows since the last user row not yet indexed: only a result without a stored uid (an older
- # build's) needs the index, so rows this build wrote never pay for it. Indexed in order: same shadowing.
- unindexed_tool_owners: list[dict[str, Any]] = []
- for row in rows:
- content = self._loaded_view_content(row["role"], self._decode_content(row["content"]))
- # Underscore-prefixed like ``_row_id``: transports strip it before the wire; compression's
- # assembly copies strip it so rotated child handoffs still flush (_fresh_compaction_message_copy).
- msg = {"role": row["role"], "content": content, _DB_PERSISTED_MARKER_KEY: True}
- if stamp_snapshot:
- msg[DB_ROW_SNAPSHOT] = transcript_row_snapshot(row)
- # Born durable (#92231): this dict is materialized FROM a durable row, so stamp the persistence
- # marker at the source instead of relying on every restore caller to thread the loaded list back
- # through a flush as ``conversation_history=`` — any identity-losing handoff (compression's
- # durable-snapshot adoption, incremental persists with no history arg) would otherwise re-append
- # the ENTIRE transcript on flush.
- if include_row_ids and row["id"] is not None:
- msg["_row_id"] = row["id"]
- # The durable identity is part of the message, like ``timestamp``: restored on EVERY projection
- # (ACP, gateway, CLI, TUI, compression adoption), never opt-in like ``_row_id``.
- _restore_identity_columns(row, msg)
- msg.update((col, row[col]) for col in ("api_content", "display_kind") if row[col])
- if row["display_metadata"] and (decoded := self._decode_display_metadata(row["display_metadata"])) is not None:
- msg["display_metadata"] = decoded
- lift_data_use(msg)
- if include_summary_markers and row["_compressed_summary"]:
- msg["_compressed_summary"] = True
- msg.update(
- (col, row[col]) for col in ("timestamp", "tool_call_id", "tool_name", "effect_disposition") if row[col])
- if row["tool_calls"]:
- msg["tool_calls"] = _json_or(
- row["tool_calls"], [], "Failed to deserialize tool_calls in conversation replay, falling back to []")
- if row["platform_message_id"]: # platform-side id exposed as ``message_id`` (JSONL transcript compat)
- msg["message_id"] = row["platform_message_id"]
- if row["observed"]:
- msg["observed"] = True
- if row["role"] == "assistant":
- msg.update((col, row[col]) for col in ("finish_reason", "reasoning") if row[col])
- if row["reasoning_content"] is not None:
- msg["reasoning_content"] = row["reasoning_content"]
- msg.update(
- (col, _json_or(row[col], None, f"Failed to deserialize {col}, falling back to None"))
- for col in ("reasoning_details", "codex_reasoning_items", "codex_message_items") if row[col])
- if msg.get("tool_calls"):
- unindexed_tool_owners.append(msg)
- elif row["role"] == "user":
- tool_uid_index.clear() # a result never pairs across a user turn
- unindexed_tool_owners.clear()
- elif row["role"] == "tool" and not row["tool_call_uid"] and row["tool_call_id"]:
- # No stored uid (a result appended by an older build or a lone append): the one its assistant
- # row named. Rows are read in id order, so the call always precedes its result. Provider ids
- # repeat: a later row's calls shadow an earlier occurrence's, and a row without a map (an older
- # writer's) leaves its results unpaired rather than mispaired.
- for owner in unindexed_tool_owners:
- index_tool_call_uids(tool_uid_index, owner)
- unindexed_tool_owners.clear()
- if tool_uid := resolve_tool_call_uid(tool_uid_index, row["tool_call_id"]):
- msg[TOOL_CALL_UID] = tool_uid
- if include_ancestors:
- skip, exact_clone_key = self._dedupe_replayed_user(messages, msg, exact_user_clones)
- if skip:
- continue
- if exact_clone_key is not None:
- exact_user_clones[exact_clone_key] = msg
- messages.append(msg)
- # Defense-in-depth: strip a background-review harness turn (older builds shared the parent's
- # session_id) plus its curator reply, and bare tool-call marker content ("[memory]") persisted as an answer.
- messages = _strip_stale_tool_call_markers(_strip_background_review_harness(messages))
- if repair_alternation and messages:
- from agent.agent_runtime_helpers import repair_message_sequence
- repaired = repair_message_sequence(None, messages)
- if repaired:
- logger.info("Repaired %d message-alternation violation(s) while "
- "restoring session %s — durable transcript kept them, "
- "see repair_message_sequence", repaired, session_id)
- return messages
- def get_resume_conversations(self, session_id: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
- """``(model_history, display_history)`` for a resume from ONE SELECT; byte-identical to the separate
- reads. model: the tip's active rows, alternation-repaired, summary marker kept for pre-compress
- checkpointing. display: the full lineage (``/branch`` stands alone), compaction-archived rows deduped.
- The display projection also includes rows preserved by IN-PLACE compaction (``active=0,
- compacted=1``), deduped by :meth:`_dedupe_display_generations`. Without them a compacted
- conversation resumes showing only its summary plus the carried-forward tail — the user's own turns
- read as deleted even though every row is still on disk, and the REST transcript read (which has
- always included them) disagreed with this one about the same session (#92080).
- """
- rows = self._fetch_conversation_rows(
- self._resume_lineage_ids(session_id), "", with_session_id=True, with_compacted=True)
- # The model projection stays active-only: it is the compressed working context. The gateway's
- # first-turn ``session_meta`` marker (tools/model/platform bookkeeping) is transcript logging,
- # never LLM input: a resuming surface that fed it to the agent would count it in the history
- # and send a prefix the previous process never sent.
- model_history = self._rows_to_conversation(
- [r for r in rows if r["session_id"] == session_id and r["active"] and r["role"] != "session_meta"],
- session_id=session_id,
- include_ancestors=False, repair_alternation=True, include_row_ids=True, include_summary_markers=True)
- display_history = self._rows_to_conversation(
- self._dedupe_display_generations(rows, drop_removed=True), session_id=session_id,
- include_ancestors=True, repair_alternation=False, include_row_ids=True)
- return model_history, display_history
- def _resume_lineage_ids(self, session_id: str) -> list[str]:
- """Session ids a display resume materializes: the VERIFIED compression lineage (root → tip; each
- hop's parent ended ``compression`` and the child is a genuine continuation), or the session alone
- for anything else — an explicit ``/branch`` copy, an API fork, a reset child, or a plain session.
- The raw parent walker ``_session_lineage_root_to_tip`` is intentionally NOT used here: it crosses
- fork/reset boundaries and misclassifies API forks (parent ended ``branched``, no marker) as
- compression lineages. Shared with the resume guard so it counts exactly what a resume loads."""
- if not session_id:
- return [session_id]
- session = self.get_session(session_id)
- if not session or self._is_explicit_fork_child_row(session):
- return [session_id]
- # _is_compression_child_row is the per-hop gate: the child must not be an explicit
- # branch/delegate/tool child AND its parent must have ended 'compression'.
- chain = [session_id]
- current = session
- seen = {session_id}
- while len(chain) < 100: # defensive bound, same as _session_lineage_root_to_tip
- if not self._is_compression_child_row(current):
- break
- parent = self.get_session(current["parent_session_id"])
- if not parent or parent["id"] in seen:
- break
- seen.add(parent["id"])
- chain.append(parent["id"])
- current = parent
- return list(reversed(chain))
- def _resume_count_scope(self, session_id: str, tip_only: bool) -> tuple[list[str], str]:
- """``tip_only``: the tip's ACTIVE rows (model restore); else the full-lineage DISPLAY set."""
- if tip_only:
- return [session_id], "active = 1"
- return self._resume_lineage_ids(session_id), "(active = 1 OR compacted = 1)"
- def get_resume_message_count(self, session_id: str, *, tip_only: bool = False) -> int:
- """Count the rows a resume would materialize (see ``_resume_count_scope``)."""
- session_ids, active_clause = self._resume_count_scope(session_id, tip_only)
- return int(self._read_one(
- f"SELECT COUNT(*) FROM messages WHERE session_id IN ({_placeholders(session_ids)}) AND {active_clause}",
- tuple(session_ids))[0])
- def assert_resume_safe(self, session_id: str, max_messages: Optional[int] = None, *, tip_only: bool = False) -> int:
- """Resume row count, or raise ``SessionResumeTooLargeError``. ``max_messages=None`` reads config; 0
- disables the guard without counting. ``tip_only`` bounds only the tip's active rows for callers that
- never materialize the lineage: a heavily compressed conversation is a success, not a rejection."""
- from hermes_state import SessionResumeTooLargeError, resolved_max_resume_messages
- if max_messages is None:
- max_messages = resolved_max_resume_messages()
- if max_messages < 0:
- raise ValueError("max_messages must be non-negative")
- if max_messages == 0:
- return 0
- session_ids, active_clause = self._resume_count_scope(session_id, tip_only)
- message_count = int(self._read_one("SELECT COUNT(*) FROM ("
- f"SELECT 1 FROM messages WHERE session_id IN ({_placeholders(session_ids)}) "
- f"AND {active_clause} LIMIT ?)", (*session_ids, max_messages + 1))[0])
- if message_count > max_messages:
- raise SessionResumeTooLargeError(
- message_count, max_messages, scope="in its tip segment" if tip_only else "across its lineage")
- return message_count
- def get_ancestor_display_prefix(self, session_id: str) -> list[dict[str, Any]]:
- """Ancestor-only display messages of a lineage (row ``session_id != tip``) that ``session.resume``
- prepends. Identified by row origin, not ``display[:len(display) - len(model)]``, so alternation
- repair cannot overcount."""
- session_ids = self._resume_lineage_ids(session_id)
- if len(session_ids) <= 1:
- return []
- rows = self._dedupe_display_generations(self._fetch_conversation_rows(
- session_ids, "", with_session_id=True, with_compacted=True), drop_removed=True)
- ancestor_ids = {int(row["id"]) for row in rows if row["session_id"] != session_id and row["id"] is not None}
- if not ancestor_ids:
- return []
- lineage = self._rows_to_conversation(
- rows, session_id=session_id, include_ancestors=True, repair_alternation=False, include_row_ids=True)
- return [{k: v for k, v in message.items() if k != "_row_id"}
- for message in lineage if message.get("_row_id") in ancestor_ids]
- def get_conversation_root(self, session_id: str) -> str:
- """ROOT id of the lineage: the stable conversation id across compression segments and delegate
- subagents (Nous Portal usage tagging). Unchanged when there is no recorded parent."""
- chain = self._session_lineage_root_to_tip(session_id)
- return chain[0] if chain and chain[0] else session_id
- @staticmethod
- def _canonical_replayed_user_content(msg: dict[str, Any]) -> tuple[Any, bool]:
- """Return canonical live content and whether *msg* is composite."""
- if msg.get("role") != "user":
- return None, False
- handoff, live_view = split_user_originated_turn(msg)
- is_composite = handoff is not None and live_view is not None
- return live_view.get("content") if is_composite else msg.get("content"), is_composite
- @staticmethod
- def _exact_replayed_user_clone_key(timestamp: Any, content: Any) -> Optional[tuple[Any, str]]:
- """Return a hashable key for a column-exact rotation clone."""
- if timestamp is None or content in (None, "", []):
- return None
- try:
- return timestamp, json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
- except (TypeError, ValueError):
- return None
- @staticmethod
- def _find_duplicate_replayed_user_message(messages: list[dict[str, Any]],
- msg: dict[str, Any]) -> Optional[tuple[int, bool]]:
- """Adjacent replay duplicate ``(index, prefer_current)`` or None. Rotation may persist the current ask
- in the parent and again inside a composite child carrier: carriers compare by canonical live payload,
- ordinary users by exact string. The child carrier wins (it owns the durable row id and scaffold)."""
- if msg.get("role") != "user":
- return None
- canonical = SessionMessagesMixin._canonical_replayed_user_content
- content, prefer_current = canonical(msg)
- if content in (None, "", []):
- return None
- for index in range(len(messages) - 1, -1, -1):
- prev = messages[index]
- if prev.get("role") == "user":
- prev_content, prev_is_composite = canonical(prev)
- if prev_content == content and (prefer_current or prev_is_composite or isinstance(content, str)):
- return index, prefer_current
- elif prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")):
- return None
- return None
- # ========================================================================= Rewind (soft-delete) — see
- # /rewind slash command + issue #21910
- # =========================================================================
- def get_active_message_ids(self, session_id: str) -> list[int]:
- """Ordered physical active ids for rewind CAS checks (includes legacy harness rows projections omit)."""
- return [int(row[0]) for row in self._read_all(_ACTIVE_IDS_SQL, (session_id,))]
- @staticmethod
- def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]:
- """Return active message/tool-call counts inside the caller's txn."""
- rows = conn.execute("SELECT tool_calls FROM messages WHERE session_id = ? AND active = 1", (session_id,)).fetchall()
- return len(rows), sum(_tool_calls_len(row[0], scalar=1) for row in rows)
- def _split_rewind_target(self, target_row: dict[str, Any], expected_target_content: Any, preserve_compaction_handoff: bool):
- """Validate an active rewind target; return its handoff scaffold (or None). ``ValueError``: inactive /
- non-user-originated / missing composite carrier; ``RuntimeError``: live payload changed."""
- if not target_row.get("active"):
- raise ValueError("rewind target is not active")
- handoff, live_view = split_user_originated_turn({
- **target_row, "content": self._decode_content(target_row.get("content")),
- "display_metadata": self._decode_display_metadata(target_row.get("display_metadata"))})
- if live_view is None:
- raise ValueError("rewind target is not a user-originated turn")
- live_content = live_view.get("content")
- if isinstance(live_content, str):
- live_content = sanitize_context(live_content).strip()
- if expected_target_content is not None and live_content != expected_target_content:
- raise RuntimeError("rewind target changed before it could be persisted")
- if preserve_compaction_handoff and handoff is None:
- raise ValueError("preserve_compaction_handoff requires an active composite carrier")
- return handoff if preserve_compaction_handoff else None
- def rewind_to_message(self, session_id: str, target_message_id: int, *, preserve_compaction_handoff: bool = False,
- expected_active_ids: Optional[list[int]] = None,
- expected_target_content: Any = None) -> dict[str, Any]:
- """Soft-delete (``active=0``) every message with id >= *target_message_id*, target included (the caller
- pre-fills it as the next prompt). Returns ``{"rewound_count", "target_message", "new_head_id"}``, plus
- ``replacement_message_id`` with ``preserve_compaction_handoff`` (archives a composite summary carrier,
- inserts its hidden handoff scaffold as the new head). ``ValueError``: target missing or not ``user``.
- ``expected_active_ids`` / ``expected_target_content`` pin the active set and canonical live payload
- in-txn before any mutation (presentation-only metadata changes don't invalidate a rewind). A live turn
- lease refuses; expired/dead holders are reclaimed. ``rewind_count`` always increments."""
- from hermes_state_mutation_transcript import rewind_in_transaction
- def _do(conn):
- target_row, ids, head_id, replacement_id = rewind_in_transaction(
- self, conn, session_id, target_message_id,
- preserve_compaction_handoff=preserve_compaction_handoff,
- expected_active_ids=expected_active_ids, expected_target_content=expected_target_content)
- # The scaffold insert stamped a ``message_uid``; read it back in the same txn so the
- # caller can address the new head by its durable identity.
- replacement_uid = replacement_id and conn.execute(
- "SELECT message_uid FROM messages WHERE id = ?", (replacement_id,)).fetchone()[0]
- return target_row, ids, head_id, replacement_id, replacement_uid
- target_row, rewound, new_head_id, replacement_message_id, replacement_message_uid = self._execute_write(_do)
- # Decode for the prompt-buffer prefill without a second fallible DB operation.
- target_row["content"] = self._decode_content(target_row.get("content"))
- return {"rewound_count": len(rewound), "target_message": target_row, "new_head_id": new_head_id,
- **({"replacement_message_id": replacement_message_id,
- "replacement_message_uid": replacement_message_uid}
- if preserve_compaction_handoff else {})}
- def message_count(self, session_id: str | None = None) -> int:
- """Count messages, optionally for a specific session."""
- sql = "SELECT COUNT(*) FROM messages" + (" WHERE session_id = ?" if session_id else "")
- return self._read_one(sql, (session_id,) if session_id else ())[0]
- def has_gateway_input_owner(self, session_id: str, owner: str) -> bool:
- """Probe the accepted-input marker without allocating message bodies or archives."""
- return self._read_one(
- "SELECT 1 FROM messages WHERE session_id = ? AND role = 'user' "
- "AND observed = 0 AND (active = 1 OR compacted = 1) "
- "AND CASE WHEN json_valid(display_metadata) "
- "THEN json_extract(display_metadata, '$.gateway_input_owner') END = ? LIMIT 1",
- (session_id, owner)) is not None
- def has_platform_message_id(self, session_id: str, platform_message_id: str) -> bool:
- """True when *platform_message_id* exists (partial-index probe; the gateway's transient-failure dedupe).
- Uses the idx_messages_platform_msg_id partial index for efficient lookup. Used by the gateway's
- transient-failure dedupe guard (#47237) to skip re-persisting a user message that was already saved
- on a prior retry of the same inbound platform message.
- """
- return self._read_one(
- "SELECT 1 FROM messages WHERE session_id = ? AND platform_message_id = ? LIMIT 1",
- (session_id, platform_message_id)) is not None
- def _is_explicit_fork_child_row(self, session: dict[str, Any], *, include_reset: bool = False) -> bool:
- """True when *session* is a branch, delegate, or tool child of its parent (``include_reset``: also a
- reset fork). Markers only count when they point at ``parent_session_id``: compression copies
- ``model_config`` onto the continuation, so presence-only matching would misclassify it (same binding
- as ``_NON_CONTINUATION_CHILD_FILTER_SQL``)."""
- if session.get("source") == "tool":
- return True
- cfg = session.get("model_config")
- if isinstance(cfg, str):
- try:
- cfg = json.loads(cfg)
- except json.JSONDecodeError:
- return False
- if not isinstance(cfg, dict):
- return False
- markers = (cfg.get("_branched_from"), cfg.get("_delegate_from"))
- if include_reset:
- markers += (cfg.get("_reset_from"),)
- parent_id = session.get("parent_session_id")
- return parent_id in markers if parent_id else any(m is not None for m in markers)
- def is_explicit_fork_child(self, session_id: str) -> bool:
- """Read-only :meth:`_is_explicit_fork_child_row`; a missing row is not a fork (prompt_cache_scope keeps
- a declared conversation key from crossing the fork boundary)."""
- session = self.get_session(session_id)
- return bool(session and self._is_explicit_fork_child_row(session))
- def latest_conversation_boundary(self, session_key: str, source: str) -> Optional[int]:
- """Conversation boundaries (``_RESET_END_REASONS`` ends) this peer has crossed, or ``None`` if never
- reset. The peer is ``(session_key, source)``, never the key alone (an API caller may legally reuse a
- Telegram row's key). Read from ``conversation_generations`` (advanced in each boundary's txn), not an
- aggregate over session rows: deletes/prunes would re-emit a retired pair. Rows are never GC'd
- (dropping one re-issues generation 1: the ABA this prevents). Wall-clock-free, so a backwards NTP
- correction cannot reorder it. DBs upgraded mid-conversation take their first generation from the
- next boundary written (a pre-upgrade reset shares its predecessor's scope once: costs a warm
- prompt-cache bucket, never crosses an identity)."""
- if not session_key or not source:
- return None
- row = self._read_one(
- "SELECT generation FROM conversation_generations WHERE source = ? AND session_key = ?",
- (source, session_key))
- generation = int(row["generation"]) if row is not None and row["generation"] is not None else 0
- return generation if generation > 0 else None
- def clear_messages(self, session_id: str) -> None:
- """Delete all messages for a session and reset its counters."""
- def _do(conn):
- conn.execute("DELETE FROM messages WHERE session_id = ?", (session_id,))
- conn.execute(_RESET_COUNTERS_SQL, (session_id,))
- self._execute_write(_do)
- def purge_stale_tool_call_markers(self, *, dry_run: bool = False, backup: bool = True) -> dict[str, Any]:
- """Permanently clear bare tool-call marker content ("[memory]") left by pre-fix sessions
- (``_rows_to_conversation`` repairs it in memory; this stops the re-scan). Only ``content`` is touched.
- ``backup``: ``VACUUM INTO`` snapshot first (none when nothing changes)."""
- from hermes_state import _STALE_TOOL_CALL_MARKER_RE
- def _find_affected(conn) -> list[int]:
- cursor = conn.execute("SELECT id, content FROM messages "
- "WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''")
- return [row["id"] for row in cursor.fetchall()
- if isinstance(row["content"], str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(row["content"].strip())]
- def _result(affected, backup_path=None):
- return {"dry_run": dry_run, "rows_affected": len(affected), "row_ids": affected, "backup_path": backup_path}
- with self._read_ctx() as conn:
- affected_ids = _find_affected(conn)
- if dry_run or not affected_ids:
- return _result(affected_ids)
- backup_path: Optional[str] = None
- if backup:
- import datetime
- stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
- backup_path = str(self.db_path.with_name(f"{self.db_path.name}.pre-clean-markers-backup-{stamp}"))
- with self._lock:
- self._conn.execute("VACUUM INTO ?", (backup_path,))
- logger.info("Backed up state.db to %s before clean-markers write", backup_path)
- def _do(conn):
- ids = _find_affected(conn)
- if ids:
- conn.execute(f"UPDATE messages SET content = '' WHERE id IN ({_placeholders(ids)})", ids)
- return ids
- affected_ids = self._execute_write(_do)
- if affected_ids:
- logger.info("Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)", len(affected_ids))
- return _result(affected_ids, backup_path)
|