shutdown_flush.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502
  1. """Flush pending messages and agent transcripts to disk before shutdown to prevent data loss.
  2. When FTS5 corruption blocks ``INSERT INTO messages``, ``_pending_messages`` and the live
  3. ``agent._session_messages`` are the only surviving copies; shutdown ``.clear()`` would drop them.
  4. All hooks write atomic JSON payloads under ``<hermes_home>/pending_messages/``:
  5. ``flush_pending_to_file`` / ``flush_overflow_to_file`` (queue head / FIFO tail, before clear),
  6. ``recover_pending_to_db`` (at startup, before resume turns; replays via ``SessionDB.append_message``,
  7. deletes each file on success), ``flush_agent_history_to_file`` (DB flush raised),
  8. ``spool_dropped_transcript_message`` / ``drain_transcript_spool``.
  9. """
  10. from __future__ import annotations
  11. import contextlib
  12. import itertools
  13. import json
  14. import logging
  15. import math
  16. import os
  17. import sqlite3
  18. import time
  19. import uuid
  20. from pathlib import Path
  21. from typing import Any, Dict, Optional
  22. from hermes_dir_acl import make_user_dir
  23. logger = logging.getLogger(__name__)
  24. # Reason tag for transcript messages dropped by the in-memory pending cap during live
  25. # operation. Payloads carry the full transcript message dict for verbatim replay.
  26. # See #78182.
  27. TRANSCRIPT_CAP_DROP_REASON = "transcript_cap_drop"
  28. # Reason tag for agent-history snapshots: kept for manual operator recovery, never replayed.
  29. AGENT_HISTORY_REASON = "shutdown-with-unpersisted-agent-history"
  30. # Suffix for a spool file whose row the database rejected; no replay glob matches it.
  31. QUARANTINE_SUFFIX = ".bad"
  32. # Monotonic tiebreaker so same-second spool files replay in drop order.
  33. _TRANSCRIPT_SPOOL_SEQ = itertools.count()
  34. # Spool files whose row was written but which could not be deleted. Replaying one again would write
  35. # its row twice, so both replay passes skip it for the rest of this process.
  36. _REPLAYED_UNREMOVABLE: set[Path] = set()
  37. def _get_flush_dir():
  38. """Return the pending-messages flush directory under the active HERMES_HOME."""
  39. from hermes_constants import get_hermes_home
  40. flush_dir = get_hermes_home() / "pending_messages"
  41. from hermes_constants import assert_named_profile_home_live
  42. assert_named_profile_home_live(flush_dir)
  43. make_user_dir(flush_dir, parents=True, exist_ok=True, mode=0o700)
  44. if os.name == "posix":
  45. os.chmod(flush_dir, 0o700)
  46. return flush_dir
  47. def _write_payload(flush_dir: Path, payload: dict[str, Any]) -> Path:
  48. """Atomically write one private, uniquely named recovery payload; return its path."""
  49. from utils import atomic_json_write
  50. final_path = flush_dir / f"pending-{uuid.uuid4().hex}.json"
  51. atomic_json_write(final_path, payload, mode=0o600, default=str)
  52. if os.name == "posix":
  53. # Persist the directory entry too; keep the published file (the only recovery copy) even if
  54. # fsync fails.
  55. try:
  56. directory_fd = os.open(flush_dir, os.O_RDONLY)
  57. except OSError as exc:
  58. logger.debug("Failed to fsync pending-message directory: %s", exc)
  59. else:
  60. try:
  61. os.fsync(directory_fd)
  62. except OSError as exc:
  63. logger.debug("Failed to fsync pending-message directory: %s", exc)
  64. finally:
  65. os.close(directory_fd)
  66. return final_path
  67. def _flush_value(flush_dir: Path, kind: str, session_key: str, value: Any, **extra: Any) -> bool:
  68. """Serialise and write one pending value; return True when a payload was written."""
  69. try:
  70. serialised = _serialise_value(value)
  71. if serialised is None:
  72. return False
  73. _write_payload(flush_dir, {"session_key": session_key, **extra, "data": serialised})
  74. return True
  75. except Exception as exc:
  76. logger.debug("Failed to flush %s message for %s: %s", kind, session_key, exc)
  77. return False
  78. def flush_pending_to_file(pending: dict[str, Any], *, reason: str = "shutdown") -> int:
  79. """Serialise non-empty ``_pending_messages`` slots (``MessageEvent`` or str); return count."""
  80. if not pending:
  81. return 0
  82. flush_dir, ts, flushed = _get_flush_dir(), int(time.time()), 0
  83. for session_key, value in list(pending.items()):
  84. if value is not None:
  85. flushed += _flush_value(flush_dir, "pending", session_key, value, reason=reason, ts=ts)
  86. if flushed:
  87. logger.info("Flushed %d pending message(s) to %s (reason=%s)", flushed, flush_dir, reason)
  88. return flushed
  89. def flush_overflow_to_file(overflow_by_session: dict[str, Any], *, reason: str = "shutdown") -> int:
  90. """Serialise the FIFO overflow tails (``queued_events``) to disk; return events flushed.
  91. The adapter slot holds the queue head and ``SessionState.conversation.queued_events`` the
  92. tail; both must survive restart. Each event is its own payload in the slot-flush shape so
  93. ``recover_pending_to_db`` replays them unchanged; ``seq`` preserves arrival order per session.
  94. """
  95. if not overflow_by_session:
  96. return 0
  97. flush_dir, ts, flushed = _get_flush_dir(), int(time.time()), 0
  98. for session_key, events in list(overflow_by_session.items()):
  99. if not session_key or not events:
  100. continue
  101. for seq, value in enumerate(list(events)):
  102. if value is not None:
  103. flushed += _flush_value(flush_dir, "overflow", session_key, value, reason=reason,
  104. ts=ts, seq=seq)
  105. if flushed:
  106. logger.info("Flushed %d queued overflow message(s) to %s (reason=%s)", flushed, flush_dir,
  107. reason)
  108. return flushed
  109. def spool_dropped_transcript_message(session_id: str, message: dict[str, Any]) -> Optional[Path]:
  110. """Spool a cap-evicted transcript message; ``None`` on failure (callers degrade to drop+log).
  111. Uses the same on-disk pending spool as :func:`flush_pending_to_file` (one atomic JSON payload per
  112. message under ``<hermes_home>/pending_messages/``), so a runtime cap rotation no longer silently
  113. discards user data while the process stays up (#78182).
  114. """
  115. try:
  116. return _write_payload(_get_flush_dir(), {
  117. "session_key": session_id, "reason": TRANSCRIPT_CAP_DROP_REASON, "ts": int(time.time()),
  118. "seq": next(_TRANSCRIPT_SPOOL_SEQ),
  119. "data": {"session_id": session_id, "message": message},
  120. })
  121. except Exception as exc:
  122. logger.debug("Failed to spool cap-dropped transcript message for %s: %s", session_id, exc)
  123. return None
  124. def drain_transcript_spool(session_id: str, replay, *, db_known_failing: bool = False) -> tuple[int, int]:
  125. """Replay cap-dropped transcript messages spooled for *session_id*; return ``(replayed,
  126. remaining)``. ``replay(message_dict)`` runs per message in drop order; a spool file is deleted
  127. only after its replay succeeds. The first failure stops the drain (the DB is likely still
  128. unhealthy) and keeps the rest for retry, unless the DB rejected that row's own values
  129. (:func:`is_row_rejection`): it is quarantined and the drain goes on. With ``db_known_failing``
  130. (the caller's last write already failed and is being logged/escalated) a replay failure is
  131. expected and logs at DEBUG, so a stalled session does not add one WARNING per append on top of
  132. its ERROR (#114266).
  133. """
  134. try:
  135. candidates = list(_get_flush_dir().glob("pending-*.json"))
  136. except Exception as exc:
  137. logger.debug("Cannot scan transcript spool: %s", exc)
  138. return 0, 0
  139. entries = []
  140. for path in candidates:
  141. if path in _REPLAYED_UNREMOVABLE:
  142. continue
  143. try:
  144. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  145. except Exception:
  146. continue
  147. # A parseable non-object file (scalar/list) cannot be attributed to any session: skip it
  148. # like unparseable JSON instead of letting ``.get`` abort the whole drain.
  149. if (not isinstance(payload, dict)
  150. or payload.get("reason") != TRANSCRIPT_CAP_DROP_REASON
  151. or payload.get("session_key") != session_id):
  152. continue
  153. message = (payload.get("data") or {}).get("message")
  154. if not isinstance(message, dict):
  155. logger.warning("Removing structurally invalid transcript spool file %s", path)
  156. path.unlink(missing_ok=True)
  157. continue
  158. entries.append((_spool_sort_key(payload, path.name), path, message))
  159. ordered, replayed, remaining = sorted(entries, key=lambda e: e[0]), 0, 0
  160. for idx, (_key, path, message) in enumerate(ordered):
  161. try:
  162. replay(message)
  163. except Exception as exc:
  164. if is_row_rejection(exc): # no retry can write it, so it must not hold the session back
  165. _quarantine_spool_file(path, session_id, exc)
  166. continue
  167. (logger.debug if db_known_failing else logger.warning)(
  168. "Replay of spooled transcript message %s for %s failed; "
  169. "keeping spool file for retry: %s", path, session_id, exc)
  170. remaining = len(ordered) - idx
  171. break
  172. _remove_replayed(path, session_id)
  173. replayed += 1
  174. if replayed:
  175. logger.info("Replayed %d spooled transcript message(s) for %s after DB recovery", replayed,
  176. session_id)
  177. return replayed, remaining
  178. def _json_safe(value: Any) -> bool:
  179. try:
  180. json.dumps(value)
  181. return True
  182. except (TypeError, ValueError):
  183. return False
  184. def _serialise_value(value: Any) -> Optional[dict]:
  185. """Convert a pending message value to a JSON-serialisable dict."""
  186. if hasattr(value, "text"): # MessageEvent-like object
  187. result: dict[str, Any] = {"text": getattr(value, "text", "")}
  188. for attr in ("session_id", "platform", "sender_id", "sender_name", "reply_to", "media",
  189. "raw_event"):
  190. val = getattr(value, attr, None)
  191. if val is not None:
  192. result[attr] = val if _json_safe(val) else str(val)
  193. return result
  194. if isinstance(value, str): # runner-level _pending_messages
  195. return {"text": value}
  196. if isinstance(value, dict) and _json_safe(value):
  197. return value
  198. return {"text": str(value)}
  199. def _sort_number(value: Any) -> float:
  200. """Coerce a spool ordering field to a float; unusable values sort first.
  201. Ordering fields are read back from JSON on disk and may be missing or
  202. corrupt. A sort key that mixes ``str`` and ``int`` raises ``TypeError``
  203. and would abort the entire recovery pass, so anything non-numeric is
  204. normalised to ``0.0``. So is a JSON integer too large for a float
  205. (``float(10**400)`` raises ``OverflowError``) and a NaN/Infinity literal
  206. (NaN compares false both ways and would scramble the sort).
  207. """
  208. if isinstance(value, bool) or not isinstance(value, (int, float)):
  209. return 0.0
  210. try:
  211. number = float(value)
  212. except OverflowError:
  213. return 0.0
  214. return number if math.isfinite(number) else 0.0
  215. def _spool_sort_key(payload: dict[str, Any], name: str) -> tuple:
  216. """Drop-order key ``(ts, seq, filename)`` for one spool payload. The live drain and the restart
  217. pass both sort on it, so they replay a session's files in the same order."""
  218. # seq is per process (_TRANSCRIPT_SPOOL_SEQ) and ts has one-second resolution, so files two
  219. # processes spool in the same second can interleave; within one process the order is exact.
  220. # Slot heads have no seq and overflow tails start at zero, so a missing seq sorts as -1.
  221. return _sort_number(payload.get("ts")), _sort_number(payload.get("seq", -1)), name
  222. def is_row_rejection(exc: BaseException) -> bool:
  223. """True when the database refused this row's own values, so no retry can succeed: a value sqlite
  224. cannot bind (an unsupported type, or an int outside 64 bits), or a malformed field that fails
  225. before the write. Lock, I/O, corruption and routing failures stay retryable, and so do constraint
  226. failures, which can be session-wide. Only a binding message counts among sqlite errors:
  227. ``InterfaceError("no more rows available")`` is WAL contention that ``SessionDB`` retries and
  228. re-raises unchanged once its patience runs out."""
  229. if isinstance(exc, sqlite3.Error):
  230. return (isinstance(exc, (sqlite3.InterfaceError, sqlite3.ProgrammingError))
  231. and "binding parameter" in str(exc).lower())
  232. return isinstance(exc, (TypeError, ValueError, AttributeError, OverflowError))
  233. def _remove_replayed(path: Path, session_id: str) -> None:
  234. """Delete a spool file whose row was just written. If it cannot be deleted, this process never
  235. replays it again, and replay goes on with the files after it, which are still older than any
  236. live row. Nothing on disk marks it, so a restart writes the row a second time."""
  237. try:
  238. path.unlink(missing_ok=True)
  239. except OSError as exc:
  240. _REPLAYED_UNREMOVABLE.add(path)
  241. logger.error("Spooled transcript row for %s was written, but %s could not be deleted (%s); "
  242. "delete it by hand, or the next start writes that row again", session_id, path, exc)
  243. def _quarantine_spool_file(path: Path, session_id: str, exc: BaseException) -> None:
  244. """Move a spool file whose row the database rejected out of every replay, so it cannot hold its
  245. session's later rows back on every write and every restart. It stays on disk for manual recovery."""
  246. target = path.with_name(path.name + QUARANTINE_SUFFIX)
  247. try:
  248. path.replace(target)
  249. except OSError as move_exc:
  250. logger.error("Spooled transcript row for %s in %s was rejected by the database (%s) and could "
  251. "not be quarantined: %s", session_id, path, exc, move_exc)
  252. return
  253. logger.warning("Spooled transcript row for %s was rejected by the database (%s); moved it to %s "
  254. "for manual recovery and continuing with the session's later rows",
  255. session_id, exc, target)
  256. def _order_flush_files(paths) -> list[Path]:
  257. """Order recovery payloads by :func:`_spool_sort_key`, as :func:`drain_transcript_spool` does.
  258. ``SessionDB`` restores a conversation by AUTOINCREMENT id and spool files have random names, so
  259. any other order permanently scrambles the recovered transcript.
  260. Keeps only the key and path, not decoded payloads; the caller re-reads each file to replay it.
  261. Unparseable files sort last by name.
  262. """
  263. entries = []
  264. for path in paths:
  265. try:
  266. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  267. # OSError: unreadable; ValueError: bad JSON or bytes; RecursionError: nesting too deep.
  268. except (OSError, ValueError, RecursionError):
  269. payload = None
  270. if isinstance(payload, dict):
  271. entries.append(((0, *_spool_sort_key(payload, path.name)), path))
  272. else:
  273. entries.append(((1, 0.0, 0.0, path.name), path))
  274. entries.sort(key=lambda entry: entry[0])
  275. return [path for _key, path in entries]
  276. def recover_pending_to_db(session_db=None, *, session_resolver=None) -> int:
  277. """Replay flush-dir ``*.json`` files into state.db; return the number of messages recovered.
  278. See :func:`recover_pending_spool`, which also reports the sessions it held back."""
  279. return recover_pending_spool(session_db, session_resolver=session_resolver)[0]
  280. def recover_pending_spool(session_db=None, *, session_resolver=None) -> tuple[int, set[str]]:
  281. """Replay flush-dir ``*.json`` files via ``SessionDB.append_message``, deleting each on success.
  282. ``session_db=None`` opens (and afterwards releases) the shared default ``state.db``.
  283. ``session_resolver`` (optional ``(session_key, not_after=ts) -> (session_id, db) | None``, e.g.
  284. ``SessionStore.resolve_session_id_for_key``) is required for real flush files: adapter
  285. ``MessageEvent`` objects carry no ``session_id``, so without it every recovery lands in the skip
  286. branch. A returned ``db`` routes the append to the profile store owning the key (multiplexed
  287. gateways); ``None`` falls back to ``session_db``. Returns ``(recovered, held_back)``: the number of
  288. messages recovered, and the session ids whose spool files this pass held back after a failed
  289. replay, so the live writer can drain them before that session's next row.
  290. """
  291. flush_files = _order_flush_files(_get_flush_dir().glob("*.json"))
  292. if not flush_files:
  293. return 0, set()
  294. own_db = session_db is None
  295. if own_db:
  296. from hermes_state_registry import acquire
  297. session_db = acquire()
  298. recovered = 0
  299. # Sessions whose spool replay already failed this pass -> files held back for them. Ordering is a
  300. # per-session property, so one unhealthy session must not hold back the others.
  301. blocked_sessions: dict[str, int] = {}
  302. try:
  303. for path in flush_files:
  304. if path in _REPLAYED_UNREMOVABLE:
  305. continue
  306. # One unparseable payload or rejected append must only skip THIS file: the file is
  307. # never unlinked, so aborting the pass would re-poison every later boot.
  308. try:
  309. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  310. # Agent-history snapshots use a different schema (reason +
  311. # messages list) and are meant for manual operator recovery,
  312. # not automatic DB insertion. Skip them silently.
  313. if payload.get("reason") == AGENT_HISTORY_REASON:
  314. continue
  315. if _recover_one_payload(session_db, path, payload,
  316. session_resolver=session_resolver,
  317. blocked_sessions=blocked_sessions):
  318. recovered += 1
  319. _remove_replayed(path, payload.get("session_key", ""))
  320. # health: allow BLE001 -- per-file boundary moved unchanged from recover_pending_to_db; a traceback per locked-DB file would only add noise
  321. except Exception as exc:
  322. logger.warning("Failed to recover pending message from %s: %s", path, exc)
  323. finally:
  324. if own_db: # shutdown cancellation/interrupt must not strand an owned DB
  325. with contextlib.suppress(Exception):
  326. from hermes_state_registry import release_or_close
  327. release_or_close(session_db)
  328. if recovered:
  329. logger.info("Recovered %d pending message(s) from shutdown flush", recovered)
  330. if blocked_sessions:
  331. # Otherwise a spool dir that never empties gives an operator no reason why.
  332. logger.info("Held back %d spooled transcript file(s) for the next start after a failed replay: %s",
  333. sum(blocked_sessions.values()),
  334. ", ".join(f"{sid} ({count})" for sid, count in blocked_sessions.items()))
  335. return recovered, set(blocked_sessions)
  336. def _recover_one_payload(session_db, path: Path, payload: dict[str, Any], *,
  337. session_resolver=None, blocked_sessions: dict[str, int]) -> bool:
  338. """Append one flush payload to ``session_db``; False when it was not replayed (the file is kept,
  339. or quarantined when the database rejected the row itself)."""
  340. # Cap-dropped transcript payloads carry the full message dict keyed by session_id: replay it
  341. # directly (#78182). This handles spool files that were never drained before a restart.
  342. if payload.get("reason") == TRANSCRIPT_CAP_DROP_REASON:
  343. from gateway.session_transcript import transcript_append_kwargs
  344. data = payload.get("data", {}) or {}
  345. spooled_sid, message = data.get("session_id", ""), data.get("message")
  346. if not spooled_sid or not isinstance(message, dict):
  347. logger.warning("Cannot recover structurally invalid transcript spool "
  348. "file %s; preserved for manual inspection", path)
  349. return False
  350. if spooled_sid in blocked_sessions:
  351. # An older message for this session could not be replayed. Writing this one now would give
  352. # it a lower row id than the message it follows, permanently inverting the transcript, so
  353. # leave it for the next start.
  354. blocked_sessions[spooled_sid] += 1
  355. return False
  356. try:
  357. session_db.append_message(**transcript_append_kwargs(spooled_sid, message, fallback_ts=payload.get("ts")))
  358. except Exception as exc:
  359. if is_row_rejection(exc): # no retry can write it, so it must not hold the session back
  360. _quarantine_spool_file(path, spooled_sid, exc)
  361. return False
  362. # Same contract as drain_transcript_spool: stop this session's replay on the first failure
  363. # and keep the remaining spool files for the next attempt.
  364. blocked_sessions[spooled_sid] = 1
  365. raise
  366. return True
  367. session_key, data = payload.get("session_key", ""), payload.get("data", {})
  368. text = data.get("text", "")
  369. if not text or not session_key:
  370. logger.warning("Cannot recover structurally invalid pending message from %s; "
  371. "the flush file has been preserved", path)
  372. return False
  373. # session_key is a gateway routing key (e.g. "agent:main:telegram:..."); appending a row
  374. # needs the real session_id, which real payloads lack — the resolver supplies it together with
  375. # the store owning the key. ``session_db`` (the owned default) serves only payloads that already
  376. # carry a session_id; a resolver-resolved payload goes to the resolver's db alone, never the
  377. # ambient root store (a None db from the resolver is not a fallback signal — it is "preserve").
  378. session_id, target_db = data.get("session_id", ""), session_db
  379. if not session_id and session_resolver is not None:
  380. try:
  381. resolved = session_resolver(session_key, not_after=payload.get("ts"))
  382. except Exception as exc:
  383. logger.debug("Session key->id resolution failed for %s: %s", session_key, exc)
  384. resolved = None
  385. if resolved and resolved[1] is not None:
  386. session_id, target_db = resolved
  387. if not session_id:
  388. logger.warning("Cannot recover pending message for %s: no session_id in flush file and "
  389. "session_key-to-id resolution failed. "
  390. "The message text is preserved in %s", session_key, path)
  391. return False
  392. target_db.append_message(session_id=session_id, role="user", content=text,
  393. timestamp=payload.get("ts", int(time.time())))
  394. return True
  395. def recover_gateway_pending(runner) -> int:
  396. """Replay every ``pending_messages`` spool this gateway owns into state.db; return the count.
  397. ``_get_flush_dir`` follows the active HERMES_HOME, so a routed turn on a multiplexed gateway spools
  398. its stalled transcript backlog under ``profiles/<name>/`` and the runtime drain forgets it on
  399. restart. After the launch home, replay each served profile inside its own home so the default
  400. store ``recover_pending_spool`` opens is that profile's state.db (#123584). The sessions a pass
  401. holds back are marked on the session store, so their next live row drains the spool first.
  402. """
  403. from gateway.run import _multiplex_profile_homes
  404. from hermes_constants import get_hermes_home, reset_hermes_home_override, set_hermes_home_override
  405. store = runner.session_store
  406. recovered, held_back = recover_pending_spool(session_resolver=store.resolve_session_id_for_key)
  407. store.mark_spooled_drop_sessions(held_back)
  408. if not getattr(runner.config, "multiplex_profiles", False):
  409. return recovered
  410. launch_home = Path(get_hermes_home()).resolve()
  411. for name, home in _multiplex_profile_homes(runner.config):
  412. if Path(home).resolve() == launch_home or not (Path(home) / "pending_messages").is_dir():
  413. continue
  414. token = set_hermes_home_override(str(home))
  415. try:
  416. count, held_back = recover_pending_spool(session_resolver=store.resolve_session_id_for_key)
  417. except Exception: # one profile's unreadable spool must not strand the others'
  418. logger.warning("Pending-message recovery failed for profile %s", name, exc_info=True)
  419. continue
  420. finally:
  421. reset_hermes_home_override(token)
  422. recovered += count
  423. store.mark_spooled_drop_sessions(held_back)
  424. return recovered
  425. def flush_agent_history_to_file(session_id: Optional[str], history: list) -> None:
  426. """Best-effort dump of an agent's in-memory transcript before teardown. Used when
  427. ``_flush_messages_to_session_db`` raises (e.g. FTS/SQLite corruption): the transcript is written
  428. outside the broken DB so an operator can salvage it after repairing state.db. Failures are
  429. swallowed — shutdown must never block on a best-effort backup."""
  430. if not history:
  431. return
  432. try:
  433. flush_dir = _get_flush_dir()
  434. snapshot = []
  435. for _m in history:
  436. try:
  437. plain = isinstance(_m, (dict, list, str, int, float, bool, type(None)))
  438. snapshot.append(_m if plain else str(_m))
  439. except Exception:
  440. continue
  441. _write_payload(flush_dir, {
  442. "reason": AGENT_HISTORY_REASON, "issue": "#72680",
  443. "session_id": session_id, "count": len(snapshot), "messages": snapshot,
  444. })
  445. logger.warning("Preserved %d in-memory message(s) for session %s "
  446. "(possible FTS corruption — recover after repairing state.db)",
  447. len(snapshot), session_id)
  448. except Exception as _e:
  449. logger.warning("Agent-history shutdown preservation failed for session %s: %s", session_id,
  450. _e)