session_persistence.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679
  1. """SessionStore storage plumbing: per-profile SessionDB handle resolution and the routing-index
  2. load/save paths (state.db gateway_routing primary, sessions.json legacy mirror). Mixin split out of
  3. ``gateway/session.py``; bound onto ``SessionStore`` via the MRO."""
  4. from __future__ import annotations
  5. import contextlib
  6. import logging
  7. import json
  8. import threading
  9. from pathlib import Path
  10. from typing import TYPE_CHECKING, Any, Dict, Optional
  11. from utils import atomic_json_write
  12. if TYPE_CHECKING:
  13. from gateway.session import SessionEntry, SessionSource
  14. # Log-record parity with the origin module.
  15. logger = logging.getLogger("gateway.session")
  16. # "No SessionDB pinned" sentinel: lets ``_db`` distinguish "resolve from the active scope" from a
  17. # deliberate ``store._db = None`` (JSONL fallback).
  18. _DB_UNPINNED = object()
  19. # Self-documenting sentinel written first into sessions.json; "_" keys are skipped on load.
  20. _SESSIONS_JSON_README = (
  21. "LEGACY MIRROR of the gateway routing index (the primary copy lives in the gateway_routing "
  22. "table in ~/.hermes/state.db). Maps messaging session keys (agent:main:<platform>:...) to "
  23. "active session IDs. This is NOT the session list. ALL sessions (CLI, TUI, and gateway) live "
  24. "in ~/.hermes/state.db and are shown by `hermes sessions list` and `/sessions`. Disable this "
  25. "file with `gateway.write_sessions_json: false` in config.yaml."
  26. )
  27. def _is_live_system_guard(exc: BaseException) -> bool:
  28. """Test-isolation guard: must stay a loud failure and is never cached."""
  29. return isinstance(exc, RuntimeError) and "live-system guard" in str(exc)
  30. class SessionPersistenceMixin:
  31. """SessionStore storage plumbing: SessionDB handle resolution and routing-index load/save."""
  32. def retire_runtime_sessions(self, session_ids):
  33. """Publish committed retirement; durable fences reject delayed saves."""
  34. with self._lock:
  35. targets = set(session_ids)
  36. for key, entry in list(self._entries.items()):
  37. if entry.session_id in targets:
  38. del self._entries[key]
  39. for sid in targets:
  40. self._session_owner_hints.pop(sid, None)
  41. def fence_retired_routes(self, session_ids, *, namespace, expiring=()) -> None:
  42. """Durably retire *session_ids* in the routing index's own database (fences + route
  43. drop in one transaction). A delete committed in another profile's store fences only that
  44. store; the shared index lives in the launch home, so its saves need the fence here too,
  45. keyed by the deleting profile's routing *namespace* (``retire_gateway_routes``).
  46. Raises when the routing store cannot take the fence: the delete must not report success."""
  47. retire = self._routing_db_method("retire_gateway_routes")
  48. if retire is None:
  49. raise RuntimeError("routing store unavailable for deletion fence")
  50. retire(session_ids, namespace=namespace, expiring=expiring)
  51. def strip_retired_mirror_routes(self, session_ids=None) -> None:
  52. """Drop from an existing sessions.json every route a routing save now skips as retired
  53. (``route_retired``, the check ``replace_gateway_routing_entries`` applies), only those naming
  54. *session_ids* when given. Runs whatever ``write_sessions_json`` says and never creates the
  55. file: the mirror is derived, and a retired route left in it would come back through the
  56. next start's legacy import once a prune's fence ages out."""
  57. db = self._routing_db
  58. sessions_file = self.sessions_dir / "sessions.json"
  59. if db is None or not sessions_file.exists():
  60. return
  61. from hermes_state_mutation_retirement import route_retired
  62. targets = None if session_ids is None else set(session_ids)
  63. with self._lazy("_save_lock", threading.Lock): # never interleaves with a full save's write
  64. with open(sessions_file, "r", encoding="utf-8-sig") as f:
  65. data = json.load(f)
  66. if not isinstance(data, dict):
  67. return
  68. kept = {}
  69. for key, entry in data.items():
  70. sid = entry.get("session_id") if isinstance(entry, dict) else None
  71. if (key.startswith("_") or not isinstance(sid, str) or (targets is not None and sid not in targets)
  72. or not route_retired(db, key, json.dumps({"session_id": sid}))):
  73. kept[key] = entry
  74. if len(kept) < len(data):
  75. atomic_json_write(sessions_file, kept, mode=0o600)
  76. def _open_session_db_for_active_scope(self, db_path: Optional[Path] = None):
  77. """SessionDB for the active profile scope. ``db_path`` pins the store; otherwise
  78. ``_default_db_path()`` follows the context-local HERMES_HOME (resolved per call so
  79. multiplexed profiles reach their own store). Handles are cached per path; failed opens enter
  80. a bounded backoff during which callers keep using the JSONL fallback.
  81. Resolving here rather than once in ``__init__`` is the whole fix for #88532: it lets the scoping
  82. that the multiplexed inbound path already performs actually reach session storage.
  83. """
  84. from hermes_state import _default_db_path
  85. from hermes_state_registry import acquire
  86. path = Path(db_path) if db_path is not None else Path(_default_db_path())
  87. def _open():
  88. try:
  89. return acquire(path) # process-wide registry: one writer per path
  90. except Exception as e:
  91. if not _is_live_system_guard(e):
  92. print(f"[gateway] Warning: SQLite session store unavailable, falling back to JSONL: {e}")
  93. raise
  94. return self._db_handle_cache.get(path, _open, non_cacheable=_is_live_system_guard)
  95. def _pinned_db(self):
  96. """Return the explicitly pinned DB (``store._db = x``), else ``_DB_UNPINNED``."""
  97. return getattr(self, "_db_pinned", _DB_UNPINNED)
  98. @property
  99. def _db(self):
  100. """The SessionDB for the active profile scope, or a pinned override. Assigning ``store._db``
  101. pins that value for every subsequent read (tests install a fake or disable the DB with
  102. ``store._db = None``); unpinned, each read resolves the scope so a multiplexed profile's
  103. writes reach its own store."""
  104. pinned = self._pinned_db()
  105. return self._open_session_db_for_active_scope() if pinned is _DB_UNPINNED else pinned
  106. @_db.setter
  107. def _db(self, value) -> None:
  108. self._db_pinned = value
  109. @property
  110. def _routing_db(self):
  111. """The one store that owns the routing index, whatever scope is active. ``_entries`` is one
  112. flat dict holding every profile's keys, so it must persist to ONE file (``_routing_home``),
  113. not whichever profile is scoped — otherwise a mid-turn rewrite and the unscoped startup
  114. load see different copies and crash markers under a secondary profile go unrecovered. A
  115. pinned handle still wins; bare test instances lacking the handle cache report no DB.
  116. Reading it through ``_db`` made that file whichever profile happened to be scoped at the time: a
  117. whole-index rewrite during one profile's turn copied every other profile's routing rows into that
  118. profile's store, and startup — which runs unscoped — then loaded a different copy than the one the
  119. last writer produced. See #66887.
  120. """
  121. pinned = self._pinned_db()
  122. if pinned is not _DB_UNPINNED:
  123. return pinned
  124. home = getattr(self, "_routing_home", None)
  125. try:
  126. if home is None:
  127. return self._db
  128. return self._open_session_db_for_active_scope(db_path=home / "state.db")
  129. except Exception:
  130. return None
  131. def _named_profile_for_key(self, session_key: Optional[str]) -> Optional[str]:
  132. """The non-default profile that owns *session_key*, or None (ambient store is authoritative:
  133. multiplexing off, or legacy ``agent:main``). Deliberately does NOT cover "that profile has
  134. no directory" — ownership and resolvability are separate questions for ``_db_for_key``."""
  135. if not getattr(self.config, "multiplex_profiles", False):
  136. return None
  137. profile = self._profile_from_session_key(session_key)
  138. return None if not profile or profile == "default" else profile
  139. def _profile_home_for_key(self, session_key: Optional[str]) -> Optional[Path]:
  140. """HERMES_HOME of the profile owning *session_key*, or None (no named owner or
  141. unresolvable)."""
  142. profile = self._named_profile_for_key(session_key)
  143. if profile is None:
  144. return None
  145. cache = self._profile_home_cache
  146. if profile in cache:
  147. return cache[profile]
  148. home: Optional[Path] = None
  149. try:
  150. from hermes_cli.profiles import get_profile_dir, profile_exists
  151. if profile_exists(profile):
  152. home = Path(get_profile_dir(profile))
  153. except Exception as exc:
  154. logger.debug("Could not resolve profile home for %r: %s", session_key, exc)
  155. home = None
  156. # Only hits are memoized: a profile directory can be provisioned *after* startup (enrollment
  157. # bridge), and a cached miss would pin that profile's rows to the ambient store for life.
  158. if home is not None:
  159. cache[profile] = home
  160. return home
  161. def _db_for_key(self, session_key: Optional[str]):
  162. """The SessionDB holding *session_key*'s rows, whatever scope is active (the owning profile
  163. is encoded in the key). ``_db`` follows the ambient HERMES_HOME that only the inbound message
  164. path installs; unscoped background work (expiry watcher) would otherwise write profile rows
  165. into the ROOT store until the stale-route self-heal drops a live conversation.
  166. Background work runs unscoped while operating on every profile's keys out of the single process-wide
  167. ``_entries`` dict — ``_session_housekeeping_watcher`` is the clearest case — so it reads and writes the
  168. ROOT store for rows that actually live under ``profiles/<name>/state.db``. The two writers then
  169. drift apart on the same logical session until the routing index disagrees with the row and the
  170. #54878 self-heal drops a live conversation (#66887).
  171. """
  172. pinned = self._pinned_db()
  173. if pinned is not _DB_UNPINNED:
  174. return pinned
  175. profile = self._named_profile_for_key(session_key)
  176. if profile is None:
  177. # Default-profile (``agent:main``) rows belong to the launch home, not to whichever
  178. # profile's scope happens to be active: a scoped drain tick or cron mirror touching a
  179. # default chat used to write its rows into the secondary's store (#102157's picture).
  180. routing_home = getattr(self, "_routing_home", None)
  181. if routing_home is None or not getattr(self.config, "multiplex_profiles", False):
  182. return self._db
  183. try:
  184. return self._open_session_db_for_active_scope(db_path=routing_home / "state.db")
  185. except Exception:
  186. return None
  187. home = self._profile_home_for_key(session_key)
  188. if home is None:
  189. # Falling back to the ambient store would split ONE session identity across two
  190. # physical stores — fail closed; callers already handle a missing DB.
  191. logger.warning(
  192. "gateway.session: profile %r has no resolvable home (key %r); refusing to fall "
  193. "back to the ambient store", profile, session_key)
  194. return None
  195. try:
  196. return self._open_session_db_for_active_scope(db_path=home / "state.db")
  197. except Exception:
  198. return None # same contract as ``_db``: a failed open degrades to JSONL fallback
  199. def _owner_key_for_session_id(self, session_id: Optional[str]) -> Optional[str]:
  200. """The routing key that owns *session_id*, or None. The published index is authoritative;
  201. ``_session_owner_hints`` covers the window where ownership is proven but routing not yet
  202. published. Deliberately lock-free: several callers already hold ``_lock``."""
  203. if not session_id:
  204. return None
  205. try:
  206. for entry in list(self._entries.values()):
  207. if entry.session_id == session_id:
  208. return entry.session_key
  209. except Exception:
  210. pass # bare stores / foreign entry objects in suites
  211. return (getattr(self, "_session_owner_hints", None) or {}).get(session_id)
  212. def _db_for_session_id(self, session_id: Optional[str]):
  213. """The SessionDB holding *session_id*'s row (owner from the index or a pre-published hint;
  214. unknown ids fall back to the ambient store)."""
  215. if not session_id:
  216. return self._db
  217. return self._db_for_key(self._owner_key_for_session_id(session_id))
  218. def close_all_db_handles(self) -> None:
  219. """Close every SessionDB handle this store opened (one per path). Closing only ``store._db``
  220. would strand secondary profiles' handles with their WAL lock held ('database is locked' on
  221. restart). Drained under the lock, closed outside it; a pinned handle is the pinner's."""
  222. def _close(db) -> None:
  223. from hermes_state_registry import release_or_close # shared instances no-op on close()
  224. try:
  225. release_or_close(db)
  226. except Exception as exc:
  227. logger.debug("SessionDB close error during handle sweep: %s", exc)
  228. self._db_handle_cache.close_all(_close)
  229. def _ensure_loaded(self) -> None:
  230. """Load sessions index from disk if not already loaded."""
  231. with self._lock:
  232. self._ensure_loaded_locked()
  233. def _entry_locked(self, session_key: str) -> Optional[SessionEntry]:
  234. """Load the index and return the entry for *session_key*. Lock held."""
  235. self._ensure_loaded_locked()
  236. return self._entries.get(session_key)
  237. def _routing_scope(self) -> str:
  238. """Namespace for this store's gateway_routing rows: the resolved sessions_dir, so stores
  239. with different dirs never share entries."""
  240. try:
  241. return str(Path(self.sessions_dir).resolve())
  242. except Exception:
  243. return str(self.sessions_dir)
  244. def _routing_db_method(self, name: str):
  245. """Bound ``_routing_db.<name>`` if the handle exists and has it, else None."""
  246. method = getattr(self._routing_db or None, name, None)
  247. return method if callable(method) else None
  248. @staticmethod
  249. def _chat_labels_meta_key(platform, scope_id, chat_id) -> str:
  250. return f"gateway_chat_labels:{getattr(platform, 'value', platform)}:{scope_id or ''}:{chat_id}"
  251. def record_chat_labels(self, source: SessionSource) -> bool:
  252. """Persist *source*'s ``chat_name`` / ``chat_topic`` as the chat's current labels, at the
  253. moment they are observed; True once written. A session origin cannot answer "what is this
  254. chat called now": it is written when its session is created, which can be after a newer
  255. observation (a delayed first turn), and a reset inherits it under a fresh ``created_at``.
  256. False when there is no database handle right now (open failed, backoff, JSONL fallback):
  257. nothing was written, so the caller must not treat these labels as recorded."""
  258. setter = self._routing_db_method("set_meta")
  259. if setter is None:
  260. return False
  261. setter(self._chat_labels_meta_key(source.platform, source.scope_id, source.chat_id),
  262. json.dumps([source.chat_name, source.chat_topic]))
  263. return True
  264. def chat_labels(self, platform, scope_id, chat_id) -> Optional[tuple]:
  265. """The ``(chat_name, chat_topic)`` last recorded for a chat, or None when none was. Raises
  266. when the store cannot be read, including when there is no database handle right now:
  267. "unreadable now" is not "never labelled"."""
  268. getter = self._routing_db_method("get_meta")
  269. if getter is None:
  270. raise LookupError("session database unavailable")
  271. raw = getter(self._chat_labels_meta_key(platform, scope_id, chat_id))
  272. if not raw:
  273. return None
  274. chat_name, chat_topic = json.loads(raw)
  275. return chat_name, chat_topic
  276. def _load_routing_rows_locked(self) -> bool:
  277. """Load state.db routing entries into ``_entries``; False when there is no loader or the
  278. load failed (warned). Lock held."""
  279. loader = self._routing_db_method("load_gateway_routing_entries")
  280. if loader is None:
  281. return False
  282. try:
  283. for key, entry_json in loader(scope=self._routing_scope()).items():
  284. entry = self._routing_entry_from_json(key, entry_json)
  285. if entry is not None:
  286. self._entries[key] = entry
  287. return True
  288. except Exception as e:
  289. logger.warning("gateway.session: state.db routing load failed: %s", e)
  290. return False
  291. @staticmethod
  292. def _routing_entry_from_json(key: str, entry_json: str) -> Optional[SessionEntry]:
  293. """Parse one gateway_routing row; None (with a warning) when invalid."""
  294. from gateway.session import SessionEntry
  295. try:
  296. entry_data = json.loads(entry_json)
  297. if isinstance(entry_data, dict):
  298. return SessionEntry.from_dict(entry_data)
  299. except (ValueError, KeyError, TypeError) as e:
  300. logger.warning("Skipping invalid routing entry %r: %s", key, e)
  301. return None
  302. def _ensure_loaded_locked(self) -> None:
  303. """Load the routing index (lock held). state.db ``gateway_routing`` is primary;
  304. sessions.json is the legacy import for keys the DB lacks (persisted on the next _save).
  305. Read order (#9006 follow-up): the ``gateway_routing`` table in state.db is the primary source;
  306. sessions.json is the legacy import path for pre-migration installs (its entries are folded in for
  307. keys the DB doesn't have, then persisted to the DB on the next _save).
  308. """
  309. if self._loaded:
  310. self._reconcile_recovered_routing_locked()
  311. return
  312. self.sessions_dir.mkdir(parents=True, exist_ok=True)
  313. db_load_succeeded = self._load_routing_rows_locked()
  314. db_had_entries = db_load_succeeded and bool(self._entries)
  315. mirror_retired = self._import_legacy_sessions_json(db_had_entries)
  316. self._loaded = True
  317. self._routing_db_loaded = db_load_succeeded
  318. # Heal before the fallback baseline: a heal after it reads as a caller edit, and recovery
  319. # would then keep the stale mirror entry over the newer DB row.
  320. self._heal_missing_origins(self._entries.values())
  321. self._routing_fallback_baseline = None if db_load_succeeded else self._entries_as_dicts()
  322. if mirror_retired and db_load_succeeded:
  323. # Rewrite the mirror while the fences that kept those routes out still hold: a prune's
  324. # ages out, and the import would then restore the retired route.
  325. self._save()
  326. if not getattr(self, "_write_sessions_json", True):
  327. # That save left an existing mirror as it was: strip the routes from it directly,
  328. # or every later start imports them again.
  329. try:
  330. self.strip_retired_mirror_routes()
  331. except Exception:
  332. logger.warning("gateway.session: could not strip retired routes from sessions.json", exc_info=True)
  333. # A hard crash skips graceful shutdown and leaves sessions.json pointing at ended sessions.
  334. self._prune_stale_sessions_locked()
  335. def _heal_missing_origins(self, entries) -> None:
  336. for entry in entries:
  337. if entry.origin is None:
  338. entry.origin = self._stored_origin(entry)
  339. def _stored_origin(self, entry: SessionEntry) -> Optional[SessionSource]:
  340. """*entry*'s messaging source rebuilt from its conversation row, if it still routes to the
  341. entry's key. A routing entry persisted without an origin (#146696) otherwise leaves restart
  342. resume, route recovery and wakes nothing to route by until the chat speaks again. Local and
  343. API routes are left alone: their receipts, not the row, name their owner."""
  344. import sqlite3
  345. from gateway.session_route_recovery import _messaging_platform, _row_source
  346. db = self._db_for_key(entry.session_key)
  347. try:
  348. row = db.get_session(entry.session_id) if db is not None else None
  349. except sqlite3.Error:
  350. return None
  351. platform = row and _messaging_platform(entry.platform.value if entry.platform else row['source'])
  352. source = _row_source(row, platform, self._named_profile_for_key(entry.session_key)) if platform else None
  353. if source is None or source.platform != platform:
  354. return None
  355. return source if self._generate_session_key(source) == entry.session_key else None
  356. def _import_legacy_sessions_json(self, db_had_entries: bool) -> int:
  357. """Legacy import: sessions.json fills only keys the DB lacks. Lock held. Returns how many
  358. mirror entries it skipped as retired routes."""
  359. from gateway.session import SessionEntry
  360. sessions_file = self.sessions_dir / "sessions.json"
  361. retired = 0
  362. if not sessions_file.exists():
  363. return retired
  364. try:
  365. with open(sessions_file, "r", encoding="utf-8-sig") as f:
  366. data = json.load(f)
  367. imported = 0
  368. for key, entry_data in data.items():
  369. # "_"-prefixed keys are sentinels (e.g. "_README"), not entries.
  370. if key.startswith("_") or key in self._entries:
  371. continue
  372. if not isinstance(entry_data, dict): # corrupt file must not abort the whole load
  373. logger.warning(
  374. "Skipping invalid session entry %r: expected dict, got %s", key,
  375. type(entry_data).__name__)
  376. continue
  377. try:
  378. from hermes_state_mutation_retirement import route_retired
  379. db = self._routing_db
  380. if db is not None and route_retired(db, key, json.dumps(
  381. {'session_id': entry_data['session_id']})):
  382. retired += 1
  383. continue
  384. self._entries[key] = SessionEntry.from_dict(entry_data)
  385. imported += 1
  386. except (ValueError, KeyError, TypeError) as e:
  387. logger.warning("Skipping invalid session entry %r: %s", key, e)
  388. if imported and db_had_entries:
  389. logger.info(
  390. "gateway.session: imported %d legacy sessions.json entr%s missing from "
  391. "state.db routing table", imported, "y" if imported == 1 else "ies")
  392. except Exception as e:
  393. print(f"[gateway] Warning: Failed to load sessions: {e}")
  394. return retired
  395. def _prune_stale_sessions_locked(self) -> None:
  396. """Remove routing entries whose session has ended in state.db (startup, lock held). Stale ==
  397. ``end_reason IS NOT NULL``; rows absent from the DB are kept; a ``None`` DB handle is a
  398. no-op; DB errors are non-fatal."""
  399. if not self._entries:
  400. return
  401. stale_keys: list = []
  402. recovered_keys = 0
  403. try:
  404. for key, entry in self._entries.items():
  405. # Ask the store that owns the key, not the ambient handle, or a live
  406. # secondary-profile session gets pruned on the root copy.
  407. db = self._db_for_key(key)
  408. if db is None:
  409. continue
  410. row = db.get_session(entry.session_id)
  411. if row is None or row.get("end_reason") is None:
  412. continue
  413. verdict = self._stale_entry_verdict(key, entry, row)
  414. if verdict == "prune":
  415. stale_keys.append(key)
  416. elif verdict is not None:
  417. self._entries[key] = verdict
  418. recovered_keys += 1
  419. except Exception as exc:
  420. logger.warning("gateway.session: stale-entry pruning skipped due to DB error: %s", exc)
  421. return
  422. for key in stale_keys:
  423. del self._entries[key]
  424. if stale_keys or recovered_keys:
  425. self._save()
  426. def _stale_entry_verdict(self, key: str, entry, row):
  427. """For a routing entry whose row has ended: ``"prune"``, a replacement entry (repoint), or
  428. None (keep as-is)."""
  429. from gateway.session_lifecycle import _now
  430. local_target = self._local_receipt_target(entry)
  431. if local_target is not None:
  432. # The authority's receipt names the route's target (a compression child after a
  433. # split); its first admitted turn reopens a stamped row. A peer-row recovery here
  434. # adopted an empty, never-admitted replacement instead (#140642).
  435. if local_target == entry.session_id:
  436. return None
  437. import copy
  438. repointed = copy.copy(entry)
  439. repointed.session_id = local_target
  440. logger.warning(
  441. "gateway.session: repointing stale sessions.json entry %r from ended %s "
  442. "(end_reason=%r) to its local receipt target %s", key, entry.session_id,
  443. row["end_reason"], local_target)
  444. return repointed
  445. recovered_entry = None
  446. if entry.origin is not None:
  447. try:
  448. recovered_entry = self._recover_session_from_db(
  449. session_key=key, source=entry.origin, now=_now(), raise_on_lookup_error=True)
  450. except Exception as exc:
  451. # Indeterminate: keep the only routing handle.
  452. logger.debug(
  453. "gateway.session: recovery lookup failed for stale sessions.json entry %r -> "
  454. "%s: %s", key, entry.session_id, exc)
  455. return None
  456. # Compression-ended parent with a newer live child for the same peer: repoint instead of
  457. # dropping, or queued/resume-pending work vanishes until the next message.
  458. if recovered_entry is not None and recovered_entry.session_id != entry.session_id:
  459. logger.warning(
  460. "gateway.session: repointing stale sessions.json entry %r from ended %s "
  461. "(end_reason=%r) to recovered %s", key, entry.session_id, row["end_reason"],
  462. recovered_entry.session_id)
  463. if entry.prompt_pin and (
  464. self._compression_tip_for_session_id(entry.session_id) == recovered_entry.session_id
  465. ):
  466. # The compression child continues this conversation: its internal turns keep the
  467. # pinned system bytes (the rebuilt entry is minimal and would drop them).
  468. recovered_entry.prompt_pin = entry.prompt_pin
  469. return recovered_entry
  470. # Same-id recovery == successful resume: keep the ORIGINAL entry object (the recovered one
  471. # is rebuilt minimal and would drop counters, model_override, resume markers, metadata).
  472. # A non-None recovery with the SAME session id is a successful resume (all recovery gates passed,
  473. # row reopened): keep the routing entry — it is proven valid, not a dead route (#95957). Nothing in
  474. # sessions.json changes, so no save is needed for this branch.
  475. if recovered_entry is not None:
  476. logger.info(
  477. "gateway.session: reopened ended session %s for sessions.json entry %r "
  478. "(end_reason=%r); keeping route", entry.session_id, key, row["end_reason"])
  479. return None
  480. logger.warning(
  481. "gateway.session: pruning stale sessions.json entry %r -> %s (end_reason=%r); left by "
  482. "a crashed gateway", key, entry.session_id, row["end_reason"])
  483. return "prune"
  484. def _entries_as_dicts(self) -> dict[str, Any]:
  485. """Serializable snapshot of ``_entries``. Lock held."""
  486. return {key: entry.to_dict() for key, entry in self._entries.items()}
  487. def _save(self) -> None:
  488. """Persist the routing index while the caller holds ``_lock``."""
  489. self._persist_routing_data(*self._snapshot_routing_locked())
  490. def _next_routing_generation_locked(self) -> int:
  491. """Bump and return the shared routing counter (lock held). Full snapshots AND single-entry
  492. fast saves MUST allocate from this one counter: the stale-write protection is a total order
  493. over serialization times and silently breaks otherwise."""
  494. self._routing_generation = getattr(self, "_routing_generation", 0) + 1
  495. return self._routing_generation
  496. def _reconcile_recovered_routing_locked(self) -> None:
  497. """Merge authoritative rows after a fallback-only startup load."""
  498. baseline = getattr(self, "_routing_fallback_baseline", None)
  499. if getattr(self, "_routing_db_loaded", False) or baseline is None:
  500. return
  501. loader = self._routing_db_method("load_gateway_routing_entries")
  502. if loader is None:
  503. return
  504. try:
  505. durable = loader(scope=self._routing_scope())
  506. except Exception as exc:
  507. logger.warning("gateway.session: recovered state.db routing load failed: %s", exc)
  508. return
  509. current = self._entries_as_dicts()
  510. for key, entry_json in durable.items():
  511. durable_entry = self._routing_entry_from_json(key, entry_json)
  512. if durable_entry is None:
  513. continue
  514. if key not in baseline:
  515. # A key created while on fallback wins over a DB-only key; otherwise restore the
  516. # authoritative row that fallback never saw.
  517. self._entries.setdefault(key, durable_entry)
  518. elif key not in current:
  519. continue # loaded from fallback and deliberately removed
  520. elif current[key] == baseline[key]:
  521. self._entries[key] = durable_entry # unchanged fallback data yields to the DB copy
  522. self._heal_missing_origins(self._entries.values())
  523. self._routing_db_loaded = True
  524. self._routing_fallback_baseline = None
  525. def _snapshot_routing_locked(self) -> tuple[dict[str, Any], int]:
  526. """Capture immutable routing data and a monotonic generation."""
  527. self._reconcile_recovered_routing_locked()
  528. return self._entries_as_dicts(), self._next_routing_generation_locked()
  529. def _persist_routing_data(self, data: dict[str, Any], generation: int) -> None:
  530. """Serialize all whole-index writers through one durable write lock."""
  531. with self._lazy("_save_lock", threading.Lock):
  532. if generation <= getattr(self, "_persisted_routing_generation", 0):
  533. return
  534. # Fold in fast upserts numbered above this snapshot: they were serialized after us and
  535. # a delayed full rewrite must not regress them.
  536. fast_persisted = getattr(self, "_fast_persisted_entries", None)
  537. if fast_persisted:
  538. for key, (revision, entry_json) in fast_persisted.items():
  539. if revision > generation:
  540. data[key] = json.loads(entry_json)
  541. db_saved = False
  542. replacer = self._routing_db_method("replace_gateway_routing_entries")
  543. if replacer is not None:
  544. try:
  545. retired = replacer({k: json.dumps(v) for k, v in data.items()}, scope=self._routing_scope())
  546. db_saved = True
  547. # A retired route the save skipped stays out of the mirror too: its fence may
  548. # age out (a prune's), and the next start's legacy import would restore it.
  549. if retired:
  550. data = {k: v for k, v in data.items() if k not in retired}
  551. except Exception as exc:
  552. logger.warning("gateway.session: state.db routing save failed: %s", exc)
  553. if getattr(self, "_write_sessions_json", True) or not db_saved:
  554. try:
  555. self._save_sessions_json(data)
  556. except Exception as exc:
  557. if not db_saved:
  558. raise
  559. # state.db is authoritative: a failed legacy mirror must not report the
  560. # already-committed primary write as failed.
  561. logger.warning(
  562. "gateway.session: sessions.json mirror save failed after state.db commit: "
  563. "%s", exc)
  564. self._persisted_routing_generation = generation
  565. # This rewrite supersedes fast records at or below its generation; newer ones stay for
  566. # the next delayed full writer.
  567. if fast_persisted:
  568. for key in [k for k, (rev, _) in fast_persisted.items() if rev <= generation]:
  569. del fast_persisted[key]
  570. def _save_sessions_json(self, data: dict[str, Any]) -> None:
  571. """Write the legacy sessions.json mirror of the routing index (atomic + fsync)."""
  572. atomic_json_write(self.sessions_dir / "sessions.json", {"_README": _SESSIONS_JSON_README, **data}, mode=0o600)
  573. def _save_entries(self) -> None:
  574. """Snapshot latest state under ``_lock`` and persist after releasing it."""
  575. with self._lock:
  576. data, generation = self._snapshot_routing_locked()
  577. self._persist_routing_data(data, generation)
  578. def _save_entry(
  579. self, session_key: str, *, entry_data: Optional[dict[str, Any]] = None,
  580. lock_held: bool = False) -> None:
  581. """Persist ONE routing entry via UPSERT — the per-turn fast path (a full rewrite fsyncs a
  582. multi-MB sessions.json). The key -> session_id mapping never changes here: structural
  583. transitions use the full rewrite (which also refreshes the sessions.json mirror; it may lag
  584. in metadata only). The revision comes from the shared routing generation counter; under
  585. ``_save_lock`` the upsert is skipped if a full snapshot or a newer fast save of this key
  586. already persisted (the reverse case lives in ``_persist_routing_data``). No DB or a failed
  587. upsert falls back to the full rewrite. ``entry_data`` persists a candidate BEFORE it is
  588. published to the live entry (failure-atomic transitions); the fallback carries it too."""
  589. guard = contextlib.nullcontext() if lock_held else self._lock
  590. with guard:
  591. entry = self._entries.get(session_key)
  592. if entry is None:
  593. return
  594. serialized = dict(entry_data) if entry_data is not None else entry.to_dict()
  595. # The O(n) full snapshot is deferred to the fallback branch.
  596. entry_json, revision = json.dumps(serialized), self._next_routing_generation_locked()
  597. saver = self._routing_db_method("save_gateway_routing_entry")
  598. if saver is not None:
  599. try:
  600. with self._lazy("_save_lock", threading.Lock):
  601. if getattr(self, "_persisted_routing_generation", 0) >= revision:
  602. return
  603. fast_persisted = self._lazy("_fast_persisted_entries", dict)
  604. persisted = fast_persisted.get(session_key)
  605. if persisted is not None and persisted[0] >= revision:
  606. return
  607. saver(session_key, entry_json, scope=self._routing_scope())
  608. fast_persisted[session_key] = (revision, entry_json)
  609. return
  610. except Exception as exc:
  611. logger.warning(
  612. "gateway.session: single-entry routing save failed for %r (%s); falling back "
  613. "to full index rewrite", session_key, exc)
  614. if entry_data is not None:
  615. # Full-snapshot fallback carrying the candidate transition.
  616. with guard:
  617. fallback_data = self._entries_as_dicts()
  618. fallback_data[session_key] = dict(entry_data)
  619. self._persist_routing_data(fallback_data, revision)
  620. else:
  621. self._save_entries()