session_recovery.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. """SessionStore durable-row recovery: session-key generation, legacy Slack key migration, rebuilding
  2. a routing entry from state.db, and the SQLite side of routing transitions (promote/reopen/create/
  3. peer). Mixin split out of ``gateway/session.py``; bound onto ``SessionStore`` via the MRO."""
  4. from __future__ import annotations
  5. import logging
  6. import json
  7. import math
  8. import threading
  9. from dataclasses import replace
  10. from datetime import datetime
  11. from gateway.config import Platform
  12. from typing import TYPE_CHECKING, Any, Dict, Optional
  13. if TYPE_CHECKING:
  14. from gateway.session import SessionEntry, SessionSource
  15. # Log-record parity with the origin module.
  16. logger = logging.getLogger("gateway.session")
  17. def _origin_json(source) -> Optional[str]:
  18. """``source.to_dict()`` as JSON, or None when absent/unserializable."""
  19. if source is None:
  20. return None
  21. try:
  22. return json.dumps(source.to_dict())
  23. except Exception:
  24. return None
  25. class SessionRecoveryMixin:
  26. """SessionStore durable-row recovery and the SQLite side of routing transitions."""
  27. def _resolve_profile_for_key(self, source: Optional[SessionSource] = None) -> Optional[str]:
  28. """Profile namespace for session keys: None when multiplexing is off (legacy
  29. ``agent:main``), else the pinned identity's runtime profile, ``source.profile`` or the
  30. active profile."""
  31. if not getattr(self.config, "multiplex_profiles", False):
  32. return None
  33. from gateway.session_identity import identity_of
  34. identity = identity_of(source)
  35. if identity is not None:
  36. return identity.session_key_profile
  37. if source is not None and source.profile:
  38. return source.profile
  39. try:
  40. from hermes_cli.profiles import get_active_profile_name
  41. return get_active_profile_name() or "default"
  42. except Exception:
  43. return None
  44. @staticmethod
  45. def _profile_from_session_key(session_key: Optional[str]) -> Optional[str]:
  46. """Extract the profile namespace encoded in a gateway session key."""
  47. from hermes_state_keys import profile_from_session_key
  48. return profile_from_session_key(session_key)
  49. @staticmethod
  50. def _active_profile_name() -> str:
  51. try:
  52. from hermes_cli.profiles import get_active_profile_name
  53. return get_active_profile_name() or "default"
  54. except Exception:
  55. return "default"
  56. def _recovered_row_allowed_for_active_profile(
  57. self, *, requested_session_key: str, recovered: dict[str, Any]
  58. ) -> bool:
  59. """Prevent a gateway from reviving another profile's row. Single-profile: the row's
  60. namespace must match the ACTIVE profile. Multiplexed: it must match the requested key's
  61. namespace (the active profile is meaningless there). Keyless rows stay adoptable.
  62. Multiplexed: several profiles serve traffic at once, so the active profile is meaningless — the
  63. requested key carries the profile the turn was routed to, and the recovered row must sit in the same
  64. ``agent:<ns>:`` namespace (#74285). Rows with no key namespace stay adoptable in both modes
  65. (legacy/keyless data owned by this store).
  66. """
  67. recovered_key = str(recovered.get("session_key") or "")
  68. if not recovered_key or recovered_key == requested_session_key:
  69. return True
  70. recovered_profile = self._profile_from_session_key(recovered_key)
  71. if recovered_profile is None:
  72. return True
  73. if getattr(self.config, "multiplex_profiles", False):
  74. requested_profile = self._profile_from_session_key(requested_session_key)
  75. return requested_profile is None or recovered_profile == requested_profile
  76. return recovered_profile == self._active_profile_name()
  77. def _generate_session_key(self, source: SessionSource, key_source: Optional[SessionSource] = None) -> str:
  78. """Session key for *source* (profile from *source*; key from *key_source* if given)."""
  79. from gateway.session import build_session_key
  80. return build_session_key(
  81. key_source if key_source is not None else source,
  82. group_sessions_per_user=getattr(self.config, "group_sessions_per_user", True),
  83. thread_sessions_per_user=getattr(self.config, "thread_sessions_per_user", False),
  84. profile=self._resolve_profile_for_key(source))
  85. def _legacy_slack_session_key(self, source: SessionSource) -> Optional[str]:
  86. """Pre-workspace Slack key for an explicitly scoped source. Deliberately Slack-only: an
  87. unscoped Slack session may be claimed by only one workspace (old key cannot tell teams)."""
  88. if source.platform != Platform.SLACK or not source.scope_id:
  89. return None
  90. return self._generate_session_key(source, replace(source, scope_id=None, guild_id=None))
  91. def _claim_legacy_slack_key(self, legacy_key: Optional[str]) -> bool:
  92. """Atomically reserve one ambiguous legacy Slack key for migration."""
  93. if not legacy_key:
  94. return False
  95. with self._lazy("_legacy_slack_claim_lock", threading.Lock):
  96. claimed = self._lazy("_claimed_legacy_slack_keys", set)
  97. if legacy_key in claimed:
  98. return False
  99. claimed.add(legacy_key)
  100. return True
  101. @staticmethod
  102. def _recovered_row_matches_source_scope(
  103. recovered: dict[str, Any], source: SessionSource
  104. ) -> bool:
  105. """Reject recovered rows whose origin belongs to another workspace: a workspace-scoped Slack
  106. lookup adopts a row only if its origin_json names the same scope_id; rows without a
  107. parseable origin are rejected (an unattributable transcript is exactly the ambiguity)."""
  108. if source.platform != Platform.SLACK or source.chat_type == "dm" or not source.scope_id:
  109. return True
  110. try:
  111. origin = json.loads(recovered.get("origin_json") or "")
  112. except (TypeError, ValueError):
  113. return False
  114. if not isinstance(origin, dict):
  115. return False
  116. return origin.get("scope_id", origin.get("guild_id")) == source.scope_id
  117. def _create_entry_from_recovered_row(
  118. self, *, row: dict[str, Any], session_key: str, source: SessionSource, now: datetime,
  119. ) -> SessionEntry:
  120. from gateway.session import SessionEntry
  121. def _ts(value, default: datetime) -> datetime:
  122. try:
  123. return datetime.fromtimestamp(float(value))
  124. except (TypeError, ValueError, OSError):
  125. return default
  126. # An invalid durable timestamp must look old, never freshly active.
  127. created_at = _ts(row.get("started_at"), datetime.fromtimestamp(0))
  128. # The finder already returns durable recency; no extra round-trip.
  129. last_activity = row.get("last_activity_at")
  130. updated_at = _ts(last_activity, created_at) if last_activity is not None else created_at
  131. had_activity = row.get("_has_messages")
  132. if had_activity is None:
  133. had_activity = bool(row.get("message_count") or 0) or last_activity is not None
  134. from gateway.session_identity import transport_profile_of
  135. return SessionEntry(
  136. session_key=session_key, session_id=str(row["id"]), created_at=created_at,
  137. updated_at=updated_at, origin=source, display_name=source.chat_name,
  138. platform=source.platform, chat_type=source.chat_type,
  139. reset_had_activity=bool(had_activity), transport_profile=transport_profile_of(source))
  140. def _find_gateway_session_row(
  141. self, *, session_key: str, source: SessionSource, allow_peer_fallback: bool,
  142. raise_on_lookup_error: bool = False) -> Optional[dict[str, Any]]:
  143. """Query one durable gateway session row. Scoped Slack lookups disable SessionDB's
  144. platform/chat/user fallback: that tuple has no workspace id and could revive another team's
  145. session; the caller performs one explicit exact lookup of the old unscoped key instead."""
  146. return self._peer_row(
  147. self._db_for_key(session_key), source=source.platform.value, session_key=session_key,
  148. user_id=source.user_id,
  149. chat_id=source.chat_id if allow_peer_fallback else None,
  150. chat_type=source.chat_type if allow_peer_fallback else None,
  151. thread_id=source.thread_id, raise_on_lookup_error=raise_on_lookup_error)
  152. @staticmethod
  153. def _peer_row(db, *, source: str, session_key: str, raise_on_lookup_error: bool = False,
  154. **peer: Any) -> Optional[dict[str, Any]]:
  155. """``db.find_latest_gateway_session_for_peer`` guarded for a missing store, a SessionDB
  156. without the finder, and a failing lookup (debug-logged -> None unless *raise_on_lookup_error*).
  157. Extra keyword arguments (user_id/chat_id/chat_type/thread_id) pass through to the finder."""
  158. finder = getattr(db, "find_latest_gateway_session_for_peer", None) if db else None
  159. if not callable(finder):
  160. return None
  161. try:
  162. return finder(source=source, session_key=session_key, **peer)
  163. except Exception as exc:
  164. logger.debug("Gateway session DB recovery failed for %s: %s", session_key, exc)
  165. if raise_on_lookup_error:
  166. raise
  167. return None
  168. def resolve_session_id_for_key(
  169. self, session_key: str, *, not_after: Optional[float] = None,
  170. ) -> Optional[tuple[str, Any]]:
  171. """Resolve a gateway session key to ``(session_id, db)`` for shutdown-flush recovery.
  172. The routing map (``peek_session_id``) is authoritative: it names the session the message
  173. was actually routed to at shutdown. When ``sessions.json`` was pruned, fall back to the
  174. durable row under the exact key (the peer finder keeps the reset fence: rows ended only by
  175. recoverable reasons match; explicit boundaries do not). A row started after the flush
  176. (``started_at > not_after``) cannot be the origin and is never adopted. Never mints a
  177. session; None means the caller must preserve the flush file. ``db`` is the store owning
  178. the key, so the append lands in the right profile partition. When that store cannot be
  179. resolved (``_db_for_key`` fails closed for a profile without a reachable home) the answer
  180. is None even if the routing map knows the id: appending to the ambient root store would
  181. split one session identity across two physical stores (#66887/#102157).
  182. """
  183. if not session_key:
  184. return None
  185. db = self._db_for_key(session_key)
  186. if db is None:
  187. return None
  188. session_id = self.peek_session_id(session_key)
  189. if session_id:
  190. return session_id, db
  191. parts = str(session_key).split(":")
  192. platform = parts[2] if len(parts) >= 3 and parts[0] == "agent" else None
  193. if not platform:
  194. return None
  195. # No scope/profile fences here, unlike _query_recoverable_row: with no chat tuple the finder
  196. # runs only the `s.session_key = ?` branch in the store _db_for_key picked for this key, so a
  197. # hit carries this very key — same profile namespace and (for scoped Slack) same scope_id slot.
  198. row = self._peer_row(db, source=platform, session_key=session_key)
  199. if not isinstance(row, dict) or not row.get("id"):
  200. return None
  201. started_at = row.get("started_at")
  202. # ``ts`` is ``int(time.time())`` while ``started_at`` is a REAL, so compare whole seconds:
  203. # a row minted in the same second as the flush is still a valid origin.
  204. if not_after is not None and started_at is not None and math.floor(float(started_at)) > int(not_after):
  205. return None
  206. return str(row["id"]), db
  207. def _recover_session_from_db(
  208. self, *, session_key: str, source: SessionSource, now: datetime,
  209. raise_on_lookup_error: bool = False) -> Optional[SessionEntry]:
  210. """Rebuild a missing session-key mapping from a recoverable durable row."""
  211. entry, migrated_legacy = self._query_recoverable_row(
  212. # The legacy (pre-workspace) Slack key fallback happens INSIDE _query_recoverable_session
  213. # (#20583/#66398 design): it performs the exact-key legacy lookup, claims the key once per
  214. # process, and rewrites the peer row to the scoped key on success.
  215. session_key=session_key, source=source, now=now,
  216. raise_on_lookup_error=raise_on_lookup_error)
  217. if entry is None:
  218. return None
  219. self._reopen_session_row(session_key, entry.session_id)
  220. if migrated_legacy:
  221. self._record_gateway_session_peer(
  222. entry.session_id, session_key, source, display_name=entry.display_name)
  223. return entry
  224. def _query_recoverable_session(self, *, session_key, source, now):
  225. """DB-only half of _recover_session_from_db (no lock needed): a SessionEntry or None; the
  226. caller assigns _entries[key] under lock. The row is NOT reopened here: the caller evaluates
  227. reset policy first (an agent_close/ws_orphan row may need promotion to a real reset)."""
  228. entry, migrated_legacy = self._query_recoverable_row(
  229. session_key=session_key, source=source, now=now)
  230. if entry is not None and migrated_legacy:
  231. self._record_gateway_session_peer(
  232. entry.session_id, session_key, source, display_name=entry.display_name)
  233. return entry
  234. def _query_recoverable_row(
  235. self, *, session_key, source, now, raise_on_lookup_error=False,
  236. ) -> tuple[Optional[SessionEntry], bool]:
  237. """Find and gate a recoverable row -> (entry or None, migrated_legacy). The legacy
  238. (pre-workspace) Slack key fallback lives here: exact-key lookup, claimed once per process;
  239. ``migrated_legacy`` tells the caller to rewrite the peer row to the scoped key."""
  240. legacy_key = self._legacy_slack_session_key(source)
  241. recovered = self._find_gateway_session_row(
  242. session_key=session_key, source=source, allow_peer_fallback=legacy_key is None,
  243. raise_on_lookup_error=raise_on_lookup_error)
  244. migrated_legacy = False
  245. if not recovered and legacy_key and self._claim_legacy_slack_key(legacy_key):
  246. recovered = self._find_gateway_session_row(
  247. session_key=legacy_key, source=source, allow_peer_fallback=False,
  248. raise_on_lookup_error=raise_on_lookup_error)
  249. migrated_legacy = bool(recovered)
  250. if not isinstance(recovered, dict):
  251. return None, False
  252. if not self._recovered_row_matches_source_scope(recovered, source):
  253. return None, False
  254. if not self._recovered_row_allowed_for_active_profile(
  255. requested_session_key=session_key, recovered=recovered):
  256. logger.warning(
  257. "Gateway session DB recovery ignored %s for %s because the row belongs to a "
  258. "different profile", recovered.get("session_key"), session_key)
  259. return None, False
  260. entry = self._create_entry_from_recovered_row(
  261. row=recovered, session_key=session_key, source=source, now=now)
  262. return entry, migrated_legacy
  263. def _promote_session_reset(self, session_key: str, session_id: str, reason: str, *, log) -> None:
  264. """End *session_id* with *reason* via ``promote_to_session_reset`` (``end_session`` on old
  265. SessionDBs). Promote, not plain end: a row already ended with a recoverable accidental
  266. reason (agent_close / ws_orphan_reap) must be upgraded to the explicit boundary, or
  267. stale-route recovery resurrects it over the reset. ``log(exc)`` reports failures."""
  268. try:
  269. db = self._db_for_key(session_key)
  270. promote = getattr(db, "promote_to_session_reset", None)
  271. if callable(promote):
  272. promote(session_id, reason)
  273. else:
  274. db.end_session(session_id, reason)
  275. from gateway.session_bot import release_superseded_bot_chat
  276. release_superseded_bot_chat(db, self, session_key, session_id)
  277. # Stop the departed conversation's schedule in its owning profile, even when
  278. # the in-memory watch still holds a pre-reset session id.
  279. heartbeat_key = f"heartbeat:{session_id}"
  280. if db.get_meta(heartbeat_key):
  281. db.set_meta(heartbeat_key, "")
  282. except Exception as exc:
  283. log(exc)
  284. def _reopen_session_row(self, session_key: str, session_id: str, *, log_prefix: str = "") -> None:
  285. """Best-effort ``reopen_session``; failures are debug-logged only."""
  286. try:
  287. self._db_for_key(session_key).reopen_session(session_id)
  288. except Exception as exc:
  289. if log_prefix:
  290. logger.debug("%s: %s", log_prefix, exc)
  291. else:
  292. logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc)
  293. def _record_gateway_session_peer(
  294. self, session_id: str, session_key: str, source: Optional[SessionSource],
  295. display_name: Optional[str] = None, include_compression_ancestors: bool = False,
  296. transport_profile: Optional[str] = None) -> None:
  297. """Persist the routing peer for an existing gateway session row. ``transport_profile`` is the
  298. entry's persisted receiving-bot profile; when the caller has no entry it is read off the
  299. source's pinned identity (None = unknown, the column keeps whatever an earlier writer set)."""
  300. db = self._db_for_key(session_key)
  301. if not db or not source:
  302. return
  303. recorder = getattr(db, "record_gateway_session_peer", None)
  304. if not callable(recorder):
  305. return
  306. from gateway.session_api import api_storage_source
  307. from gateway.session_a2a import storage_source
  308. from gateway.session_identity import transport_profile_of
  309. peer = dict(
  310. source=storage_source(db, source, session_id, api_storage_source(db, session_id, source.platform.value)), user_id=source.user_id, session_key=session_key,
  311. chat_id=source.chat_id, chat_type=source.chat_type, thread_id=source.thread_id)
  312. try:
  313. recorder(
  314. session_id, **peer, display_name=display_name or source.chat_name,
  315. origin_json=_origin_json(source),
  316. include_compression_ancestors=include_compression_ancestors,
  317. transport_profile=transport_profile or transport_profile_of(source))
  318. except TypeError:
  319. try: # older SessionDB without display_name/origin_json kwargs
  320. recorder(session_id, **peer)
  321. except Exception as exc:
  322. logger.debug("Gateway session peer record failed for %s: %s", session_key, exc)
  323. except Exception as exc:
  324. logger.debug("Gateway session peer record failed for %s: %s", session_key, exc)
  325. def _adopt_legacy_slack_entry(self, source: SessionSource, session_key: str) -> None:
  326. """One-time migration of pre-workspace-scope Slack keys: MOVE (not copy) the legacy entry so
  327. a second workspace with identical Slack ids cannot attach to the same transcript. Adopt when
  328. the legacy origin names the same workspace; a scope-less DM is claimed once by the first
  329. workspace; a scope-less channel/group is refused (channel ids collide across workspaces)."""
  330. legacy_key = self._legacy_slack_session_key(source)
  331. if not legacy_key:
  332. return
  333. migrated: Optional[SessionEntry] = None
  334. with self._lock:
  335. self._ensure_loaded_locked()
  336. legacy_entry = self._entries.get(legacy_key)
  337. if session_key not in self._entries and legacy_entry is not None:
  338. origin_scope = getattr(legacy_entry.origin, "scope_id", None)
  339. if origin_scope is not None:
  340. adopt = origin_scope == source.scope_id
  341. else:
  342. adopt = source.chat_type == "dm"
  343. if adopt and self._claim_legacy_slack_key(legacy_key):
  344. migrated = self._entries.pop(legacy_key)
  345. migrated.session_key = session_key
  346. migrated.origin = source
  347. migrated.platform = source.platform
  348. migrated.chat_type = source.chat_type
  349. self._entries[session_key] = migrated
  350. if migrated is not None:
  351. self._save_entries()
  352. self._record_gateway_session_peer(
  353. migrated.session_id, session_key, source, display_name=migrated.display_name)
  354. def _finish_route_transition(
  355. self, session_key: str, *, end_session_id: Optional[str], end_reason: str,
  356. create_kwargs: Optional[dict[str, Any]], origin: Optional[SessionSource],
  357. display_name: Optional[str], during: str = "") -> None:
  358. """SQLite side of a routing transition, outside ``_lock``: promote the predecessor row to an
  359. explicit reset boundary (with the specific reason so state.db is auditable, e.g.
  360. ``suspended`` vs plain ``session_reset``), then INSERT the new row + routing
  361. peer. Both best-effort: failures are warned and self-healed by the next peer refresh."""
  362. if end_session_id:
  363. from hermes_cli.observability.relay_shared_metrics import close_session_run
  364. close_session_run(end_session_id)
  365. if self._db_for_key(session_key) and end_session_id:
  366. self._promote_session_reset(
  367. session_key, end_session_id, end_reason,
  368. log=lambda e: logger.warning(
  369. "Failed to end predecessor session row %s for %s%s: %s — the old row remains "
  370. "open and may win restart recovery until the next successful peer refresh",
  371. end_session_id, session_key, during, e),
  372. )
  373. if self._db_for_key(session_key) and create_kwargs:
  374. self._create_session_row(
  375. session_key, create_kwargs, origin, display_name,
  376. log=lambda e: logger.warning(
  377. "Failed to create session row %s for %s%s: %s — deferring to the "
  378. "self-healing peer refresh on the next turn",
  379. create_kwargs.get("session_id"), session_key, during, e),
  380. )
  381. def _announce_route(self, session_key: str, entry) -> None:
  382. """*session_key* now names *entry*'s conversation: tell the owner (``route_named``), so a
  383. live conversation still on that route leaves it before anything else can run there."""
  384. listener = getattr(self, "route_named", None)
  385. if listener is not None and entry is not None:
  386. listener(session_key, entry.session_id, entry.origin)
  387. def _row_settled(self, session_id: str) -> None:
  388. """The route transition that published *session_id* returned (its state.db half done or
  389. failed): from here a missing or ended row is real again (see ``_is_session_ended_in_db``)."""
  390. with self._lock:
  391. self._rows_pending.discard(session_id)
  392. @staticmethod
  393. def _session_create_kwargs(
  394. *, session_id, session_key, origin, source_value, display_name, parent_session_id,
  395. ) -> dict[str, Any]:
  396. """kwargs for ``SessionDB.create_session``. Identity (origin_json) and lineage
  397. (parent/_reset_from) land atomically in the INSERT so a crash right after cannot strand the
  398. row unroutable."""
  399. from gateway.session_identity import transport_profile_of
  400. return {
  401. "session_id": session_id,
  402. "source": source_value,
  403. "user_id": origin.user_id if origin else None,
  404. "session_key": session_key,
  405. "chat_id": origin.chat_id if origin else None,
  406. "chat_type": origin.chat_type if origin else None,
  407. "thread_id": origin.thread_id if origin else None,
  408. "profile_name": origin.profile if origin else None,
  409. "transport_profile": transport_profile_of(origin),
  410. "origin_json": _origin_json(origin),
  411. "display_name": display_name,
  412. "parent_session_id": parent_session_id,
  413. "model_config": {"_reset_from": parent_session_id} if parent_session_id else None,
  414. }
  415. def _create_session_row(self, session_key, db_create_kwargs, origin, display_name, *, log) -> None:
  416. """INSERT a session row and record its routing peer; ``log(exc)`` on failure. A failed
  417. create is a routing hazard (visible warning), but the row is self-healed with full identity
  418. by the next per-turn peer refresh."""
  419. try:
  420. self._db_for_key(session_key).create_session(**db_create_kwargs)
  421. self._record_gateway_session_peer(
  422. db_create_kwargs["session_id"], session_key, origin, display_name=display_name)
  423. except Exception as e:
  424. log(e)