session_lifecycle.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. """SessionStore explicit suspension, crash-recovery markers, pruning and shared clock/id helpers."""
  2. from __future__ import annotations
  3. import logging
  4. import os
  5. import time
  6. import uuid
  7. from datetime import datetime, timedelta, timezone, UTC
  8. from typing import TYPE_CHECKING, Optional
  9. from hermes_state_ids import new_session_id
  10. if TYPE_CHECKING:
  11. from gateway.session import SessionEntry, SessionSource
  12. # Log-record parity with the origin module.
  13. logger = logging.getLogger("gateway.session")
  14. def _now() -> datetime:
  15. """Return the current local time."""
  16. return datetime.now()
  17. def _new_session_id(now: datetime) -> str:
  18. return new_session_id(now, hex_len=8)
  19. def _iso(dt: Optional[datetime]) -> Optional[str]:
  20. return dt.isoformat() if dt else None
  21. def _parse_iso(value) -> Optional[datetime]:
  22. """``datetime.fromisoformat`` that returns None for empty/malformed input."""
  23. if not value:
  24. return None
  25. try:
  26. return datetime.fromisoformat(value)
  27. except (TypeError, ValueError):
  28. return None
  29. # Auto-continue freshness window (1 hour) after the ``resume_pending`` mark; ``gateway/run.py``
  30. # bridges config.yaml ``agent.gateway_auto_continue_freshness`` into the env var at startup.
  31. _AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT = 60 * 60
  32. def auto_continue_freshness_window() -> float:
  33. """Resume-scheduler freshness window; stale automation never discards the transcript."""
  34. raw = os.environ.get("HERMES_AUTO_CONTINUE_FRESHNESS")
  35. try:
  36. return float(raw) if raw else float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT)
  37. except (TypeError, ValueError):
  38. return float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT)
  39. class SessionLifecycleMixin:
  40. """SessionStore explicit boundaries and crash-recovery markers."""
  41. def _is_session_ended_in_db(self, session_id: str, session_key: Optional[str] = None) -> bool:
  42. """True iff state.db says the session is gone: ended (non-null end_reason) or hard-deleted
  43. (no row in a readable owning DB). No DB or a DB error -> False (same failure mode as
  44. ``_prune_stale_sessions_locked``). Lets routing self-heal a session finalized or deleted
  45. while the gateway stays alive. Store resolved from the owning profile.
  46. Used by ``get_or_create_session`` to self-heal at routing time: ``_prune_stale_sessions_locked``
  47. only runs at startup, so a session ended in the DB while the gateway stays alive (any path that
  48. finalizes the row without clearing sessions.json) would otherwise be reused as a live routing key
  49. and silently swallow every subsequent message until the next restart (#54878 — the live-gateway
  50. variant of #52804/FM9). A hard delete is the same shape one step further: the row is GONE, not
  51. merely ended, and reusing the route makes run_agent's INSERT OR IGNORE resurrect the deleted
  52. session with its old id (#42422) — so a missing row is treated exactly like an ended one. DB
  53. errors are non-fatal — never block routing on a failed lookup.
  54. The store is resolved from the row's owning profile rather than the ambient scope: an unscoped
  55. background writer keeps its own copy of the same session, and comparing against that copy reports a
  56. live session as ended (#66887).
  57. Pass *session_key* when the owning key is known but the id may have left the routing index:
  58. after the self-heal re-homes the key, the id has no owner and would resolve to the launch
  59. store, which never holds a routed profile's rows (#118862).
  60. """
  61. with self._lock:
  62. if session_id in self._rows_pending:
  63. # A route transition published this id and has not inserted/reopened its row yet
  64. # (a /new racing a turn that resolves the route as it finishes): healing it here
  65. # would hand the route back to the conversation the user just left.
  66. return False
  67. db = self._db_for_key(session_key) if session_key else self._db_for_session_id(session_id)
  68. if not db or not session_id:
  69. return False
  70. try:
  71. row = db.get_session(session_id)
  72. except Exception:
  73. return False
  74. return row is None or row.get("end_reason") is not None
  75. def _local_receipt_target(self, entry: Optional[SessionEntry]) -> Optional[str]:
  76. """The physical target the authority's receipt names for *entry*'s local route, else None
  77. (not a local route, no receipt, or the receipt belongs to another route).
  78. Such a route moves only through the authority (compression and reset advance the receipt
  79. and the route in one transaction), and its first admitted turn reopens a stamped row
  80. (``reopen_local_session``). A row stamped ended (a viewer's ``ws_orphan_reap``) is
  81. therefore never a stale route: dropping it minted a replacement id the authority refuses
  82. (``admission_conflict``) and stranded the conversation (#140642)."""
  83. origin = getattr(entry, "origin", None)
  84. if origin is None or getattr(origin, "platform", None) is None or origin.platform.value != "local":
  85. return None
  86. db = self._db_for_key(entry.session_key)
  87. if db is None or getattr(db, "_read_ctx", None) is None:
  88. return None
  89. import sqlite3
  90. from hermes_state_local import local_receipt
  91. from hermes_state_runtime import RuntimeStoreError
  92. try:
  93. receipt = local_receipt(db, str(origin.chat_id))
  94. if receipt.get("route") != entry.session_key:
  95. return None
  96. return str(receipt["entry"]["session_id"]) or None
  97. except (RuntimeStoreError, sqlite3.Error, KeyError, TypeError, AttributeError):
  98. return None
  99. def _api_bound(self, entry: Optional[SessionEntry]) -> bool:
  100. """*entry*'s API route is the one its binding receipt names (``bind_api_session``). Like a
  101. local receipt route it moves only through the authority, and its next admitted turn reopens a
  102. stamped row (``reopen_api_session``): replacing it forked the caller's conversation onto a
  103. minted id every later run on it refused (``admission_conflict``)."""
  104. origin = getattr(entry, "origin", None)
  105. if origin is None or getattr(origin, "platform", None) is None or origin.platform.value != "api_server":
  106. return False
  107. db = self._db_for_key(entry.session_key)
  108. if db is None or getattr(db, "_read_ctx", None) is None:
  109. return False
  110. import json
  111. import sqlite3
  112. from hermes_state_local import API_BINDING_PREFIX
  113. try:
  114. with db._read_ctx() as conn:
  115. saved = conn.execute("SELECT value FROM state_meta WHERE key=?",
  116. (API_BINDING_PREFIX + str(origin.chat_id),)).fetchone()
  117. return saved is not None and json.loads(saved[0])["entry"]["session_key"] == entry.session_key
  118. except (sqlite3.Error, ValueError, KeyError, TypeError):
  119. return False
  120. def _route_reset_reason(self, entry: SessionEntry) -> Optional[str]:
  121. """Only explicit suspension replaces a routed conversation; time never does."""
  122. return "suspended" if entry.suspended else None
  123. def _update_entry(self, session_key: str, mutate) -> bool:
  124. """Apply ``mutate(entry)`` under ``_lock`` and full-save; False when the entry is missing
  125. or *mutate* returned False (nothing to persist)."""
  126. with self._lock:
  127. entry = self._entry_locked(session_key)
  128. if entry is None or mutate(entry) is False:
  129. return False
  130. self._save()
  131. return True
  132. def _update_all_entries_locked(self, mutate) -> int:
  133. """Apply ``mutate(entry) -> bool`` to every entry under ``_lock``; save once if any
  134. returned True. Returns the count that did."""
  135. with self._lock:
  136. self._ensure_loaded_locked()
  137. changed = sum(1 for entry in self._entries.values() if mutate(entry))
  138. if changed:
  139. self._save()
  140. return changed
  141. def suspend_session(self, session_key: str) -> bool:
  142. """Mark a session suspended so it auto-resets on next access (/stop). True if it existed.
  143. Used by ``/stop`` to prevent stuck sessions from being resumed after a gateway restart (#7536).
  144. """
  145. return self._update_entry(session_key, lambda e: setattr(e, "suspended", True))
  146. def _set_turn_marker_locked(self, session_key: str, entry: SessionEntry, token, started_at) -> None:
  147. """Persist the active-turn pair BEFORE publishing it in memory, so a failed write can
  148. neither leak an unowned token nor drop a live one. Lock held."""
  149. candidate = entry.to_dict()
  150. candidate["active_turn_token"] = token
  151. candidate["active_turn_started_at"] = _iso(started_at)
  152. touched = _now() if started_at is not None else None
  153. if touched is not None:
  154. # Keeps the legacy 120s startup heuristic working for an older binary during a rolling
  155. # downgrade/upgrade window.
  156. candidate["updated_at"] = touched.isoformat()
  157. self._save_entry(session_key, entry_data=candidate, lock_held=True)
  158. entry.active_turn_token = token
  159. entry.active_turn_started_at = started_at
  160. if touched is not None:
  161. entry.updated_at = touched
  162. def mark_turn_active(self, session_key: str) -> Optional[str]:
  163. """Persist exact ownership of the running agent turn; returns the opaque token for
  164. :meth:`clear_turn_active`. Re-marking replaces the previous token so a stale asynchronous
  165. unwind cannot clear a newer turn."""
  166. token = uuid.uuid4().hex
  167. with self._lock:
  168. entry = self._entry_locked(session_key)
  169. if entry is None:
  170. return None
  171. # Aware UTC, unlike the local wall clock elsewhere: the next process compares it with
  172. # epoch transcript timestamps and may run in another zone (DST, container vs unit TZ).
  173. self._set_turn_marker_locked(session_key, entry, token, datetime.now(UTC))
  174. return token
  175. def clear_turn_active(self, session_key: str, token: str) -> bool:
  176. """Compare-and-swap clear an active-turn marker; ``False`` when the entry disappeared or a
  177. newer turn owns it."""
  178. with self._lock:
  179. entry = self._entry_locked(session_key)
  180. if entry is None or entry.active_turn_token != token:
  181. return False
  182. self._set_turn_marker_locked(session_key, entry, None, None)
  183. return True
  184. def recover_interrupted_turns(self, max_age_seconds: int = 60 * 60) -> int:
  185. """Promote crash-left turn markers into ``resume_pending`` (unclean startup only).
  186. Old/invalid markers are cleared without resuming; suspended sessions are never re-armed.
  187. Returns the number of newly promoted sessions."""
  188. now, epoch_now = _now(), time.time()
  189. promoted = 0
  190. def _promote(entry: SessionEntry) -> bool:
  191. nonlocal promoted
  192. if not entry.active_turn_token:
  193. return False
  194. started_at = entry.active_turn_started_at
  195. # Epoch arithmetic: a pre-upgrade naive marker reads as local time, an aware one exactly.
  196. marker_is_stale = started_at is None or (
  197. max_age_seconds > 0 and epoch_now - started_at.timestamp() > max_age_seconds
  198. )
  199. if not marker_is_stale and not entry.suspended:
  200. if entry.resume_pending:
  201. # A drain-timeout marker is more specific; keep it.
  202. if entry.last_resume_marked_at is None:
  203. entry.last_resume_marked_at = now
  204. else:
  205. entry.resume_pending = True
  206. entry.resume_reason = "restart_interrupted"
  207. entry.last_resume_marked_at = now # freshness starts at discovery
  208. promoted += 1
  209. entry.active_turn_token = None
  210. entry.active_turn_started_at = None
  211. return True
  212. self._update_all_entries_locked(_promote)
  213. return promoted
  214. def discard_active_turn_markers(self) -> int:
  215. """Clear orphan turn markers after a verified clean shutdown."""
  216. def _discard(entry: SessionEntry) -> bool:
  217. if not entry.active_turn_token and entry.active_turn_started_at is None:
  218. return False
  219. entry.active_turn_token = None
  220. entry.active_turn_started_at = None
  221. return True
  222. return self._update_all_entries_locked(_discard)
  223. def mark_resume_pending(self, session_key: str, reason: str = "restart_timeout") -> bool:
  224. """Mark a session resumable after a restart interruption (keeps the session_id/transcript,
  225. unlike ``suspend_session``). True if marked."""
  226. def _apply(entry: SessionEntry):
  227. if entry.suspended: # never override an explicit ``suspended`` (hard forced-wipe)
  228. return False
  229. entry.resume_pending = True
  230. entry.resume_reason = reason
  231. entry.last_resume_marked_at = _now()
  232. return self._update_entry(session_key, _apply)
  233. def clear_resume_pending(self, session_key: str) -> bool:
  234. """Clear the resume-pending flag after a successful resumed turn; True if cleared."""
  235. def _apply(entry: SessionEntry):
  236. if not entry.resume_pending:
  237. return False
  238. entry.resume_pending = False
  239. entry.resume_reason = None
  240. entry.last_resume_marked_at = None
  241. return self._update_entry(session_key, _apply)
  242. def prune_old_entries(self, max_age_days: int) -> int:
  243. """Drop routing entries idle (by ``updated_at``) for more than max_age_days; suspended
  244. entries and entries with active background processes are kept. Only the key -> session_id
  245. mapping is dropped (the transcript stays). ``max_age_days <= 0`` disables. Returns count."""
  246. if max_age_days is None or max_age_days <= 0:
  247. return 0
  248. cutoff = _now() - timedelta(days=max_age_days)
  249. with self._lock:
  250. self._ensure_loaded_locked()
  251. removed_keys = [
  252. key for key, entry in list(self._entries.items())
  253. if not entry.suspended
  254. # The callback is keyed by session_key, NOT session_id.
  255. and not self._has_active_processes_safe(entry.session_key, context="prune")
  256. and entry.updated_at < cutoff
  257. ]
  258. for key in removed_keys:
  259. self._entries.pop(key, None)
  260. if removed_keys:
  261. self._save()
  262. if removed_keys:
  263. logger.info("SessionStore pruned %d entries older than %d days",
  264. len(removed_keys), max_age_days)
  265. return len(removed_keys)