turn_facade_lease.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495
  1. """Durable cross-process session turn lease for ``TurnFacadeMixin.run_conversation``.
  2. One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI
  3. resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or
  4. returns the early result the façade must hand back); ``DurableTurnLease`` owns the periodic
  5. refresher, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing. Both
  6. timers run via the shared scheduler (``agent/periodic_scheduler.py``; timer thread orders,
  7. bodies run on per-handle workers), not per-turn threads.
  8. """
  9. import logging
  10. import os
  11. import threading
  12. import time
  13. from contextlib import nullcontext
  14. from dataclasses import dataclass
  15. from typing import Any, Dict, List, Optional
  16. from hermes_state_pidns import holder_namespace_token
  17. # Same logger name as the origin module so log records / caplog filters are unchanged.
  18. logger = logging.getLogger("run_agent")
  19. # ``tool_reason`` for a lost session turn lease: attributes the stop to the lease, not the user (#112647).
  20. _REASON_LEASE_LOST = "session turn lease lost"
  21. LEASE_TTL_SECONDS = 300.0
  22. LEASE_WAIT_SECONDS = 1800.0
  23. # SessionDB's default write patience: a renewal waits this long for the write lock, never past the authority.
  24. _REFRESH_WRITE_PATIENCE_S = 20.0
  25. # Headroom before expiry for one BEGIN IMMEDIATE blocked in the writer's 1s busy handler, plus jitter.
  26. _REFRESH_EXPIRY_MARGIN_S = 2.0
  27. class DurableTurnLease:
  28. """An admitted session turn lease plus the periodic timers that keep it alive and watch the turn.
  29. ``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every
  30. interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and
  31. written only under ``_lock``.
  32. """
  33. def __init__(self, agent, db, session_id: str, holder: str, *, expires_at: Optional[float] = None) -> None:
  34. self.agent = agent
  35. self.db = db
  36. self.session_id = session_id # id at admission; release always targets this row
  37. self.holder = holder
  38. # Wall clock, same as the row's expires_at, never later than it: the row's committed expiry at
  39. # admission, then a timestamp taken BEFORE each successful renewal (the store stamps expiry
  40. # before its lock wait). Only a successful renewal moves it.
  41. self._authority_deadline = time.time() + LEASE_TTL_SECONDS if expires_at is None else expires_at
  42. self.stop = threading.Event()
  43. self.refresh_interval = float(getattr(agent, "_session_turn_lease_refresh_interval", 60.0))
  44. self._lock = threading.Lock()
  45. self.turn_active = False
  46. self.interrupt_message: Optional[str] = None
  47. self.watchdog = None # TurnLivenessWatchdog when configured
  48. self.timer_handles: list = [] # periodic_scheduler handles, cancelled in join_threads
  49. def _current_session_id(self) -> str:
  50. return getattr(self.agent, "session_id", None) or self.session_id
  51. def build_threads(self) -> None:
  52. """Create (not schedule) the liveness watchdog when configured: lease renewal is NOT
  53. evidence of progress; a silently stalled turn would renew forever."""
  54. try:
  55. from hermes_cli.config import load_config_readonly
  56. liveness_config = load_config_readonly() or {}
  57. except Exception:
  58. liveness_config = {}
  59. from agent import turn_liveness
  60. timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config)
  61. if timeout_s is not None:
  62. self.watchdog = turn_liveness.TurnLivenessWatchdog(
  63. self.agent, session_id=self._current_session_id(), timeout_s=timeout_s,
  64. poll_s=poll_s, stop_event=self.stop,
  65. activity_lock=self.agent._liveness_activity_lock(),
  66. is_turn_active=self.is_turn_active, commit_abort=self.commit_liveness_abort,
  67. deactivate_turn=self.stop_refresher,
  68. )
  69. def start(self) -> None:
  70. with self._lock:
  71. self.turn_active = True
  72. # Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so
  73. # without this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one.
  74. self.agent._touch_activity("starting new turn")
  75. from hermes_cli.observability.shared_metrics_process import arm_turn
  76. arm_turn(self.agent)
  77. from agent.periodic_scheduler import schedule
  78. self.timer_handles.append(schedule(self.refresh_tick, self.refresh_interval))
  79. if self.watchdog is not None:
  80. self.timer_handles.append(self.watchdog.schedule())
  81. def stop_refresher(self) -> None:
  82. """Stop renewal and deactivate the turn. Also the watchdog's deactivate callback: a wedge the
  83. hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry lets
  84. stale-turn cleanup reclaim the row."""
  85. with self._lock:
  86. self.turn_active = False
  87. self.stop.set()
  88. deactivate_after_liveness_abort = stop_refresher
  89. def join_threads(self, timeout: float = 1.0) -> None:
  90. """Cancel both timers; ``wait=timeout`` mirrors the old ``thread.join(timeout)`` so an
  91. in-flight tick finishes before ``clear_interrupt`` runs."""
  92. for handle in self.timer_handles:
  93. handle.cancel(wait=timeout)
  94. # The watchdog holds this lease's bound methods: drop it so the finished turn frees by refcount.
  95. self.timer_handles, self.watchdog = [], None
  96. def release(self) -> None:
  97. """Release the row and drop the agent's holder attrs (only if they still name this lease)."""
  98. agent = self.agent
  99. try:
  100. self.db.release_session_turn_lease(self.session_id, self.holder)
  101. except Exception:
  102. logger.error("Failed to release session turn lease: %s", self.session_id, exc_info=True)
  103. if getattr(agent, "_active_session_turn_lease_holder", None) == self.holder:
  104. agent._active_session_turn_lease_holder = None
  105. agent._active_session_turn_lease_ttl_seconds = None
  106. def is_turn_active(self) -> bool:
  107. with self._lock:
  108. return self.turn_active
  109. def _interrupt_turn(self, message: str) -> None:
  110. """Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means
  111. this process no longer owns the session. Only the watchdog's stalls can be spuriously stale."""
  112. with self._lock:
  113. if self.stop.is_set() or not self.turn_active:
  114. return
  115. self.interrupt_message = message
  116. try:
  117. self.agent.interrupt(message, hard_cancel=True, tool_reason=_REASON_LEASE_LOST)
  118. except Exception:
  119. self.agent._interrupt_requested = True
  120. self.agent._interrupt_message = message
  121. self.agent._tool_interrupt_reason = _REASON_LEASE_LOST
  122. def commit_liveness_abort(self, snapshot, message: str) -> bool:
  123. """Commit point for the watchdog's stall observation.
  124. Revalidates the observed ``(generation, timestamp)`` under the SAME lock ``_touch_activity``
  125. uses, so a turn that resumed while the stall was logged is never hard-cancelled; the
  126. revalidated generation is consumed by ``interrupt(require_generation=...)`` with the first
  127. publication in ONE critical section. If ``interrupt`` raises, the abort declines FAIL-CLOSED.
  128. Returns False when stale or already winding down."""
  129. agent = self.agent
  130. with agent._liveness_activity_lock():
  131. current_generation = getattr(agent, "_turn_liveness_activity_generation", 0)
  132. if (current_generation, getattr(agent, "_last_activity_ts", None)) != (
  133. snapshot.generation, snapshot.activity_ts
  134. ):
  135. return False
  136. with self._lock:
  137. if self.stop.is_set() or not self.turn_active:
  138. return False
  139. try:
  140. published = agent.interrupt(
  141. message, hard_cancel=True, tool_reason="turn liveness watchdog",
  142. require_generation=current_generation,
  143. )
  144. except Exception:
  145. logger.debug("Turn liveness abort interrupt raised; declining the abort", exc_info=True)
  146. published = False
  147. if published is False:
  148. # Claim went stale between revalidation and the hammer: real progress landed.
  149. return False
  150. with self._lock:
  151. self.interrupt_message = message
  152. return True
  153. def clear_interrupt(self) -> None:
  154. """Clear only the interrupt admitted by this lease's refresher/watchdog. Run AFTER join."""
  155. message = self.interrupt_message
  156. if not message:
  157. return
  158. agent = self.agent
  159. from tools.interrupt import set_interrupt as _set_interrupt
  160. with getattr(agent, "_pending_redirect_lock", None) or nullcontext():
  161. if getattr(agent, "_interrupt_message", None) != message:
  162. return
  163. agent._interrupt_requested = False
  164. agent._interrupt_message = None
  165. getattr(agent, "_hard_interrupt_requested", threading.Event()).clear()
  166. agent._interrupt_thread_signal_pending = False
  167. if agent._execution_thread_id is not None:
  168. _set_interrupt(False, agent._execution_thread_id)
  169. def _stop_on_exhausted_authority(self) -> bool:
  170. """State.db stayed locked until no renewal can land before the row expires: stop now."""
  171. logger.warning("Session turn lease refresh stayed locked through its lifetime: %s", self._current_session_id())
  172. self._interrupt_turn("Session turn lease could not be refreshed; stopping to protect the transcript.")
  173. return False
  174. def refresh_tick(self):
  175. """One periodic renewal (every ``refresh_interval`` via the shared scheduler). A real
  176. miss (rowcount 0) or a non-lock error interrupts the turn. A SQLite lock is a missed
  177. tick only while the next attempt still lands before ``_authority_deadline``. Returning
  178. False stops the timer.
  179. Acquisition reclaims a row once ``expires_at <= now``, even if the old process is
  180. alive. Holder-qualified release does not block that path, so a run of lock errors
  181. must not keep this turn active up to and past the deadline. A successful renewal
  182. is the only thing that moves the deadline.
  183. The holder-qualified UPDATE fences a late refresher from a successor lease. The façade's
  184. finally sets ``stop`` before releasing, so a holder-fenced miss observed after stop is not
  185. a loss."""
  186. if self.stop.is_set():
  187. return False
  188. started = time.time()
  189. # The whole renewal must finish before the authority runs out: a renewal still waiting on the
  190. # write lock past expiry keeps this turn running while a successor reclaims the row. The
  191. # margin covers one BEGIN IMMEDIATE that blocks in SQLite's busy handler past the patience.
  192. patience = self._authority_deadline - _REFRESH_EXPIRY_MARGIN_S - started
  193. if patience <= 0:
  194. return self._stop_on_exhausted_authority()
  195. try:
  196. if self.db.refresh_session_turn_lease(
  197. self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS,
  198. patience_s=min(_REFRESH_WRITE_PATIENCE_S, patience),
  199. ):
  200. self._authority_deadline = started + LEASE_TTL_SECONDS
  201. return None
  202. if self.stop.is_set():
  203. return False
  204. logger.error(
  205. "Lost session turn lease while turn is active: %s", self._current_session_id()
  206. )
  207. self._interrupt_turn("Session turn lease lost; stopping to protect the transcript.")
  208. except Exception as exc:
  209. if self.stop.is_set():
  210. return False
  211. from hermes_state_errors import is_sqlite_lock_error
  212. if is_sqlite_lock_error(exc):
  213. # The scheduler will not try again until one interval from now. If that attempt
  214. # could not finish before the row expiry, stop before a successor can claim it.
  215. if time.time() + self.refresh_interval + _REFRESH_EXPIRY_MARGIN_S >= self._authority_deadline:
  216. return self._stop_on_exhausted_authority()
  217. logger.warning(
  218. "Session turn lease refresh hit a SQLite lock; will retry: %s",
  219. self._current_session_id(),
  220. )
  221. return None
  222. logger.warning(
  223. "Failed to refresh session turn lease: %s", self._current_session_id(), exc_info=True,
  224. )
  225. self._interrupt_turn(
  226. "Session turn lease could not be refreshed; stopping to protect the transcript."
  227. )
  228. return False
  229. @dataclass
  230. class TurnLeaseAdmission:
  231. """Outcome of ``admit_durable_turn_lease``: exactly one of ``lease`` / ``early_result`` may be set."""
  232. lease: Optional[DurableTurnLease] = None
  233. early_result: Optional[dict[str, Any]] = None
  234. conversation_history: Optional[list[dict[str, Any]]] = None
  235. def _committed_lease_expiry(db, session_id: str, holder: str, floor: float) -> float:
  236. """The admitted row's committed ``expires_at``; when unreadable, ``floor + TTL`` (taken before
  237. acquisition, so never later than the real expiry). Overstating it lets a successor reclaim the
  238. row while this turn still believes it holds the lease."""
  239. reader = getattr(db, "session_turn_lease_expires_at", None)
  240. if callable(reader):
  241. try:
  242. committed = reader(session_id, holder)
  243. if isinstance(committed, (int, float)):
  244. return float(committed)
  245. except Exception:
  246. logger.debug("Could not read the admitted turn lease expiry; using the pre-acquire floor", exc_info=True)
  247. return floor + LEASE_TTL_SECONDS
  248. def _durable_session_exists(db, session_id: str) -> Optional[bool]:
  249. """True / False when the row read answered; None when it failed and the state is unknown."""
  250. try:
  251. return db.get_session(session_id) is not None
  252. except Exception:
  253. # A locked / non-WAL read proves neither presence nor absence: get_session returns None —
  254. # it does not raise — when the row is missing. See #84234.
  255. logger.warning(
  256. "Could not check durable session after turn lease admission; row state is unknown",
  257. exc_info=True,
  258. )
  259. return None
  260. def admit_durable_turn_lease(
  261. agent, *, session_id: str, relay_turn_id: str, task_context: dict[str, Any],
  262. conversation_history: Optional[list[dict[str, Any]]],
  263. ) -> TurnLeaseAdmission:
  264. """Acquire the session turn lease (the row need not exist yet); build (not start) its threads.
  265. Mutates ``task_context["session_id"]`` and ``agent.session_id`` when the wait forced a resume-id
  266. reload. Returns an ``early_result`` (interrupted / timed out) instead of a lease when admission
  267. fails; the caller returns it verbatim."""
  268. db = getattr(agent, "_session_db", None)
  269. admission = TurnLeaseAdmission(conversation_history=conversation_history)
  270. if db is None or not session_id:
  271. return admission
  272. # Check the concrete type: MagicMock-style shims accept any attribute without the protocol.
  273. if (
  274. getattr(agent, "_persist_disabled", False)
  275. or not callable(getattr(type(db), "acquire_session_turn_lease", None))
  276. ):
  277. return admission
  278. # A session id without a row still takes the lease: client-addressed ids (API server
  279. # X-Hermes-Session-Id, /v1/runs session_id, fingerprint-derived chat ids) are not
  280. # process-unique, and the first turn creates the row mid-turn, so a second writer would
  281. # otherwise find the row, take an unheld lease and interleave its turn into this one.
  282. # pidns stamp: see hermes_state_pidns.
  283. holder = (
  284. f"pid={os.getpid()}{holder_namespace_token()}:turn={relay_turn_id}"
  285. f":platform={task_context['platform'] or 'unknown'}"
  286. )
  287. reload_needed = announced = False
  288. def _on_contended() -> None:
  289. # A busy state.db, not a known holder: say nothing, but still reload after admission,
  290. # since the busy writer may have been the previous holder's final flush (one that fits
  291. # inside the write patience never gets here, before or after this signal existed).
  292. nonlocal reload_needed
  293. reload_needed = True
  294. def _on_wait(elapsed: float) -> None:
  295. nonlocal reload_needed, announced
  296. reload_needed = announced = True
  297. agent._emit_status(
  298. "⏳ Another Hermes process is using this session; "
  299. "waiting for it to finish before starting your turn..."
  300. if elapsed < 1.0 else
  301. f"⏳ Still waiting for the other Hermes process on this session ({int(elapsed)}s)..."
  302. )
  303. authority_floor = time.time() # every acquisition attempt stamps its expiry after this instant
  304. if not db.acquire_session_turn_lease(
  305. session_id, holder, ttl_seconds=LEASE_TTL_SECONDS, wait_seconds=LEASE_WAIT_SECONDS,
  306. on_wait=_on_wait, on_contended=_on_contended,
  307. should_abort=lambda: getattr(agent, "_interrupt_requested", False),
  308. ):
  309. admission.early_result = _lease_not_acquired_result(agent, session_id, conversation_history,
  310. waited=announced)
  311. return admission
  312. # Assign only after admission so the finally cannot release a holder that never owned the
  313. # row; persist paths read the agent attr so a late flush is fenced in the same transaction.
  314. lease = DurableTurnLease(
  315. agent, db, session_id, holder, expires_at=_committed_lease_expiry(db, session_id, holder, authority_floor),
  316. )
  317. agent._active_session_turn_lease_holder = holder
  318. agent._active_session_turn_lease_ttl_seconds = LEASE_TTL_SECONDS
  319. try:
  320. # Read the row only now: the previous holder may have created or deleted it while this
  321. # turn waited, so an answer from before admission can be stale either way.
  322. durable = _durable_session_exists(db, session_id)
  323. if durable and getattr(agent, "_session_db_reserved", False) is not True:
  324. # Row proven to exist — suppress the redundant create attempt. A missing row leaves
  325. # the flag alone: the flush heals a row deleted under a live agent (#123583). So does
  326. # an unknown one: the create is an upsert that never overwrites an existing row.
  327. # A child reserved only its identity for transcript readers. Its first turn must
  328. # still persist constructor config (including lineage) and the completed prompt.
  329. agent._session_db_created = True
  330. # Reload only a transcript that may exist: callers may seed a fresh id in memory before its
  331. # row is written, and reloading an absent row would erase that seed. An unknown row still
  332. # reads the transcript after a wait and adopts it only if it returns rows; that read
  333. # raising ends the turn.
  334. if reload_needed and durable is not False:
  335. if announced:
  336. agent._emit_status("Session is free; loading the latest transcript...")
  337. # The holder may have compressed/rotated the session while we waited: reload only
  338. # AFTER admission; an immediate acquisition skips this (needless prompt-cache miss).
  339. latest_session_id = db.resolve_resume_session_id(session_id)
  340. if latest_session_id:
  341. agent.session_id = latest_session_id
  342. task_context["session_id"] = latest_session_id
  343. reloaded = db.get_messages_as_conversation(
  344. agent.session_id, repair_alternation=True, include_row_ids=True
  345. )
  346. # Decide on the stored rows alone: an unknown row that reloads nothing keeps the
  347. # caller's history, which already holds any carried input below.
  348. if durable or reloaded:
  349. # A follow-up that aborted an earlier wait carries that turn's never-persisted
  350. # input only in memory (see carry_unadmitted_user_message); the reload drops it.
  351. from agent.session_persistence import _PERSIST_AFTER_ADMISSION_INTERRUPT
  352. reloaded.extend(
  353. m for m in (conversation_history or [])
  354. if isinstance(m, dict) and m.get(_PERSIST_AFTER_ADMISSION_INTERRUPT)
  355. and "_row_id" not in m
  356. )
  357. admission.conversation_history = reloaded
  358. lease.build_threads()
  359. except BaseException:
  360. # The façade never saw this lease; release here so an admitted row is not leaked.
  361. lease.release()
  362. raise
  363. admission.lease = lease
  364. return admission
  365. def carry_unadmitted_user_message(
  366. early_result: dict[str, Any], user_message: Any, persist_user_message: Any, *,
  367. timestamp: Optional[float], display_kind: Optional[str], display_metadata: Optional[dict[str, Any]],
  368. platform_id: Optional[str],
  369. ) -> None:
  370. """A follow-up that interrupted the lease wait must not consume the accepted input: append it to
  371. the early result's history so the follow-up turn sees it and persists it (the flush honours
  372. ``_PERSIST_AFTER_ADMISSION_INTERRUPT`` because this turn never owned the lease). A hard stop
  373. (``/stop``) cancels the input instead."""
  374. hard_interrupted = early_result.pop("_hard_interrupted", False)
  375. if hard_interrupted or not early_result.get("interrupted") or user_message in (None, ""):
  376. return
  377. from agent.message_metadata import append_message
  378. from agent.session_persistence import _PERSIST_AFTER_ADMISSION_INTERRUPT
  379. durable_content = user_message
  380. if persist_user_message is not None and (
  381. not isinstance(user_message, list) or isinstance(persist_user_message, list)
  382. ):
  383. durable_content = persist_user_message
  384. deferred_user: dict[str, Any] = {
  385. "role": "user", "content": durable_content, _PERSIST_AFTER_ADMISSION_INTERRUPT: True,
  386. }
  387. if isinstance(user_message, str) and user_message != durable_content:
  388. deferred_user["api_content"] = user_message
  389. if display_kind:
  390. deferred_user["display_kind"] = display_kind
  391. if display_metadata:
  392. deferred_user["display_metadata"] = display_metadata
  393. if platform_id is not None:
  394. deferred_user["platform_message_id"] = platform_id
  395. append_message(early_result["messages"], deferred_user, timestamp=timestamp)
  396. def _lease_not_acquired_result(agent, session_id: str, conversation_history, *, waited: bool) -> dict[str, Any]:
  397. """``waited``: another holder was seen (the user was told so), not merely an interrupt that
  398. landed before the first acquisition attempt, which no other process was involved in."""
  399. base = {"messages": list(conversation_history or []), "api_calls": 0, "completed": False}
  400. if getattr(agent, "_interrupt_requested", False):
  401. logger.info("session turn lease wait aborted by interrupt: %s", session_id)
  402. hard_event = getattr(agent, "_hard_interrupt_requested", None)
  403. hard_interrupted = bool(
  404. callable(getattr(hard_event, "is_set", None)) and hard_event.is_set()
  405. )
  406. result = {
  407. "final_response": (
  408. "Stopped waiting for another Hermes process on this session. "
  409. "Your message was not processed."
  410. if waited else
  411. "Stopped before this turn started. Your message was not processed."
  412. ),
  413. **base,
  414. "interrupted": True,
  415. }
  416. if hard_interrupted:
  417. result["_hard_interrupted"] = True
  418. if getattr(agent, "_interrupt_message", None):
  419. result["interrupt_message"] = agent._interrupt_message
  420. # The finalizer never runs on this early return; clear so a cached agent doesn't
  421. # fail-close the next turn.
  422. try:
  423. agent.clear_interrupt()
  424. except Exception:
  425. agent._interrupt_requested = False
  426. agent._interrupt_message = None
  427. return result
  428. # Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare TimeoutError.
  429. timeout_msg = (
  430. "⏳ Another Hermes process kept this session busy too long. Your message was not "
  431. "processed - wait for the other process to finish, then send it again."
  432. )
  433. logger.error("session turn lease wait timed out for %s", session_id)
  434. try:
  435. agent._emit_warning(timeout_msg)
  436. except Exception:
  437. logger.debug("Failed to emit session turn lease timeout warning", exc_info=True)
  438. # Stamped so Desktop/TUI show "session busy, send again" instead of code="unknown".
  439. return {
  440. "final_response": timeout_msg,
  441. **base,
  442. "failed": True,
  443. "error": f"session_turn_lease_timeout:{session_id}",
  444. "failure_reason": "session_busy",
  445. "failure_retryable": True,
  446. }