turn_liveness.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. """Turn liveness watchdog: force-abort turns that stall silently.
  2. A turn can wedge mid-flight with no error and its durable lease still renewing, so nothing
  3. ever frees the session. This module owns config resolution (``agent.turn_liveness``,
  4. validated — a typo, NaN or Inf warns and falls back), the sampling state machine, and the
  5. watcher thread. Race safety: the abort decision is bound to the observed
  6. ``(generation, timestamp)``; the commit callback revalidates it under the same lock
  7. ``_touch_activity`` stamps with, so a turn that resumed is never hard-cancelled.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. import math
  12. import threading
  13. import time
  14. from typing import Any, Callable, Dict, NamedTuple, Optional, Tuple
  15. from agent.session_activity import AwakeIdleMeter
  16. logger = logging.getLogger(__name__)
  17. DEFAULT_TURN_LIVENESS_TIMEOUT_S = 600.0
  18. DEFAULT_TURN_LIVENESS_POLL_S = 15.0
  19. MIN_TURN_LIVENESS_POLL_S = 0.01
  20. _CONFIG_TIMEOUT_KEY = "agent.turn_liveness.timeout_s"
  21. _CONFIG_POLL_KEY = "agent.turn_liveness.poll_s"
  22. class ActivitySnapshot(NamedTuple):
  23. """One activity-clock observation; ``(generation, activity_ts)`` must be
  24. revalidated by the commit callback under the shared lock."""
  25. generation: int
  26. activity_ts: Optional[float]
  27. idle_seconds: float
  28. def _warn_invalid_value(key: str, raw: Any, default: float) -> None:
  29. logger.warning("Invalid %s in config.yaml: %r — falling back to default %.1f.", key, raw, default)
  30. def _resolve_finite_seconds(raw: Any, *, default: float, key: str) -> float:
  31. """Coerce one duration knob, rejecting typos, NaN and Inf (never raises).
  32. NaN would silently disable the timeout via the ``> 0`` comparison and
  33. Inf would freeze the poll loop in ``Event.wait``.
  34. """
  35. try:
  36. value = float(raw)
  37. except (TypeError, ValueError):
  38. value = math.nan
  39. if not math.isfinite(value):
  40. _warn_invalid_value(key, raw, default)
  41. return default
  42. return value
  43. def resolve_turn_liveness_settings(
  44. config: Optional[dict[str, Any]] = None,
  45. ) -> tuple[Optional[float], float]:
  46. """Resolve ``(timeout_s, poll_s)``; ``timeout_s <= 0`` opts out (``None``).
  47. Invalid values (typo, NaN, Inf, non-positive poll) warn and fall back to
  48. the default. Never raises.
  49. """
  50. agent_cfg = config.get("agent") if isinstance(config, dict) else None
  51. raw_section = agent_cfg.get("turn_liveness") if isinstance(agent_cfg, dict) else None
  52. section: dict[str, Any] = raw_section if isinstance(raw_section, dict) else {}
  53. if raw_section is not None and not isinstance(raw_section, dict):
  54. _warn_invalid_value("agent.turn_liveness", raw_section, DEFAULT_TURN_LIVENESS_TIMEOUT_S)
  55. timeout_s = _resolve_finite_seconds(
  56. section.get("timeout_s", DEFAULT_TURN_LIVENESS_TIMEOUT_S),
  57. default=DEFAULT_TURN_LIVENESS_TIMEOUT_S, key=_CONFIG_TIMEOUT_KEY,
  58. )
  59. poll_s = _resolve_finite_seconds(
  60. section.get("poll_s", DEFAULT_TURN_LIVENESS_POLL_S), default=DEFAULT_TURN_LIVENESS_POLL_S,
  61. key=_CONFIG_POLL_KEY,
  62. )
  63. if poll_s <= 0:
  64. _warn_invalid_value(_CONFIG_POLL_KEY, poll_s, DEFAULT_TURN_LIVENESS_POLL_S)
  65. poll_s = DEFAULT_TURN_LIVENESS_POLL_S
  66. if timeout_s <= 0:
  67. timeout_s = None
  68. return timeout_s, poll_s
  69. class TurnLivenessWatchdog:
  70. """Sampled-idle watchdog bound to one conversation turn (via the shared
  71. periodic scheduler; timer thread orders, body runs on its own worker).
  72. ``activity_lock`` must be the SAME lock ``AIAgent._touch_activity`` stamps
  73. the activity clock with; run_agent owns the lease state and callbacks.
  74. """
  75. def __init__(
  76. self, agent: Any, *, session_id: str, timeout_s: float, poll_s: float,
  77. stop_event: threading.Event, activity_lock: threading.Lock,
  78. is_turn_active: Callable[[], bool], commit_abort: Callable[[ActivitySnapshot, str], bool],
  79. deactivate_turn: Callable[[], None],
  80. ) -> None:
  81. self._agent = agent
  82. self._session_id = session_id
  83. self._timeout_s = float(timeout_s)
  84. self._poll_s = max(MIN_TURN_LIVENESS_POLL_S, float(poll_s))
  85. self._stop_event = stop_event
  86. self._activity_lock = activity_lock
  87. self._is_turn_active = is_turn_active
  88. self._commit_abort = commit_abort
  89. self._deactivate_turn = deactivate_turn
  90. # A sleeping host is not a stalled turn: time asleep never counts toward the bound.
  91. self._awake_idle = AwakeIdleMeter()
  92. def schedule(self):
  93. """Start polling via the shared periodic scheduler; returns the cancel handle.
  94. Scheduled at turn entry, after the turn-active flag and activity clock are stamped."""
  95. from agent.periodic_scheduler import schedule
  96. return schedule(self._tick, self._poll_s)
  97. def _tick(self):
  98. """One poll. Returns False when the watchdog is finished."""
  99. if self._stop_event.is_set():
  100. return False
  101. snapshot = self._sample()
  102. if snapshot is None:
  103. return False # turn no longer active
  104. if self._awake_idle.measure(snapshot.idle_seconds) < self._timeout_s:
  105. return None
  106. # Observational only: the commit below can still veto the abort if progress
  107. # resumed; the definitive settlement is _surface_committed_abort.
  108. # Pre-commit surface is OBSERVATIONAL only: it reports the stall and that a recovery attempt is
  109. # beginning. It must not claim the abort or the lease withdrawal has committed — the next operation
  110. # can still veto the outcome. The definitive aborted/lease-stopped settlement is published by
  111. # _surface_committed_abort only after _commit_abort succeeds and the turn is deactivated (#95663
  112. # review).
  113. self._surface_stall(snapshot)
  114. message = f"Turn made no progress for {int(snapshot.idle_seconds)}s; aborting to release the session."
  115. if not self._commit_abort(snapshot, message):
  116. return None
  117. # Stop renewing the lease so a wedge the interrupt cannot unwind expires via TTL.
  118. self._deactivate_turn()
  119. self._surface_committed_abort(snapshot)
  120. from hermes_cli.observability.shared_metrics_process import record_watchdog_turn_abort
  121. record_watchdog_turn_abort(self._agent)
  122. return False
  123. def _sample(self) -> Optional[ActivitySnapshot]:
  124. with self._activity_lock:
  125. if not self._is_turn_active():
  126. return None
  127. generation = getattr(self._agent, "_turn_liveness_activity_generation", 0)
  128. activity_ts = getattr(self._agent, "_last_activity_ts", None)
  129. idle_seconds = 0.0 if activity_ts is None else max(0.0, time.time() - activity_ts)
  130. return ActivitySnapshot(generation, activity_ts, idle_seconds)
  131. def _dump_turn_thread_stack(self) -> None:
  132. """Dump the turn thread's stack to the error log when the watchdog fires (#131740).
  133. A stall this deep cannot be localized from the activity description alone — the
  134. reported wedge sat between the ``Turn ended`` log line and ``run_conversation``
  135. returning, where no activity stamps exist. The frame at the moment of the stall
  136. is the only direct evidence; the turn thread's frame is marked so the log names
  137. the wedged call site (in-tree precedent for thread-stack diagnostics:
  138. ``tools/delegate_tool_child_run.py``). Best-effort: never raises into the tick.
  139. """
  140. try:
  141. import sys
  142. import traceback
  143. frames = sys._current_frames()
  144. lines = ["## All thread stacks at turn stall (watchdog dump)"]
  145. by_ident = {t.ident: t for t in threading.enumerate() if t.ident}
  146. turn_ident = getattr(self._agent, "_execution_thread_id", None)
  147. dumped = 0
  148. for ident, frame in frames.items():
  149. if dumped >= 40:
  150. lines.append(f" <{len(frames) - dumped} more threads omitted>")
  151. break
  152. dumped += 1
  153. thread = by_ident.get(ident)
  154. name = thread.name if thread is not None else "?"
  155. marker = " <- turn thread" if ident == turn_ident else ""
  156. lines.append(f"### thread {name} tid={ident}{marker}")
  157. lines.extend(
  158. f" {sub}"
  159. for frame_line in traceback.format_stack(frame)
  160. for sub in frame_line.rstrip().split("\n")
  161. )
  162. logger.error("\n".join(lines))
  163. except Exception:
  164. logger.debug("watchdog stack dump failed", exc_info=True)
  165. def _emit_warning(self, text: str, debug_msg: str) -> None:
  166. emit_warning = getattr(self._agent, "_emit_warning", None)
  167. if not callable(emit_warning):
  168. return
  169. try:
  170. emit_warning(text)
  171. except Exception:
  172. logger.debug(debug_msg, exc_info=True)
  173. def _surface_stall(self, snapshot: ActivitySnapshot) -> None:
  174. """Log + UI-warn that recovery is beginning (not that it committed). Rate-limited per
  175. activity generation so repeatedly declined aborts do not re-log every poll."""
  176. generation = snapshot.generation
  177. if getattr(self, "_last_surfaced_generation", None) == generation:
  178. return
  179. self._last_surfaced_generation = generation
  180. self._dump_turn_thread_stack()
  181. logger.error(
  182. "Turn liveness watchdog fired for session %s: "
  183. "no progress for %.1fs (last activity: %r). "
  184. "Attempting recovery: force-interrupting the turn and "
  185. "stopping lease renewal if it cannot resume (#95548).",
  186. getattr(self._agent, "session_id", None) or self._session_id,
  187. snapshot.idle_seconds,
  188. getattr(self._agent, "_last_activity_desc", None),
  189. )
  190. self._emit_warning(
  191. "⚠️ This turn stopped making progress "
  192. f"({int(snapshot.idle_seconds)}s without activity); "
  193. "attempting recovery so the session can continue.",
  194. "Failed to emit turn liveness warning",
  195. )
  196. def _surface_committed_abort(self, snapshot: ActivitySnapshot) -> None:
  197. """Publish the definitive settlement once the abort has authority.
  198. Runs only once ``_commit_abort`` succeeded (the interrupt was published) and the turn lease was
  199. deactivated: the turn IS force-aborted and lease renewal IS stopped, so stating that is now true.
  200. Separated from the pre-commit surface so a declined abort never reports a committed outcome (#95663
  201. review).
  202. """
  203. logger.error(
  204. "Turn liveness watchdog aborted turn for session %s: "
  205. "no progress for %.1fs; turn interrupted and lease renewal "
  206. "stopped (#95548).",
  207. getattr(self._agent, "session_id", None) or self._session_id,
  208. snapshot.idle_seconds,
  209. )
  210. self._emit_warning(
  211. "⚠️ Turn aborted by the liveness watchdog "
  212. f"({int(snapshot.idle_seconds)}s without activity); "
  213. "lease renewal stopped so the session can be reclaimed. "
  214. "You can retry your message.",
  215. "Failed to emit committed-abort warning",
  216. )