| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250 |
- """Turn liveness watchdog: force-abort turns that stall silently.
- A turn can wedge mid-flight with no error and its durable lease still renewing, so nothing
- ever frees the session. This module owns config resolution (``agent.turn_liveness``,
- validated — a typo, NaN or Inf warns and falls back), the sampling state machine, and the
- watcher thread. Race safety: the abort decision is bound to the observed
- ``(generation, timestamp)``; the commit callback revalidates it under the same lock
- ``_touch_activity`` stamps with, so a turn that resumed is never hard-cancelled.
- """
- from __future__ import annotations
- import logging
- import math
- import threading
- import time
- from typing import Any, Callable, Dict, NamedTuple, Optional, Tuple
- from agent.session_activity import AwakeIdleMeter
- logger = logging.getLogger(__name__)
- DEFAULT_TURN_LIVENESS_TIMEOUT_S = 600.0
- DEFAULT_TURN_LIVENESS_POLL_S = 15.0
- MIN_TURN_LIVENESS_POLL_S = 0.01
- _CONFIG_TIMEOUT_KEY = "agent.turn_liveness.timeout_s"
- _CONFIG_POLL_KEY = "agent.turn_liveness.poll_s"
- class ActivitySnapshot(NamedTuple):
- """One activity-clock observation; ``(generation, activity_ts)`` must be
- revalidated by the commit callback under the shared lock."""
- generation: int
- activity_ts: Optional[float]
- idle_seconds: float
- def _warn_invalid_value(key: str, raw: Any, default: float) -> None:
- logger.warning("Invalid %s in config.yaml: %r — falling back to default %.1f.", key, raw, default)
- def _resolve_finite_seconds(raw: Any, *, default: float, key: str) -> float:
- """Coerce one duration knob, rejecting typos, NaN and Inf (never raises).
- NaN would silently disable the timeout via the ``> 0`` comparison and
- Inf would freeze the poll loop in ``Event.wait``.
- """
- try:
- value = float(raw)
- except (TypeError, ValueError):
- value = math.nan
- if not math.isfinite(value):
- _warn_invalid_value(key, raw, default)
- return default
- return value
- def resolve_turn_liveness_settings(
- config: Optional[dict[str, Any]] = None,
- ) -> tuple[Optional[float], float]:
- """Resolve ``(timeout_s, poll_s)``; ``timeout_s <= 0`` opts out (``None``).
- Invalid values (typo, NaN, Inf, non-positive poll) warn and fall back to
- the default. Never raises.
- """
- agent_cfg = config.get("agent") if isinstance(config, dict) else None
- raw_section = agent_cfg.get("turn_liveness") if isinstance(agent_cfg, dict) else None
- section: dict[str, Any] = raw_section if isinstance(raw_section, dict) else {}
- if raw_section is not None and not isinstance(raw_section, dict):
- _warn_invalid_value("agent.turn_liveness", raw_section, DEFAULT_TURN_LIVENESS_TIMEOUT_S)
- timeout_s = _resolve_finite_seconds(
- section.get("timeout_s", DEFAULT_TURN_LIVENESS_TIMEOUT_S),
- default=DEFAULT_TURN_LIVENESS_TIMEOUT_S, key=_CONFIG_TIMEOUT_KEY,
- )
- poll_s = _resolve_finite_seconds(
- section.get("poll_s", DEFAULT_TURN_LIVENESS_POLL_S), default=DEFAULT_TURN_LIVENESS_POLL_S,
- key=_CONFIG_POLL_KEY,
- )
- if poll_s <= 0:
- _warn_invalid_value(_CONFIG_POLL_KEY, poll_s, DEFAULT_TURN_LIVENESS_POLL_S)
- poll_s = DEFAULT_TURN_LIVENESS_POLL_S
- if timeout_s <= 0:
- timeout_s = None
- return timeout_s, poll_s
- class TurnLivenessWatchdog:
- """Sampled-idle watchdog bound to one conversation turn (via the shared
- periodic scheduler; timer thread orders, body runs on its own worker).
- ``activity_lock`` must be the SAME lock ``AIAgent._touch_activity`` stamps
- the activity clock with; run_agent owns the lease state and callbacks.
- """
- def __init__(
- self, agent: Any, *, session_id: str, timeout_s: float, poll_s: float,
- stop_event: threading.Event, activity_lock: threading.Lock,
- is_turn_active: Callable[[], bool], commit_abort: Callable[[ActivitySnapshot, str], bool],
- deactivate_turn: Callable[[], None],
- ) -> None:
- self._agent = agent
- self._session_id = session_id
- self._timeout_s = float(timeout_s)
- self._poll_s = max(MIN_TURN_LIVENESS_POLL_S, float(poll_s))
- self._stop_event = stop_event
- self._activity_lock = activity_lock
- self._is_turn_active = is_turn_active
- self._commit_abort = commit_abort
- self._deactivate_turn = deactivate_turn
- # A sleeping host is not a stalled turn: time asleep never counts toward the bound.
- self._awake_idle = AwakeIdleMeter()
- def schedule(self):
- """Start polling via the shared periodic scheduler; returns the cancel handle.
- Scheduled at turn entry, after the turn-active flag and activity clock are stamped."""
- from agent.periodic_scheduler import schedule
- return schedule(self._tick, self._poll_s)
- def _tick(self):
- """One poll. Returns False when the watchdog is finished."""
- if self._stop_event.is_set():
- return False
- snapshot = self._sample()
- if snapshot is None:
- return False # turn no longer active
- if self._awake_idle.measure(snapshot.idle_seconds) < self._timeout_s:
- return None
- # Observational only: the commit below can still veto the abort if progress
- # resumed; the definitive settlement is _surface_committed_abort.
- # Pre-commit surface is OBSERVATIONAL only: it reports the stall and that a recovery attempt is
- # beginning. It must not claim the abort or the lease withdrawal has committed — the next operation
- # can still veto the outcome. The definitive aborted/lease-stopped settlement is published by
- # _surface_committed_abort only after _commit_abort succeeds and the turn is deactivated (#95663
- # review).
- self._surface_stall(snapshot)
- message = f"Turn made no progress for {int(snapshot.idle_seconds)}s; aborting to release the session."
- if not self._commit_abort(snapshot, message):
- return None
- # Stop renewing the lease so a wedge the interrupt cannot unwind expires via TTL.
- self._deactivate_turn()
- self._surface_committed_abort(snapshot)
- from hermes_cli.observability.shared_metrics_process import record_watchdog_turn_abort
- record_watchdog_turn_abort(self._agent)
- return False
- def _sample(self) -> Optional[ActivitySnapshot]:
- with self._activity_lock:
- if not self._is_turn_active():
- return None
- generation = getattr(self._agent, "_turn_liveness_activity_generation", 0)
- activity_ts = getattr(self._agent, "_last_activity_ts", None)
- idle_seconds = 0.0 if activity_ts is None else max(0.0, time.time() - activity_ts)
- return ActivitySnapshot(generation, activity_ts, idle_seconds)
- def _dump_turn_thread_stack(self) -> None:
- """Dump the turn thread's stack to the error log when the watchdog fires (#131740).
- A stall this deep cannot be localized from the activity description alone — the
- reported wedge sat between the ``Turn ended`` log line and ``run_conversation``
- returning, where no activity stamps exist. The frame at the moment of the stall
- is the only direct evidence; the turn thread's frame is marked so the log names
- the wedged call site (in-tree precedent for thread-stack diagnostics:
- ``tools/delegate_tool_child_run.py``). Best-effort: never raises into the tick.
- """
- try:
- import sys
- import traceback
- frames = sys._current_frames()
- lines = ["## All thread stacks at turn stall (watchdog dump)"]
- by_ident = {t.ident: t for t in threading.enumerate() if t.ident}
- turn_ident = getattr(self._agent, "_execution_thread_id", None)
- dumped = 0
- for ident, frame in frames.items():
- if dumped >= 40:
- lines.append(f" <{len(frames) - dumped} more threads omitted>")
- break
- dumped += 1
- thread = by_ident.get(ident)
- name = thread.name if thread is not None else "?"
- marker = " <- turn thread" if ident == turn_ident else ""
- lines.append(f"### thread {name} tid={ident}{marker}")
- lines.extend(
- f" {sub}"
- for frame_line in traceback.format_stack(frame)
- for sub in frame_line.rstrip().split("\n")
- )
- logger.error("\n".join(lines))
- except Exception:
- logger.debug("watchdog stack dump failed", exc_info=True)
- def _emit_warning(self, text: str, debug_msg: str) -> None:
- emit_warning = getattr(self._agent, "_emit_warning", None)
- if not callable(emit_warning):
- return
- try:
- emit_warning(text)
- except Exception:
- logger.debug(debug_msg, exc_info=True)
- def _surface_stall(self, snapshot: ActivitySnapshot) -> None:
- """Log + UI-warn that recovery is beginning (not that it committed). Rate-limited per
- activity generation so repeatedly declined aborts do not re-log every poll."""
- generation = snapshot.generation
- if getattr(self, "_last_surfaced_generation", None) == generation:
- return
- self._last_surfaced_generation = generation
- self._dump_turn_thread_stack()
- logger.error(
- "Turn liveness watchdog fired for session %s: "
- "no progress for %.1fs (last activity: %r). "
- "Attempting recovery: force-interrupting the turn and "
- "stopping lease renewal if it cannot resume (#95548).",
- getattr(self._agent, "session_id", None) or self._session_id,
- snapshot.idle_seconds,
- getattr(self._agent, "_last_activity_desc", None),
- )
- self._emit_warning(
- "⚠️ This turn stopped making progress "
- f"({int(snapshot.idle_seconds)}s without activity); "
- "attempting recovery so the session can continue.",
- "Failed to emit turn liveness warning",
- )
- def _surface_committed_abort(self, snapshot: ActivitySnapshot) -> None:
- """Publish the definitive settlement once the abort has authority.
- Runs only once ``_commit_abort`` succeeded (the interrupt was published) and the turn lease was
- deactivated: the turn IS force-aborted and lease renewal IS stopped, so stating that is now true.
- Separated from the pre-commit surface so a declined abort never reports a committed outcome (#95663
- review).
- """
- logger.error(
- "Turn liveness watchdog aborted turn for session %s: "
- "no progress for %.1fs; turn interrupted and lease renewal "
- "stopped (#95548).",
- getattr(self._agent, "session_id", None) or self._session_id,
- snapshot.idle_seconds,
- )
- self._emit_warning(
- "⚠️ Turn aborted by the liveness watchdog "
- f"({int(snapshot.idle_seconds)}s without activity); "
- "lease renewal stopped so the session can be reclaimed. "
- "You can retry your message.",
- "Failed to emit committed-abort warning",
- )
|