activity_tracking.py 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. """Turn-liveness activity tracking for ``AIAgent`` (gateway watchdog + session activity persistence).
  2. ``_touch_activity`` is the single write path; persistence is rate-limited and never raises.
  3. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
  4. """
  5. import logging
  6. import os
  7. import threading
  8. import time
  9. from contextlib import suppress
  10. from typing import Optional
  11. from agent.session_activity import ActivityProvenance
  12. # Same logger name as the origin module so log records / caplog filters are unchanged.
  13. logger = logging.getLogger("run_agent")
  14. def _activity_lock(obj) -> threading.Lock:
  15. """Lazy per-instance ``_turn_liveness_activity_lock`` (so ``__new__``/SimpleNamespace doubles work)."""
  16. _lock = getattr(obj, "_turn_liveness_activity_lock", None)
  17. if _lock is None:
  18. _lock = threading.Lock()
  19. obj._turn_liveness_activity_lock = _lock
  20. return _lock
  21. class ActivityTrackingMixin:
  22. """Liveness timestamps/labels and rate-limited session activity persistence."""
  23. def _liveness_activity_lock(self) -> threading.Lock:
  24. """Shared lock for the activity clock and its generation counter.
  25. ``_touch_activity`` stamps under it and the liveness watchdog samples/commits under it, so a stall
  26. observation can never abort a turn that resumed in between.
  27. Created lazily so ``AIAgent.__new__``-based test doubles keep working. See #95663.
  28. """
  29. return _activity_lock(self)
  30. def _touch_activity(
  31. self, desc: str, *, provenance: Optional[ActivityProvenance] = None,
  32. force_persist: bool = False,
  33. ) -> None:
  34. """Update the last-activity timestamp and description (thread-safe).
  35. Bumps a monotonic generation under the activity lock so the watchdog can bind a stall observation to
  36. the exact ``(generation, timestamp)`` it sampled. Also bridges (rate-limited, best-effort) to the
  37. kanban heartbeat when this is a dispatcher-spawned worker, and to the durable SessionDB activity
  38. projection. ``provenance`` names special writers (compression); ``force_persist`` bypasses the
  39. SessionDB rate limit — as does any terminal compression provenance, which must converge the durable
  40. row at once (nothing writes after it). Module-level lock helper, not ``self._liveness_activity_lock()``:
  41. doubles bind only ``_touch_activity`` (tests/agent/test_session_activity_persist.py).
  42. Bridge is rate-limited (60s) and best-effort — it never raises into the agent loop. See #31752.
  43. See #72016, #72039.
  44. """
  45. from agent.session_activity import (
  46. bound_activity_description, is_terminal_compression_provenance,
  47. normalize_activity_provenance, reset_session_activity_persist_window,
  48. )
  49. resolved_provenance = normalize_activity_provenance(provenance)
  50. with _activity_lock(self):
  51. self._turn_liveness_activity_generation = (
  52. getattr(self, "_turn_liveness_activity_generation", 0) + 1
  53. )
  54. self._last_activity_ts = time.time()
  55. self._last_activity_desc = bound_activity_description(desc)
  56. self._last_activity_provenance = resolved_provenance
  57. # Real progress invalidates a reserved abort claim; an in-flight watchdog interrupt must abandon
  58. # itself at the final mutation edge.
  59. self._turn_liveness_abort_claim = None
  60. if os.environ.get("HERMES_KANBAN_TASK"):
  61. # Never let the bridge break the loop; this guard covers import-time failures.
  62. with suppress(Exception):
  63. from tools.kanban_tools import (
  64. heartbeat_current_worker_from_env, inject_new_comments_from_env
  65. )
  66. heartbeat_current_worker_from_env()
  67. # Fold new operator notes into the running turn (OUT-OF-BAND steer).
  68. inject_new_comments_from_env(self)
  69. if force_persist or is_terminal_compression_provenance(resolved_provenance):
  70. # A terminal compression stamp must land durably NOW. The heartbeat wrote "context compression
  71. # in progress" moments ago, so deferring this write to the next 60s window leaves the durable
  72. # row advertising a compression that has already ended — and no later writer corrects it once
  73. # the turn is over or the host is gone (the permanently "stuck" chat, #72039 follow-up).
  74. reset_session_activity_persist_window(self)
  75. self._persist_session_activity_if_due()
  76. def _persist_session_activity_if_due(self) -> None:
  77. """Best-effort durable activity heartbeat for SessionDB consumers.
  78. Cadence pinned by ``SESSION_ACTIVITY_HEARTBEAT_MIN_INTERVAL_SECONDS`` (config-independent). Fail-open:
  79. a failed write never raises into the agent loop.
  80. """
  81. session_id = getattr(self, "session_id", None)
  82. session_db = getattr(self, "_session_db", None)
  83. if not session_id or session_db is None:
  84. return
  85. touch = getattr(session_db, "touch_session_activity", None)
  86. if not callable(touch):
  87. return
  88. from agent.session_activity import (
  89. SESSION_ACTIVITY_HEARTBEAT_MIN_INTERVAL_SECONDS, normalize_activity_provenance
  90. )
  91. now_mono = time.monotonic()
  92. last_mono = getattr(self, "_session_activity_last_persist_mono", 0.0)
  93. # 0.0 is the never-persisted / reset sentinel and always writes through: CLOCK_MONOTONIC is
  94. # roughly host uptime, so on a freshly booted VM ``now_mono`` itself can be under the interval.
  95. if last_mono and (now_mono - last_mono) < SESSION_ACTIVITY_HEARTBEAT_MIN_INTERVAL_SECONDS:
  96. return
  97. self._session_activity_last_persist_mono = now_mono
  98. from agent.runtime_session_store import call_off_event_loop # Relay callbacks run on a loop (#139029)
  99. try:
  100. call_off_event_loop(
  101. touch,
  102. session_id,
  103. getattr(self, "_last_activity_ts", None),
  104. description=getattr(self, "_last_activity_desc", None),
  105. provenance=normalize_activity_provenance(
  106. getattr(self, "_last_activity_provenance", None)
  107. ),
  108. )
  109. except Exception:
  110. # Heartbeat is observation-only; never let its I/O break the loop.
  111. logger.debug("session activity heartbeat write failed (ignored)", exc_info=True)
  112. def _reset_activity_labels_after_turn(self) -> None:
  113. """Drop mid-turn activity labels once the turn is no longer running.
  114. Keeps ``_last_activity_ts`` so idle/watchdog clocks stay continuous across turns; clears description +
  115. provenance so idle agents / SessionDB listings stop advertising the last mid-turn stamp.
  116. See #15654, #72039.
  117. """
  118. self._last_activity_desc = ""
  119. self._last_activity_provenance = ActivityProvenance.UNKNOWN
  120. session_id = getattr(self, "session_id", None)
  121. session_db = getattr(self, "_session_db", None)
  122. if not session_id or session_db is None:
  123. return
  124. clear = getattr(session_db, "clear_session_activity_labels", None)
  125. if not callable(clear):
  126. return
  127. from agent.runtime_session_store import call_off_event_loop
  128. with suppress(Exception): # never let durable cleanup I/O break turn teardown
  129. call_off_event_loop(clear, session_id)