kanban_watchers.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. """Kanban board watcher methods for GatewayRunner.
  2. Background loops that subscribe to kanban boards, deliver notifications and
  3. artifacts, and drive the multi-agent dispatcher. They use only ``self`` state,
  4. so they live on a mixin ``GatewayRunner`` inherits. Per-tick work lives in
  5. ``kanban_watchers_notifier`` / ``kanban_watchers_dispatcher``; shared plumbing
  6. in ``kanban_watchers_common``.
  7. """
  8. from __future__ import annotations
  9. import asyncio
  10. import os
  11. import time
  12. from pathlib import Path
  13. from typing import Any, Optional
  14. from gateway.kanban_watchers_common import (
  15. _acquire_singleton_lock,
  16. _kanban_dispatch_allowed,
  17. _release_singleton_lock,
  18. _resolve_auto_decompose_settings,
  19. _gc_retention_days,
  20. _to_thread_process_service,
  21. logger,
  22. )
  23. from gateway.kanban_watchers_notifier import _KanbanNotification, _notifier_collect
  24. from gateway.kanban_watchers_dispatcher import (
  25. _KanbanDispatcher,
  26. _log_spawn_results,
  27. _resolve_dispatcher_settings,
  28. )
  29. _IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
  30. _VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"}
  31. _GC_INTERVAL_SECONDS = 3600.0
  32. _HEALTH_WINDOW = 6
  33. class GatewayKanbanWatchersMixin:
  34. """Kanban watcher / notifier / dispatcher loops for GatewayRunner."""
  35. def _owns_kanban_dispatcher_lock(self) -> bool:
  36. return getattr(self, "_kanban_dispatcher_lock_handle", None) is not None
  37. def _release_kanban_dispatcher_lock(self) -> None:
  38. """Clear notifier-visible ownership before releasing the OS lock."""
  39. handle = getattr(self, "_kanban_dispatcher_lock_handle", None)
  40. self._kanban_dispatcher_lock_handle = None
  41. _release_singleton_lock(handle)
  42. async def _sleep_between_ticks(self, interval: float) -> None:
  43. """Sleep *interval* (floored to 1s) in 1s slices so stop() never waits a full interval."""
  44. interval = max(interval, 1.0)
  45. slept = 0.0
  46. while slept < interval and self._running:
  47. await asyncio.sleep(min(1.0, interval - slept))
  48. slept += 1.0
  49. async def _kanban_notifier_watcher(self, interval: float = 5.0) -> None:
  50. """Poll ``kanban_notify_subs`` and deliver terminal events to users.
  51. Per subscription, claims ``task_events`` newer than the stored cursor
  52. (kinds in TERMINAL_KINDS), sends one message per event, then advances
  53. the cursor. The subscription is removed only when the task is
  54. ``archived``: ``done`` is reversible, so the cursor — not unsubscribing
  55. — is the dedup mechanism (unsub-on-terminal dropped users when the
  56. dispatcher respawned a crashed task). All SQLite work runs in a thread;
  57. one tick's failure never stops the next.
  58. """
  59. try:
  60. from hermes_cli.config import load_config as _load_config
  61. cfg = _load_config()
  62. kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
  63. except Exception as exc:
  64. logger.warning("kanban notifier: cannot load config (%s); continuing enabled", exc)
  65. kanban_cfg = {}
  66. if not kanban_cfg.get("notify_in_gateway", True):
  67. logger.info("kanban notifier: disabled via config kanban.notify_in_gateway=false")
  68. return
  69. from gateway.config import Platform as _Platform
  70. try:
  71. from hermes_cli import kanban_db as _kb
  72. except Exception:
  73. logger.warning("kanban notifier: kanban_db not importable; notifier disabled")
  74. return
  75. sub_fail_counts: dict[tuple, int] = getattr(self, "_kanban_sub_fail_counts", {})
  76. self._kanban_sub_fail_counts = sub_fail_counts
  77. notifier_profile = getattr(self, "_kanban_notifier_profile", None) or self._active_profile_name()
  78. self._kanban_notifier_profile = notifier_profile
  79. # Initial delay so the gateway can finish wiring adapters.
  80. await asyncio.sleep(5)
  81. # Stale done-sub GC: subs survive ``done``, so boards that never
  82. # archive would accumulate rows scanned every tick. One DELETE per
  83. # board, at startup (0 → first tick) and at most hourly.
  84. _gc_next_at = 0.0
  85. while self._running:
  86. try:
  87. _gc_due = time.monotonic() >= _gc_next_at
  88. _retention = 30
  89. if _gc_due:
  90. _gc_next_at = time.monotonic() + _GC_INTERVAL_SECONDS
  91. _retention = _gc_retention_days()
  92. deliveries = await asyncio.to_thread(
  93. _notifier_collect, self, _kb,
  94. notifier_profile=notifier_profile, gc_due=_gc_due, gc_retention_days=_retention,
  95. )
  96. for d in deliveries:
  97. await _KanbanNotification(
  98. self, d, platform_cls=_Platform, sub_fail_counts=sub_fail_counts,
  99. ).deliver()
  100. except Exception as exc:
  101. logger.warning("kanban notifier tick failed: %s", exc)
  102. await self._sleep_between_ticks(interval)
  103. def _kanban_sub_op(self, board: Optional[str], op: str, sub: dict, **extra: Any) -> None:
  104. """Sync helper (runs in to_thread): call ``kanban_db_notify.<op>`` for one subscription on its board."""
  105. from hermes_cli import kanban_db as _kb
  106. from hermes_cli import kanban_db_connect as _kbc
  107. from hermes_cli import kanban_db_notify as _kbn
  108. # Cursor writes are machine flow: the sub's board slug must resolve
  109. # through the env pin on a dispatcher-pinned box (same as the notifier
  110. # tick), or advance/rewind land on a per-slug DB the notifier never
  111. # reads and cursors silently reset.
  112. with _kb.pin_first_board_resolution():
  113. conn = _kbc.connect(board=board)
  114. try:
  115. getattr(_kbn, op)(
  116. conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"],
  117. thread_id=sub.get("thread_id") or "", **extra,
  118. )
  119. finally:
  120. conn.close()
  121. def _kanban_advance(self, sub: dict, cursor: int, board: Optional[str] = None) -> None:
  122. self._kanban_sub_op(board, "advance_notify_cursor", sub, new_cursor=cursor)
  123. def _kanban_unsub(self, sub: dict, board: Optional[str] = None) -> None:
  124. self._kanban_sub_op(board, "remove_notify_sub", sub)
  125. def _kanban_rewind(self, sub: dict, claimed_cursor: int, old_cursor: int, board: Optional[str] = None) -> None:
  126. """Undo a claimed notification cursor after send failure."""
  127. self._kanban_sub_op(board, "rewind_notify_cursor", sub, claimed_cursor=claimed_cursor, old_cursor=old_cursor)
  128. async def _deliver_kanban_artifacts(self, *, adapter, chat_id: str, metadata: dict, event_payload: Optional[dict], task) -> None:
  129. """Upload artifact files referenced by a completed kanban task.
  130. Sources, in priority order: ``event_payload['artifacts']``,
  131. ``event_payload['summary']``, then ``task.result`` (legacy). Paths are
  132. deduplicated, missing files are skipped (may be mentioned for
  133. reference only), and upload errors are logged, never raised.
  134. """
  135. raw_paths: list[str] = []
  136. prose_paths: list[str] = []
  137. if isinstance(event_payload, dict):
  138. raw = event_payload.get("artifacts")
  139. if isinstance(raw, (list, tuple)):
  140. raw_paths += [item for item in raw if isinstance(item, str)]
  141. summary = event_payload.get("summary")
  142. if isinstance(summary, str) and summary:
  143. prose_paths += adapter.extract_local_files(summary)[0]
  144. if task is not None and getattr(task, "result", None):
  145. prose_paths += adapter.extract_local_files(str(task.result))[0]
  146. # A staged copy and the scratch original it was copied from are the
  147. # same deliverable; on a review handoff the original still exists, so
  148. # prose mentions of it must not upload the file a second time.
  149. staged_names = {os.path.basename(p) for p in raw_paths}
  150. raw_paths += [p for p in prose_paths if os.path.basename(p) not in staged_names]
  151. candidates: list[str] = []
  152. for path in raw_paths:
  153. expanded = os.path.expanduser(path) if path else ""
  154. if expanded and expanded not in candidates and os.path.isfile(expanded):
  155. candidates.append(expanded)
  156. if not candidates:
  157. return
  158. from gateway.platforms.base import BasePlatformAdapter
  159. candidates = BasePlatformAdapter.filter_local_delivery_paths(candidates)
  160. if not candidates:
  161. return
  162. from urllib.parse import quote as _quote
  163. # Images ride one send_multiple_images call (batch uploads on Signal/Slack).
  164. image_paths = [p for p in candidates if Path(p).suffix.lower() in _IMAGE_EXTS]
  165. other_paths = [p for p in candidates if Path(p).suffix.lower() not in _IMAGE_EXTS]
  166. if image_paths:
  167. try:
  168. batch = [(f"file://{_quote(p)}", "") for p in image_paths]
  169. await adapter.send_multiple_images(chat_id=chat_id, images=batch, metadata=metadata)
  170. except Exception as exc:
  171. logger.warning("kanban notifier: image batch upload failed: %s", exc)
  172. for path in other_paths:
  173. try:
  174. if Path(path).suffix.lower() in _VIDEO_EXTS:
  175. await adapter.send_video(chat_id=chat_id, video_path=path, metadata=metadata)
  176. else:
  177. await adapter.send_document(chat_id=chat_id, file_path=path, metadata=metadata)
  178. except Exception as exc:
  179. logger.warning("kanban notifier: artifact upload (%s) failed: %s", path, exc)
  180. def _kanban_dispatcher_boot(self) -> Optional[tuple]:
  181. """Resolve config, kanban_db and the singleton lock; None when the dispatcher must not run.
  182. Config is read once at boot (restart to apply), except the auto-decompose
  183. toggle which is re-read every tick. The env var is an escape hatch to
  184. disable without editing YAML.
  185. """
  186. try:
  187. from hermes_cli.config import load_config as _load_config
  188. except Exception:
  189. logger.warning("kanban dispatcher: config loader unavailable; disabled")
  190. return None
  191. env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower()
  192. if env_override in {"0", "false", "no", "off"}:
  193. logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env")
  194. return None
  195. try:
  196. cfg = _load_config()
  197. except Exception as exc:
  198. logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc)
  199. return None
  200. kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
  201. if not kanban_cfg.get("dispatch_in_gateway", True):
  202. logger.info("kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false")
  203. return None
  204. try:
  205. from hermes_cli import kanban_db as _kb
  206. except Exception:
  207. logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled")
  208. return None
  209. # Single-dispatcher backstop (see _acquire_singleton_lock). The lock
  210. # lives at the machine-global kanban root, so it serialises ALL gateways.
  211. self._kanban_dispatcher_lock_handle = None
  212. _lock_path = _kb.kanban_home() / "kanban" / ".dispatcher.lock"
  213. _lock_handle, _lock_state = _acquire_singleton_lock(_lock_path)
  214. if _lock_state == "contended":
  215. logger.info("kanban dispatcher: another gateway already holds the dispatcher "
  216. "lock (%s); this gateway will NOT dispatch.", _lock_path)
  217. return None
  218. if _lock_state == "held":
  219. self._kanban_dispatcher_lock_handle = _lock_handle # hold for process lifetime
  220. logger.info("kanban dispatcher: holding singleton dispatcher lock (%s)", _lock_path)
  221. else:
  222. logger.warning("kanban dispatcher: advisory lock unavailable at %s; proceeding "
  223. "on config control alone.", _lock_path)
  224. return _load_config, _kb, kanban_cfg
  225. async def _kanban_dispatcher_watcher(self) -> None:
  226. """Embedded kanban dispatcher — one tick every `dispatch_interval_seconds`.
  227. Gated by `kanban.dispatch_in_gateway` (default True); when false the
  228. loop exits and an external `hermes kanban daemon` is expected. Each
  229. tick runs :func:`kanban_db_dispatch.dispatch_once` in a thread; one tick's
  230. failure never stops the next. Shutdown: ``self._running`` is checked
  231. between ticks and the in-flight ``to_thread`` returns on its own.
  232. """
  233. boot = self._kanban_dispatcher_boot()
  234. if boot is None:
  235. return
  236. _load_config, _kb, kanban_cfg = boot
  237. settings = _resolve_dispatcher_settings(kanban_cfg, _kb)
  238. interval = settings.interval
  239. # Initial delay so adapters are wired before workers spawn (matches the notifier).
  240. await asyncio.sleep(5)
  241. # Health telemetry (mirrors `_cmd_daemon`): warn when the ready queue
  242. # is non-empty but spawns are 0 for N consecutive ticks — usually a
  243. # broken PATH, missing venv, or credential loss.
  244. bad_ticks = 0
  245. last_warn_at = 0
  246. results: Optional[list] = None
  247. dispatcher = _KanbanDispatcher(_kb, settings)
  248. logger.info("kanban dispatcher: embedded in gateway (interval=%.1fs)", interval)
  249. while self._running:
  250. try:
  251. # Reap zombies before per-board work so a board DB failure
  252. # cannot block cleanup of unrelated workers.
  253. from hermes_cli import kanban_db_dispatch as _kbd
  254. pids = await _to_thread_process_service(_kbd.reap_worker_zombies)
  255. if pids:
  256. logger.info("kanban dispatcher: reaped %d zombie worker(s), pids=%s", len(pids), pids)
  257. except Exception:
  258. logger.exception("kanban dispatcher: zombie reaper failed")
  259. try:
  260. # Emergency stop (`hermes pause`): no auto-decompose or
  261. # dispatch while paused; running workers finish naturally.
  262. if not _kanban_dispatch_allowed():
  263. bad_ticks = 0
  264. else:
  265. # Re-read the auto-decompose toggle live so disabling it
  266. # takes effect on the next tick, not on restart.
  267. _ad_enabled, _ad_per_tick = _resolve_auto_decompose_settings(_load_config)
  268. # See #49638.
  269. if _ad_enabled:
  270. await _to_thread_process_service(dispatcher.auto_decompose_tick, _ad_per_tick)
  271. results = await _to_thread_process_service(dispatcher.tick_once)
  272. any_spawned = _log_spawn_results(results)
  273. ready_pending = await _to_thread_process_service(dispatcher.ready_nonempty)
  274. bad_ticks = bad_ticks + 1 if ready_pending and not any_spawned else 0
  275. now = int(time.time())
  276. if bad_ticks >= _HEALTH_WINDOW and now - last_warn_at >= 300:
  277. held = _kbd.describe_suppression(res for _slug, res in (results or []))
  278. logger.warning(
  279. "kanban dispatcher stuck: ready queue non-empty for "
  280. "%d consecutive ticks but 0 workers spawned.%s Check "
  281. "profile health (venv, PATH, credentials) and "
  282. "`hermes kanban list --status ready`.",
  283. bad_ticks, f" Last tick held back: {held}." if held else "",
  284. )
  285. last_warn_at = now
  286. except asyncio.CancelledError:
  287. logger.debug("kanban dispatcher: cancelled")
  288. self._release_kanban_dispatcher_lock()
  289. raise
  290. except Exception:
  291. logger.exception("kanban dispatcher: unexpected watcher error")
  292. await self._sleep_between_ticks(interval)
  293. self._release_kanban_dispatcher_lock()