kanban_watchers_notifier.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. """Kanban notifier: claim terminal task events per subscription and deliver them.
  2. ``GatewayKanbanWatchersMixin._kanban_notifier_watcher`` owns the loop and
  3. the GC cadence; the per-tick claim (``_notifier_collect``) and the
  4. per-subscription delivery (``_KanbanNotification``) live here.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import contextlib
  9. import re
  10. from functools import partial
  11. from pathlib import Path
  12. import weakref
  13. from typing import Any, Callable, Optional
  14. from agent.i18n import t
  15. from gateway.kanban_watchers_common import _list_boards, _to_thread_process_service, logger
  16. from gateway.wake import session_owned_by_profile
  17. def _kbc():
  18. from hermes_cli import kanban_db_connect
  19. return kanban_db_connect
  20. def _kbn():
  21. from hermes_cli import kanban_db_notify
  22. return kanban_db_notify
  23. def _pin_first():
  24. """Machine-flow board resolution: env pins outrank the enumerated slug.
  25. The slug here came from ``list_boards()``, not from a user — on a box whose
  26. env pins ``HERMES_KANBAN_DB`` every board must resolve to the pinned file
  27. or the notifier reads per-slug DBs nobody writes (see
  28. ``kanban_db.pin_first_board_resolution``)."""
  29. from hermes_cli import kanban_db
  30. return kanban_db.pin_first_board_resolution()
  31. # "status" covers dashboard drag-drop and `_set_status_direct()`.
  32. # ``review_requested`` wakes the origin like a block but is not one;
  33. # the task is not archived so later review cycles keep notifying.
  34. TERMINAL_KINDS = ("completed", "blocked", "gave_up", "crashed", "timed_out", "status", "archived", "unblocked", "block_loop_detected", "review_requested", "changes_requested")
  35. # Kinds that hand a decision back to the origin, which must take a turn.
  36. # status/archived/unblocked are bookkeeping.
  37. _WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked", "review_requested", "changes_requested", "block_loop_detected")
  38. def diagnostic_event(ev) -> bool:
  39. """Infrastructure attention is distinct from an explicit owner decision."""
  40. if ev.kind in {"crashed", "timed_out", "gave_up"}:
  41. return True
  42. if ev.kind in {"blocked", "block_loop_detected"}:
  43. return (ev.payload or {}).get("kind") != "needs_input"
  44. return ev.kind == "status" and (ev.payload or {}).get("status") in {"blocked", "triage"}
  45. # Consecutive send failures (adapter raised OR reported SendResult(success=False))
  46. # before a sub is dropped as a dead chat. 12 ≈ 60s at the 5s cadence: a transient
  47. # API outage must not permanently unsubscribe a live review-gate channel.
  48. # Subscriptions are removed only when the task reaches the irreversible archived status. ``done`` is
  49. # reversible in review/controller flows, so removing its subscription would silence a later reopen. We used
  50. # to also unsub on any terminal event kind (gave_up / crashed / timed_out / blocked), but that silently
  51. # dropped the user out of the loop whenever the dispatcher respawned the task: a worker that crashes, gets
  52. # reclaimed, runs again, and crashes a second time would only notify on the first crash because the
  53. # subscription was deleted after the first event. Same shape as the reblock-after-unblock cycle that PR
  54. # #22941 fixed for `blocked`. Keeping the subscription alive until the task is archived lets the cursor
  55. # (advanced atomically by claim_unseen_events_for_sub) handle dedup, and any retry-loop event reaches the
  56. # user. Per-subscription send-failure counter. Adapter.send raising means the chat is dead (deleted, bot
  57. # kicked, etc.) — after N consecutive send failures the sub is dropped so we don't spin against a dead chat
  58. # every 5 seconds forever. A genuinely dead chat still drops, just ~60s later — a fine trade for an
  59. # unattended gate where a false drop means silent work pileup.
  60. MAX_SEND_FAILURES = 12
  61. _LOCAL_PATH_RE = re.compile(r"(?<![\w:/])(?:/(?:Users|home|private|tmp|var|etc|workspace)/[^\s,;]+|" r"[A-Za-z]:\\[^\s,;]+)")
  62. def _safe_review_reason(value: Any, limit: int = 160) -> str:
  63. """Return a mobile-friendly review reason safe for external delivery."""
  64. from agent.redact import redact_sensitive_text
  65. reason = redact_sensitive_text("" if value is None else str(value), force=True, redact_url_credentials=True)
  66. reason = " ".join(_LOCAL_PATH_RE.sub("[local path]", reason).split())
  67. if len(reason) > limit:
  68. reason = reason[: limit - 1].rstrip() + "…"
  69. return reason
  70. def _wake_scope_id(adapter: Any, sub: dict) -> Optional[str]:
  71. """Return the tenant scope (Slack workspace) a subscription's wake keys to.
  72. ``build_session_key()`` includes ``scope_id`` on multi-tenant platforms,
  73. so the wake must carry the same scope as inbound messages. Persisted
  74. ``delivery_metadata`` wins (it records the creating scope); the adapter's
  75. live chat → scope map only covers rows without metadata. ``None`` means
  76. unscoped, matching an unscoped platform's key.
  77. """
  78. delivery_meta = sub.get("delivery_metadata")
  79. if isinstance(delivery_meta, dict):
  80. for key in ("scope_id", "guild_id", "slack_team_id", "team_id"):
  81. value = delivery_meta.get(key)
  82. if value:
  83. return str(value)
  84. resolver = getattr(adapter, "scope_id_for_chat", None)
  85. if not callable(resolver):
  86. return None
  87. try:
  88. resolved = resolver(str(sub.get("chat_id") or ""))
  89. except Exception as exc:
  90. # An adapter-side lookup failure yields no scope, never an error.
  91. logger.debug("kanban notifier: scope lookup failed for chat %s: %s", sub.get("chat_id"), exc, exc_info=True)
  92. return None
  93. return str(resolved) if resolved else None
  94. _ANCHORLESS_WARNED: set[tuple] = set()
  95. def _warn_anchorless_thread_sub_once(sub: dict, platform: str) -> None:
  96. """A thread-shaped subscription without ``parent_chat_id`` cannot match a channel-level
  97. ``profile_routes`` entry, so the fail-closed route gate skips it on every tick. Say so ONCE per
  98. row at WARNING — a subscription that can never deliver was invisible below DEBUG (#110919)."""
  99. metadata = sub.get("delivery_metadata") or {}
  100. thread_like = bool(sub.get("thread_id")) or (sub.get("chat_type") or metadata.get("chat_type")) in {
  101. "thread", "forum", "forum_post", "forum-post", "topic"}
  102. if not thread_like or metadata.get("parent_chat_id"):
  103. return
  104. key = (sub.get("task_id"), platform, sub.get("chat_id"), sub.get("thread_id") or "")
  105. if key in _ANCHORLESS_WARNED:
  106. return
  107. _ANCHORLESS_WARNED.add(key)
  108. logger.warning(
  109. "kanban notifier: subscription for %s on %s thread %s has no parent_chat_id anchor and matched no "
  110. "profile route; it will not be delivered. Re-subscribe with `hermes kanban notify-subscribe ... "
  111. "--parent-chat-id <channel id> [--guild-id <guild id>]`.",
  112. sub.get("task_id"), platform, sub.get("chat_id"),
  113. )
  114. _UNROUTABLE_WARNED: set[tuple] = set()
  115. def _warn_unroutable_sub_once(sub: dict, platform: Any, message: str, *extra_args: Any) -> None:
  116. """A routed subscription the credential gate fail-closes is a permanent dead-end: delivery
  117. rewinds every tick with only a DEBUG line. Say so ONCE per row at WARNING, mirroring
  118. ``_warn_anchorless_thread_sub_once`` (#115460)."""
  119. key = (sub.get("task_id"), platform, sub.get("chat_id"), sub.get("thread_id") or "")
  120. if key in _UNROUTABLE_WARNED:
  121. return
  122. _UNROUTABLE_WARNED.add(key)
  123. logger.warning(message, sub.get("task_id"), getattr(platform, "value", platform),
  124. sub.get("chat_id"), *extra_args)
  125. def _platform_names(mapping: Any) -> set[str]:
  126. """Lower-cased platform names of an adapters mapping (Platform enums or strings)."""
  127. return {getattr(platform, "value", str(platform)).lower() for platform in mapping}
  128. def _adapter_for_subscription(runner: Any, platform: Any, sub: dict, owner_profile: Optional[str]) -> Any:
  129. """Resolve a durable route without turning a missing secondary bot into primary authority."""
  130. adapter = runner._authorization_adapter(platform, owner_profile)
  131. config = getattr(runner, "config", None)
  132. if not getattr(config, "multiplex_profiles", False):
  133. return adapter
  134. primary = runner.adapters.get(platform)
  135. if adapter is not None and adapter is not primary:
  136. return adapter
  137. profile = owner_profile or getattr(runner, "_kanban_notifier_profile", None)
  138. primary_profile = getattr(runner, "_primary_profile_name", None) or runner._active_profile_name()
  139. profile = profile or primary_profile
  140. # A profile holding its OWN adapter for this platform is an independent credential boundary —
  141. # ``_authorization_adapter`` already answered for it, so the primary never stands in. Adapters
  142. # on OTHER platforms do not gate this one: the primary bot is the only credential serving the
  143. # pinned chat, for inbound turns and for these notifications alike (#115460).
  144. own_adapters = (getattr(runner, "_profile_adapters", {}) or {}).get(profile) or {}
  145. if getattr(platform, "value", str(platform)).lower() in _platform_names(own_adapters):
  146. return None
  147. metadata = sub.get("delivery_metadata") or {}
  148. guild = metadata.get("scope_id") or metadata.get("guild_id")
  149. parent = metadata.get("parent_chat_id")
  150. chat, thread = sub.get("chat_id"), sub.get("thread_id") or None
  151. user_id = sub.get("user_id") or None
  152. thread_like = bool(thread) or (sub.get("chat_type") or metadata.get("chat_type")) in {
  153. "thread", "forum", "forum_post", "forum-post", "topic",
  154. }
  155. # Preserve canonical route order, including equal-specificity ties. An older
  156. # row missing an anchor must not skip a potentially winning route. Reuse the
  157. # route's matcher (including platform-specific identity aliases), not a second
  158. # hand-maintained equality implementation.
  159. for route in getattr(config, "profile_routes", None) or []:
  160. if route.matches(platform.value, guild_id=guild, chat_id=chat,
  161. thread_id=thread, parent_chat_id=parent, user_id=user_id):
  162. if route.profile != profile:
  163. _warn_unroutable_sub_once(
  164. sub, platform,
  165. "kanban notifier: subscription for %s on %s chat %s is stamped with profile %s but a "
  166. "profile_routes entry pins that chat to profile %s; it will not be delivered. "
  167. "Re-subscribe with `hermes kanban notify-subscribe ... --notifier-profile %s`.",
  168. profile, route.profile, route.profile)
  169. return None
  170. from gateway.run import _multiplex_profile_homes
  171. served = {name for name, _home in _multiplex_profile_homes(config)}
  172. return primary if profile in served else None
  173. if route.matches(platform.value, guild_id=guild or route.guild_id, chat_id=chat,
  174. thread_id=thread, parent_chat_id=parent or (route.chat_id if thread_like else None),
  175. user_id=user_id or route.user_id):
  176. return None
  177. # A stateless (api_server) subscription carries a RAW session id, not a routable chat, so no
  178. # profile_routes entry can anchor it — and a platform-wide api_server route would deny the
  179. # default profile's own api_server destinations. The shared listener mirrors /p/<profile>/ for
  180. # every served profile, so the owner's own session store is the proof: authorize exactly the
  181. # session that lives in the served profile's state.db, never the platform. The default profile
  182. # keeps the historical fallthrough below (no store read).
  183. if profile != primary_profile and getattr(platform, "value", platform) == "api_server" \
  184. and session_owned_by_profile(config, profile, chat):
  185. return primary
  186. return primary if profile == primary_profile else None
  187. # --- Collection (runs in a worker thread) ---
  188. class _Collector:
  189. """One tick's claim state: which profiles/platforms this gateway serves and the GC gate."""
  190. def __init__(self, runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> None:
  191. self.runner = runner
  192. self.kb = kb
  193. self.notifier_profile = notifier_profile
  194. self.gc_due = gc_due
  195. self.gc_retention_days = gc_retention_days
  196. self.deliveries: list[dict] = []
  197. self.include_unowned = runner._owns_kanban_dispatcher_lock()
  198. self.profile_adapters = getattr(runner, "_profile_adapters", {})
  199. self.notifier_profiles = {notifier_profile}
  200. self.notifier_profiles.update(str(p).strip() for p in self.profile_adapters if str(p).strip())
  201. config = getattr(runner, "config", None)
  202. if getattr(config, "multiplex_profiles", False):
  203. self.notifier_profiles.update(
  204. route.profile for route in config.profile_routes
  205. if route.enabled and route.platform in _platform_names(runner.adapters)
  206. )
  207. # Include every platform any secondary profile has live. This is only a
  208. # coarse pre-filter; exact destination authorization runs before claim
  209. # and again at delivery, rewinding if the route or adapter changed.
  210. self.active_platforms = _platform_names(runner.adapters).union(
  211. *(_platform_names(m) for m in self.profile_adapters.values()))
  212. def collect(self) -> list[dict]:
  213. if not self.active_platforms:
  214. logger.debug("kanban notifier: no connected adapters; skipping tick")
  215. return self.deliveries
  216. # Poll each resolved DB path once: several slugs can map to one DB when
  217. # HERMES_KANBAN_DB pins the board path. The whole tick resolves pin-first:
  218. # on a dispatcher-pinned box each enumerated slug must map to the pinned
  219. # DB, never to that slug's own (empty) physical file.
  220. kb = self.kb
  221. with _pin_first():
  222. seen_db_paths: set[str] = set()
  223. for board_meta in _list_boards(kb):
  224. slug = board_meta.get("slug") or kb.DEFAULT_BOARD
  225. db_path = board_meta.get("db_path")
  226. try:
  227. resolved_db_path = str(Path(db_path).expanduser().resolve()) if db_path else str(kb.kanban_db_path(slug).resolve())
  228. except Exception:
  229. resolved_db_path = f"slug:{slug}"
  230. if resolved_db_path in seen_db_paths:
  231. logger.debug("kanban notifier: skipping duplicate board slug %s for DB %s", slug, resolved_db_path)
  232. continue
  233. seen_db_paths.add(resolved_db_path)
  234. self.collect_board(slug)
  235. return self.deliveries
  236. def _board_has_subs(self, slug: str) -> bool:
  237. """Cheap read-only probe before the writable connect() (schema init, WAL
  238. sidecars, checkpoints); a probe failure falls back to the writable open."""
  239. try:
  240. count = _kbn().count_notify_subs(
  241. board=slug, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned)
  242. except Exception as exc:
  243. logger.debug("kanban notifier: read-only subscription probe failed "
  244. "for board %s (%s); falling back to writable open", slug, exc)
  245. return True
  246. if count == 0:
  247. logger.debug("kanban notifier: board %s has no subscriptions owned by %s; skipping open",
  248. slug, sorted(self.notifier_profiles))
  249. return count != 0
  250. def _gc_stale_subs(self, conn: Any, slug: str) -> None:
  251. """Best-effort stale-sub sweep: a failed sweep never blocks delivery; the next hourly gate retries."""
  252. try:
  253. _purged = _kbn().purge_stale_done_notify_subs(conn, max_age_days=self.gc_retention_days)
  254. if _purged:
  255. logger.info("kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)",
  256. _purged, slug, self.gc_retention_days)
  257. except Exception as _gc_exc:
  258. logger.debug("kanban notifier: stale-sub GC failed for board %s: %s", slug, _gc_exc)
  259. def _claim_for_sub(self, conn: Any, slug: str, sub: dict) -> Optional[dict]:
  260. """Claim one subscription's unseen events; None when skipped or nothing new."""
  261. owner_profile = sub.get("notifier_profile") or None
  262. platform = (sub.get("platform") or "").lower()
  263. if platform not in self.active_platforms:
  264. logger.debug("kanban notifier: subscription for %s on %s skipped; adapter not connected",
  265. sub.get("task_id"), platform or "<missing>")
  266. return None
  267. from gateway.config import Platform
  268. if _adapter_for_subscription(self.runner, Platform(platform), sub, owner_profile or self.notifier_profile) is None:
  269. _warn_anchorless_thread_sub_once(sub, platform)
  270. return None
  271. old_cursor, cursor, events = _kbn().claim_unseen_events_for_sub(
  272. conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"],
  273. thread_id=sub.get("thread_id") or "", kinds=TERMINAL_KINDS,
  274. )
  275. if not events:
  276. return None
  277. task = self.kb.get_task(conn, sub["task_id"])
  278. logger.debug("kanban notifier: claimed %d event(s) for %s on board %s cursor %s→%s",
  279. len(events), sub["task_id"], slug, old_cursor, cursor)
  280. return {"sub": sub, "old_cursor": old_cursor, "cursor": cursor, "events": events, "task": task, "board": slug}
  281. def collect_board(self, slug: str) -> None:
  282. """Claim events on one board, appending delivery dicts to ``deliveries``."""
  283. if not self._board_has_subs(slug):
  284. return
  285. try:
  286. conn = _kbc().connect(board=slug)
  287. except Exception as exc:
  288. logger.debug("kanban notifier: cannot open board %s: %s", slug, exc)
  289. return
  290. try:
  291. if self.gc_due:
  292. self._gc_stale_subs(conn, slug)
  293. # No explicit init_db(): connect() already runs the migration once per
  294. # process, and init_db() would re-run it on a second connection racing
  295. # the first.
  296. subs = _kbn().list_notify_subs(conn, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned)
  297. if not subs:
  298. logger.debug("kanban notifier: board %s has no subscriptions", slug)
  299. for sub in subs:
  300. try:
  301. claimed = self._claim_for_sub(conn, slug, sub)
  302. if claimed is not None:
  303. self.deliveries.append(claimed)
  304. except Exception as sub_exc:
  305. # One bad subscription must not block the rest of the tick.
  306. logger.warning("kanban notifier: subscription for %s on board %s failed: %s",
  307. sub.get("task_id"), slug, sub_exc)
  308. finally:
  309. conn.close()
  310. def _notifier_collect(runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> list[dict]:
  311. """Claim unseen terminal events for every owned subscription on every board.
  312. Each gateway polls only subscriptions owned by profiles whose adapters it
  313. hosts; legacy rows without a profile stamp are visible only to the process
  314. holding the singleton dispatcher lock.
  315. """
  316. return _Collector(
  317. runner, kb, notifier_profile=notifier_profile, gc_due=gc_due, gc_retention_days=gc_retention_days,
  318. ).collect()
  319. # --- Per-event message formatting: kind -> (msg, wake_handoff, wake_review_detail) ---
  320. # ``None`` for handoff / review_detail leaves the accumulated wake value untouched.
  321. def _payload(ev: Any, key: str) -> Any:
  322. """Shared "payload present and truthy" read."""
  323. return ev.payload.get(key) if ev.payload and ev.payload.get(key) else None
  324. def _clip(ev: Any, key: str, msg_key: str, limit: int) -> str:
  325. """Catalog message ``msg_key`` (``{value}`` placeholder) rendered with the truncated payload
  326. value, or ``""`` when absent."""
  327. value = _payload(ev, key)
  328. return t(msg_key, value=str(value)[:limit]) if value else ""
  329. _NL = "\n{}"
  330. def _first_line(text: str, limit: int) -> str:
  331. lines = text.strip().splitlines()
  332. return lines[0][:limit] if lines else text[:limit]
  333. def _fmt_completed(ev, n) -> tuple:
  334. # Prefer the run summary from the event payload; fall back to task.result for legacy rows.
  335. wake_handoff = None
  336. payload_summary = _payload(ev, "summary")
  337. if payload_summary:
  338. wake_handoff = _first_line(str(payload_summary), 200)
  339. elif n.task and n.task.result:
  340. wake_handoff = _first_line(n.task.result, 160)
  341. handoff = f"\n{wake_handoff}" if wake_handoff is not None else ""
  342. return t("gateway.kanban.ping.completed", head=n.head, title=n.title, handoff=handoff), wake_handoff, None
  343. def _fmt_review_requested(ev, n) -> tuple:
  344. # Implementation done; task moved to the review lane. Carry the handoff
  345. # into the wake turn like ``completed`` so the reviewer needn't re-read the board.
  346. handoff = ""
  347. wake_handoff = None
  348. summary = _payload(ev, "summary")
  349. if summary:
  350. summary = str(summary)
  351. handoff = f"\n{summary[:200]}"
  352. wake_handoff = _first_line(summary, 200)
  353. return t("gateway.kanban.ping.review_requested", head=n.head, title=n.title, handoff=handoff), wake_handoff, None
  354. def _fmt_changes_requested(ev, n) -> tuple:
  355. payload = ev.payload or {}
  356. reason = _safe_review_reason(payload.get("reason"))
  357. reviewer = _safe_review_reason(payload.get("reviewer"), 48)
  358. implementer = _safe_review_reason(payload.get("implementer"), 48)
  359. reason_text = reason or t("gateway.kanban.ping.changes_default_reason")
  360. provenance = t("gateway.kanban.ping.reviewer_suffix", reviewer=reviewer) if reviewer else ""
  361. if implementer:
  362. provenance += t("gateway.kanban.ping.implementer_suffix", implementer=implementer)
  363. msg = t("gateway.kanban.ping.changes_requested",
  364. board_tag=n.board_tag, task_id=n.task_id, reason=reason_text, provenance=provenance)
  365. return msg, None, reason_text
  366. def _fmt_block_loop_detected(ev, n) -> tuple:
  367. """Re-blocked for the same cause past the limit and routed to `triage`.
  368. It emits no blocked/status event, so ping loudly here. A repeated-block
  369. circuit breaker establishes that orchestration attention is needed; it
  370. does NOT establish that a human decision or owner input exists. Use
  371. neutral orchestration wording unless the block was typed as a genuine
  372. owner-input request (`needs_input`, the only kind that carries a concrete
  373. question for the owner).
  374. """
  375. kind = _payload(ev, "kind")
  376. decision = kind == "needs_input"
  377. msg = t(
  378. "gateway.kanban.ping.triage", head=n.head,
  379. why=t("gateway.kanban.ping.triage_decision" if decision else "gateway.kanban.ping.triage_attention"),
  380. recurrences=_clip(ev, "recurrences", "gateway.kanban.ping.triage_recurrences", 200),
  381. reason=_clip(ev, "reason", "gateway.kanban.ping.reason_suffix", 160),
  382. )
  383. return msg, None, None
  384. def _fmt_gave_up(ev, n) -> tuple:
  385. # The dispatcher auto-blocked the task after ``failures`` consecutive non-success attempts
  386. # (spawn failure, crash, or timeout alike): it is now Blocked and waiting for a human.
  387. failures = _payload(ev, "failures")
  388. count = (t("gateway.kanban.ping.failed_n_times", count=int(failures)) if failures
  389. else t("gateway.kanban.ping.kept_failing"))
  390. last = _clip(ev, "error", "gateway.kanban.ping.last_error", 160)
  391. return t("gateway.kanban.ping.gave_up", head=n.head, count=count, last=last, task_id=n.task_id), None, None
  392. def _fmt_timed_out(ev, n) -> tuple:
  393. limit = int(_payload(ev, "limit_seconds") or 0)
  394. minutes = max(1, round(limit / 60)) if limit else 0
  395. span = t("gateway.kanban.ping.limit_minutes", minutes=minutes) if minutes else t("gateway.kanban.ping.limit_generic")
  396. return t("gateway.kanban.ping.timed_out", head=n.head, span=span), None, None
  397. # archived / unblocked are claimed (so the cursor advances past them) but
  398. # intentionally silent (no formatter), and excluded from _WAKE_KINDS so they
  399. # never wake the creator.
  400. _EVENT_FORMATTERS: dict[str, Callable[[Any, _KanbanNotification], tuple]] = {
  401. "completed": _fmt_completed,
  402. "blocked": lambda ev, n: (
  403. t("gateway.kanban.ping.blocked", head=n.head, reason=_clip(ev, "reason", "gateway.kanban.ping.reason_suffix", 160)),
  404. None, None,
  405. ),
  406. "gave_up": _fmt_gave_up,
  407. "crashed": lambda ev, n: (t("gateway.kanban.ping.crashed", head=n.head), None, None),
  408. "timed_out": _fmt_timed_out,
  409. "status": lambda ev, n: (t("gateway.kanban.ping.status", head=n.head, status=_payload(ev, "status") or ""), None, None),
  410. "review_requested": _fmt_review_requested,
  411. "changes_requested": _fmt_changes_requested,
  412. "block_loop_detected": _fmt_block_loop_detected,
  413. }
  414. # --- Delivery of one claimed batch (one subscription, N events) ---
  415. class _KanbanNotification:
  416. """Deliver one subscription's claimed events, then settle the cursor.
  417. Both legs of notify+wake must succeed before settling. Sent pings have a
  418. separate durable checkpoint so wake retries do not resend them. Admission
  419. is at-least-once queueing, not an execution or final-response receipt.
  420. """
  421. def __init__(self, runner: Any, d: dict, *, platform_cls: Any, sub_fail_counts: dict) -> None:
  422. self.runner = runner
  423. self.d = d
  424. self.platform_cls = platform_cls
  425. self.sub_fail_counts = sub_fail_counts
  426. self.sub = sub = d["sub"]
  427. self.task = task = d["task"]
  428. self.board_slug = d.get("board")
  429. self.platform_str = (sub["platform"] or "").lower()
  430. self.task_id = sub["task_id"]
  431. self.sub_profile = sub.get("notifier_profile") or ""
  432. self.title = (task.title if task else sub["task_id"])[:120]
  433. self.board_tag = t("gateway.kanban.ping.board_tag", board=self.board_slug) if self.board_slug else ""
  434. # Attribute the ping to the worker that did the work.
  435. tag = t("gateway.kanban.ping.assignee_tag", assignee=task.assignee) if task and task.assignee else ""
  436. self.head = t("gateway.kanban.ping.head", board_tag=self.board_tag, assignee_tag=tag, task_id=self.task_id)
  437. # The wake self-post path needs the key even when every event was skipped.
  438. self.sub_key = (sub["task_id"], sub["platform"], sub["chat_id"], sub.get("thread_id") or "")
  439. mode = sub.get("delivery_mode") or "notify"
  440. self.wake_agent = mode in ("notify+wake", "wake")
  441. self.send_passive = mode != "wake"
  442. # Worker handoff carried into the synthetic wake turn so the woken
  443. # creator doesn't re-decompose work already on the board.
  444. self.wake_handoff = self.wake_review_detail = self.session_key = self.synth = ""
  445. self.plat: Any = None
  446. self.adapter: Any = None
  447. self.is_push_adapter = True
  448. self.wake_kinds: set = set()
  449. # -- cursor / subscription ops (blocking, run in a fresh-context thread) --
  450. async def rewind(self) -> None:
  451. await _to_thread_process_service(
  452. self.runner._kanban_rewind, self.sub, self.d["cursor"], self.d.get("old_cursor", 0), self.board_slug,
  453. )
  454. async def advance(self) -> None:
  455. await _to_thread_process_service(self.runner._kanban_advance, self.sub, self.d["cursor"], self.board_slug)
  456. async def unsub(self) -> None:
  457. await _to_thread_process_service(self.runner._kanban_unsub, self.sub, self.board_slug)
  458. def clear_failures(self) -> None:
  459. self.sub_fail_counts.pop(self.sub_key, None)
  460. async def delivery_failed(self, fmt: str, prefix: tuple, drop_fmt: str, exc: Exception, exc_info: bool) -> None:
  461. """Bump the failure counter; drop the sub past the limit, else rewind the claim so the next tick retries."""
  462. fails = self.sub_fail_counts.get(self.sub_key, 0) + 1
  463. self.sub_fail_counts[self.sub_key] = fails
  464. logger.warning(fmt, *prefix, fails, MAX_SEND_FAILURES, exc, exc_info=exc_info)
  465. if fails >= MAX_SEND_FAILURES:
  466. logger.warning(drop_fmt, self.task_id, self.platform_str, fails)
  467. await self.unsub()
  468. self.clear_failures()
  469. else:
  470. await self.rewind()
  471. async def _wake_failed(self, fmt: str, exc: Exception) -> None:
  472. drop_fmt = "kanban notifier: dropping subscription %s on %s after %d consecutive wake failures"
  473. await self.delivery_failed(fmt, (self.task_id,), drop_fmt, exc, True)
  474. # -- formatting --
  475. def format_event(self, ev: Any) -> Optional[str]:
  476. """Render one event; accumulates wake handoff/review detail. None → silent kind."""
  477. formatter = _EVENT_FORMATTERS.get(ev.kind)
  478. if formatter is None:
  479. return None
  480. msg, handoff, review_detail = formatter(ev, self)
  481. if handoff is not None:
  482. self.wake_handoff = handoff
  483. if review_detail is not None:
  484. self.wake_review_detail = review_detail
  485. return msg
  486. def build_wake_text(self) -> None:
  487. """Set ``wake_kinds`` / ``session_key`` / ``synth`` for the wake paths."""
  488. task, sub = self.task, self.sub
  489. self.wake_kinds = {ev.kind for ev in self.d["events"] if ev.kind in _WAKE_KINDS} if self.wake_agent else set()
  490. self.wake_diagnostic = all(diagnostic_event(ev) for ev in self.d["events"] if ev.kind in self.wake_kinds)
  491. if not self.wake_kinds:
  492. return
  493. if self.is_push_adapter:
  494. self.session_key = getattr(task, "session_id", None) or ""
  495. else:
  496. # Non-push wakes target sub["chat_id"] (the raw session id the
  497. # subscriber registered). task.session_id may be a WORKER session
  498. # for child tasks; use it only for legacy rows.
  499. self.session_key = sub["chat_id"] or getattr(task, "session_id", None) or ""
  500. # i18n keys: gateway.kanban.wake.<kind> for each _WAKE_KINDS entry.
  501. _parts = [t(f"gateway.kanban.wake.{k}") for k in _WAKE_KINDS if k in self.wake_kinds]
  502. _status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
  503. synth = t(
  504. "gateway.kanban.wake.message",
  505. task_id=sub["task_id"], status=_status, title=self.title,
  506. assignee=task.assignee if task else "", board=self.board_slug,
  507. )
  508. # Label as an automatic notification and carry the handoff so the
  509. # creator inspects the board instead of re-decomposing.
  510. if self.wake_handoff:
  511. synth += "\n" + t("gateway.kanban.wake.handoff", summary=self.wake_handoff)
  512. if self.wake_review_detail:
  513. synth += "\n" + t("gateway.kanban.wake.review_detail", reason=self.wake_review_detail)
  514. self.synth = synth + "\n\n" + t("gateway.kanban.wake.guidance")
  515. def _log_woke(self) -> None:
  516. logger.info("kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
  517. self.task_id, self.platform_str, self.sub["chat_id"], self.sub_profile or "default", self.wake_kinds)
  518. def _served_wake_profile(self) -> Optional[str]:
  519. """The subscription's profile when THIS gateway is a multiplexer serving it, else ``None``.
  520. ``None`` keeps the historical path: a standalone ``hermes -p <name>`` gateway owns its own
  521. listener and key, so its api_server wakes keep using the HTTP self-post.
  522. """
  523. if not self.sub_profile:
  524. return None
  525. if not getattr(getattr(self.runner, "config", None), "multiplex_profiles", False):
  526. return None
  527. return self.sub_profile
  528. def _owner_scope(self):
  529. """Runtime scope of the subscription's profile under multiplex, else a no-op context."""
  530. runner = self.runner
  531. served_profile = self._served_wake_profile()
  532. if not served_profile:
  533. return contextlib.nullcontext()
  534. from gateway.run import _async_profile_runtime_scope
  535. from gateway.session import SessionSource
  536. source = SessionSource(platform=self.plat, chat_id=self.sub["chat_id"], profile=served_profile)
  537. return _async_profile_runtime_scope(runner._resolve_profile_home_for_source(source))
  538. async def wake(self) -> None:
  539. """Wake the creator session (raises on failure): push adapters get a full SessionSource, non-push a raw self-post."""
  540. from gateway.wake import deliver_wake
  541. sub = self.sub
  542. if not self.is_push_adapter:
  543. # A served profile's raw-session wake runs in THAT profile's scope, in-process: the
  544. # shared listener's /p/<profile>/ self-post would need the profile's own
  545. # API_SERVER_KEY, which a route-only profile legitimately does not have, and an
  546. # unprefixed self-post would resume the session in the DEFAULT profile's store.
  547. async with self._owner_scope():
  548. await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key,
  549. profile=self._served_wake_profile(),
  550. notification_category="diagnostic" if self.wake_diagnostic else "result")
  551. self._log_woke()
  552. return
  553. from gateway.session import SessionSource
  554. # Rebuild the creator's real session scope from the persisted chat_type:
  555. # build_session_key() keys DMs differently from group/thread, so a
  556. # hardcoded "group" mis-routed DM/thread creators into a fresh session.
  557. # Legacy rows may carry chat_type in delivery_metadata; last resort is
  558. # "group". A mismatch only degrades to a fresh session.
  559. # Legacy rows written before the column existed may still carry chat_type in delivery_metadata
  560. # (#60600 rows) — fall back to that, then to "group" (the historical default that suits the
  561. # dashboard/group flows). handle_message() get_or_create_session's the target, so a mismatch only
  562. # ever degrades to a fresh session, never an exception.
  563. _delivery_meta = sub.get("delivery_metadata") or {}
  564. _chat_type = str(sub.get("chat_type") or _delivery_meta.get("chat_type") or "").strip()
  565. _source = SessionSource(
  566. platform=self.plat, chat_id=sub["chat_id"], chat_type=_chat_type or "group",
  567. thread_id=sub.get("thread_id") or None, user_id=sub.get("user_id"), user_id_alt=sub.get("user_id_alt"),
  568. profile=self.sub_profile or None, scope_id=_wake_scope_id(self.adapter, sub),
  569. parent_chat_id=_delivery_meta.get("parent_chat_id"),
  570. )
  571. _source._transport_adapter_ref = weakref.ref(self.adapter)
  572. from gateway.run import _async_profile_runtime_scope
  573. if self.sub_profile and getattr(getattr(self.runner, "config", None), "multiplex_profiles", False):
  574. from hermes_cli.profiles import profile_exists
  575. if not profile_exists(self.sub_profile):
  576. raise RuntimeError(f"Kanban wake profile {self.sub_profile!r} no longer exists")
  577. async with _async_profile_runtime_scope(self.runner._resolve_profile_home_for_source(_source)):
  578. await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key, source=_source,
  579. notification_category="diagnostic" if self.wake_diagnostic else "result")
  580. self._log_woke()
  581. async def _send_event(self, ev: Any, msg: str) -> bool:
  582. """Send one text ping; raises on adapter exception or SendResult(success=False)."""
  583. from gateway.warning_notifications import present_notification
  584. sub, adapter = self.sub, self.adapter
  585. delivery_metadata = sub.get("delivery_metadata")
  586. metadata: dict[str, Any] = dict(delivery_metadata) if isinstance(delivery_metadata, dict) else {}
  587. if sub.get("thread_id") and not metadata.get("thread_id"):
  588. metadata["thread_id"] = sub["thread_id"]
  589. _send_res = None
  590. async def send_ping():
  591. nonlocal _send_res
  592. _send_res = await adapter.send(sub["chat_id"], msg, metadata=metadata)
  593. if not await present_notification(send_ping, platform=self.platform_str, diagnostic=diagnostic_event(ev)):
  594. return False
  595. # SendResult(success=False) without an exception is a FAILED delivery
  596. # (else the event is lost); None / non-SendResult keeps the
  597. # "no exception == delivered" contract.
  598. if getattr(_send_res, "success", True) is False:
  599. raise RuntimeError(f"adapter send() reported failure: {getattr(_send_res, 'error', None) or 'unknown error'}")
  600. logger.debug("kanban notifier: delivered %s event for %s to %s/%s on board %s",
  601. ev.kind, self.task_id, self.platform_str, sub["chat_id"], self.board_slug)
  602. # Upload artifact paths from the handoff payload / legacy result as
  603. # native files. Both handoff kinds stage files for exactly this: a
  604. # review-bound card's files exist precisely so the human sees them at
  605. # handoff time. Retry exposure matches ``completed`` (the sub cursor is
  606. # rewound only when a send failed).
  607. if ev.kind in ("completed", "review_requested"):
  608. try:
  609. await self.runner._deliver_kanban_artifacts(
  610. adapter=adapter, chat_id=sub["chat_id"], metadata=metadata,
  611. event_payload=getattr(ev, "payload", None), task=self.task,
  612. )
  613. except Exception as art_exc:
  614. logger.debug("kanban notifier: artifact delivery for %s failed: %s", self.task_id, art_exc)
  615. return True
  616. async def _send_pings(self) -> bool:
  617. """Send every text ping; False when a send failed (claim already rewound/dropped)."""
  618. for ev in self.d["events"]:
  619. msg = self.format_event(ev)
  620. if msg is None:
  621. continue
  622. # Non-push adapters (api_server) always report SendResult(success=False)
  623. # from send(); treating that as failure would drop the sub forever and
  624. # make the wake path unreachable. Skip the doomed send; the self-post
  625. # IS the delivery and resolves the failure counter.
  626. if not self.is_push_adapter and self.wake_agent:
  627. logger.debug(
  628. "kanban notifier: adapter %s has no push channel; skipping text ping for %s, relying "
  629. "on wake self-post instead", self.platform_str, self.task_id,
  630. )
  631. continue
  632. if not self.send_passive:
  633. # Wake-only: the wake path is the sole delivery and resolves the counter.
  634. continue
  635. if ev.id <= self.sub.get("last_ping_event_id", 0):
  636. continue
  637. try:
  638. if await self._send_event(ev, msg) is False:
  639. continue
  640. await _to_thread_process_service(partial(
  641. self.runner._kanban_sub_op, self.board_slug, "record_notify_ping", self.sub,
  642. event_id=ev.id,
  643. ))
  644. self.clear_failures()
  645. except Exception as exc:
  646. await self.delivery_failed(
  647. "kanban notifier: send failed for %s on %s (attempt %d/%d): %s", (self.task_id, self.platform_str),
  648. "kanban notifier: dropping subscription %s on %s after %d consecutive send failures", exc, False,
  649. )
  650. return False
  651. return True
  652. async def deliver(self) -> None:
  653. try:
  654. self.plat = self.platform_cls(self.platform_str)
  655. except ValueError:
  656. await self.advance()
  657. return
  658. # Recheck the exact route after claiming: config/adapters can change between ticks. The
  659. # recheck reads the served profile's session store for a stateless destination, so it runs
  660. # off the event loop (the claim path already collects in a worker thread).
  661. adapter = await asyncio.to_thread(
  662. _adapter_for_subscription, self.runner, self.plat, self.sub, self.sub_profile or None)
  663. if adapter is None:
  664. logger.debug("kanban notifier: adapter %s disconnected before delivery for %s; rewinding claim",
  665. self.platform_str, self.task_id)
  666. await self.rewind()
  667. return
  668. self.adapter = adapter
  669. from gateway.wake import adapter_supports_push
  670. self.is_push_adapter = adapter_supports_push(adapter)
  671. # Pings, artifact uploads (media policy) and the wake text (display.language) all read the
  672. # SUBSCRIBER profile's config; the notifier thread itself runs in the launch profile's scope.
  673. async with self._owner_scope():
  674. if not await self._send_pings():
  675. return
  676. # All text pings delivered (or skipped for non-push / wake-only).
  677. original_events = self.d["events"]
  678. from gateway.warning_notifications import warning_notifications_enabled
  679. split = not warning_notifications_enabled(self.platform_str)
  680. wake_groups = ([original_events] if not split else [
  681. [ev for ev in original_events if diagnostic_event(ev)],
  682. [ev for ev in original_events if not diagnostic_event(ev)],
  683. ])
  684. wake_payloads = []
  685. for events in wake_groups:
  686. if not events:
  687. continue
  688. self.d = {**self.d, "events": events}
  689. self.wake_handoff = self.wake_review_detail = ""
  690. for ev in events:
  691. self.format_event(ev)
  692. self.build_wake_text()
  693. if self.wake_kinds:
  694. wake_payloads.append((self.synth, self.wake_diagnostic, self.wake_kinds))
  695. self.d = {**self.d, "events": original_events}
  696. is_push = self.is_push_adapter
  697. from gateway.wake import WakeNotAccepted
  698. # A requested wake is required even when its passive ping already landed.
  699. if wake_payloads:
  700. try:
  701. for self.synth, self.wake_diagnostic, self.wake_kinds in wake_payloads:
  702. await self.wake()
  703. self.clear_failures()
  704. except WakeNotAccepted:
  705. # Startup / full queue is not a dead destination. Keep the durable
  706. # subscription alive regardless of how long admission takes.
  707. await self.rewind()
  708. return
  709. except Exception as _wk_err:
  710. await self._wake_failed(
  711. "kanban notifier: wake-only delivery failed for %s (attempt %d/%d): %s" if is_push
  712. else "kanban notifier: wake self-post failed for %s (attempt %d/%d): %s",
  713. _wk_err,
  714. )
  715. return
  716. # Delivery complete: advance the cursor (the dedup mechanism).
  717. await self.advance()
  718. if not is_push:
  719. self.clear_failures()
  720. # Unsubscribe only on archive; ``done`` is reversible.
  721. if self.task and self.task.status == "archived":
  722. await self.unsub()