kanban_diagnostics.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910
  1. """Kanban diagnostics — structured, actionable distress signals for tasks.
  2. A ``Diagnostic`` carries a **kind** (canonical code the UI/tests match on), a
  3. **severity**, title/detail text, and **actions** the dashboard renders as
  4. buttons and the CLI as hints. Rules are stateless and read-only over
  5. (task, events, runs, optional graph); callers compute on demand. Only
  6. operator-fixable signals (not a one-off provider 502); every diagnostic has a
  7. recovery action and auto-clears when the failure mode resolves.
  8. """
  9. from __future__ import annotations
  10. from dataclasses import asdict, dataclass, field
  11. from typing import Any, Callable, Iterable, Optional
  12. import json
  13. import time
  14. # Least → most urgent; sorted outputs put critical first.
  15. SEVERITY_ORDER = ("warning", "error", "critical")
  16. def severity_at_or_above(severity: Optional[str], threshold: Optional[str]) -> bool:
  17. """Return True when ``severity`` meets or exceeds ``threshold``."""
  18. if threshold is None:
  19. return True
  20. if severity not in SEVERITY_ORDER or threshold not in SEVERITY_ORDER:
  21. return False
  22. return SEVERITY_ORDER.index(severity) >= SEVERITY_ORDER.index(threshold)
  23. @dataclass
  24. class DiagnosticAction:
  25. """A recovery action. ``kind`` drives rendering: ``reclaim``/``reassign``
  26. POST to /tasks/:id/*; ``unblock`` PATCHes status to ready; ``cli_hint``
  27. shows ``payload.command``; ``open_docs`` links ``payload.url``; ``comment``
  28. nudges the operator. ``suggested=True`` = recommended first step."""
  29. kind: str
  30. label: str
  31. payload: dict = field(default_factory=dict)
  32. suggested: bool = False
  33. def to_dict(self) -> dict:
  34. return asdict(self)
  35. @dataclass
  36. class Diagnostic:
  37. """One active distress signal on a task."""
  38. kind: str
  39. severity: str # "warning" | "error" | "critical"
  40. title: str
  41. detail: str
  42. actions: list[DiagnosticAction] = field(default_factory=list)
  43. first_seen_at: int = 0
  44. last_seen_at: int = 0
  45. count: int = 1
  46. run_id: Optional[int] = None # None = task-wide
  47. data: dict = field(default_factory=dict) # structured payload for the UI
  48. def to_dict(self) -> dict:
  49. return asdict(self)
  50. # --- Rule helpers ---
  51. def _task_field(task, name, default=None):
  52. """Read a field from a sqlite3.Row, a kanban_db.Task dataclass, or a dict."""
  53. if task is None:
  54. return default
  55. try:
  56. if hasattr(task, "keys") and name in task.keys():
  57. return task[name]
  58. except Exception:
  59. pass
  60. if isinstance(task, dict):
  61. return task.get(name, default)
  62. return getattr(task, name, default)
  63. def _parse_payload(ev) -> dict:
  64. """Tolerate event.payload being either a dict or a JSON string."""
  65. p = _task_field(ev, "payload", None)
  66. if isinstance(p, dict):
  67. return p
  68. if isinstance(p, str):
  69. try:
  70. return json.loads(p) or {}
  71. except Exception:
  72. return {}
  73. return {}
  74. def _event_kind(ev) -> str:
  75. return _task_field(ev, "kind", "") or ""
  76. def _event_ts(ev) -> int:
  77. return int(_task_field(ev, "created_at", 0) or 0)
  78. def _first_field(task, primary: str, legacy: str, default=None):
  79. """``task[primary]`` unless it is None, else ``task[legacy]`` (old DB rows)."""
  80. v = _task_field(task, primary, None)
  81. return v if v is not None else _task_field(task, legacy, default)
  82. def _latest_event_ts(events: Iterable[Any], kinds: set[str]) -> int:
  83. """Max ``created_at`` over events whose kind is in ``kinds`` (0 if none)."""
  84. return max([0, *(_event_ts(ev) for ev in events if _event_kind(ev) in kinds)])
  85. def _latest_gave_up_is_terminal_provider(events: Iterable[Any]) -> bool:
  86. """True when the most recent breaker trip was a terminal provider error (credential
  87. revoked, model gone) and nothing has resumed the task since."""
  88. for ev in reversed(list(events)):
  89. kind = _event_kind(ev)
  90. if kind == "gave_up":
  91. return bool(_parse_payload(ev).get("terminal_provider"))
  92. if kind in {"unblocked", "promoted", "completed", "claimed"}:
  93. return False
  94. return False
  95. def _cli_hint(label: str, command: str, *, suggested: bool = False) -> DiagnosticAction:
  96. return DiagnosticAction(kind="cli_hint", label=label, payload={"command": command},
  97. suggested=suggested)
  98. def _log_hint_action(task_id: str) -> DiagnosticAction:
  99. cmd = f"hermes kanban log {task_id}"
  100. return _cli_hint(f"Check logs: {cmd}", cmd, suggested=True)
  101. def _error_snippet(last_err) -> str:
  102. """First 500 chars of the error (with ellipsis), or "" when absent."""
  103. err_text = (last_err or "").strip() if last_err else ""
  104. return err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
  105. def _active_hallucination_events(events: Iterable[Any], kind: str) -> list[Any]:
  106. """Events of ``kind`` with no ``completed``/``edited`` event strictly after
  107. them. Requires id-sorted (arrival-order) input, which the DB provides."""
  108. active: list[Any] = []
  109. for ev in events:
  110. k = _event_kind(ev)
  111. if k in {"completed", "edited"}:
  112. active.clear()
  113. elif k == kind:
  114. active.append(ev)
  115. return active
  116. def _unique_payload_ids(hits: list[Any], key: str) -> list[str]:
  117. """Ordered, de-duplicated ``payload[key]`` entries across ``hits``."""
  118. out: list[str] = []
  119. for ev in hits:
  120. for pid in _parse_payload(ev).get(key, []) or []:
  121. if pid not in out:
  122. out.append(pid)
  123. return out
  124. def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAction]:
  125. """Baseline recovery primitives every diagnostic can fall back on."""
  126. out: list[DiagnosticAction] = []
  127. if running:
  128. out.append(DiagnosticAction(kind="reclaim", label="Reclaim task", payload={}))
  129. out.append(DiagnosticAction(
  130. kind="reassign", label="Reassign to different profile", payload={"reclaim_first": running},
  131. ))
  132. return out
  133. def _is_running(task) -> bool:
  134. return _task_field(task, "status") == "running"
  135. def _runs_newest_first(runs) -> list[Any]:
  136. # reversed(sorted()) not sorted(reverse=True): equal ids must keep the
  137. # last-listed run first.
  138. return list(reversed(sorted(runs, key=lambda r: _task_field(r, "id", 0))))
  139. # --- Rule implementations ---
  140. # Each rule: (task, events, runs, now_ts, config) -> list[Diagnostic].
  141. # ``events``/``runs`` are kanban_db rows/dataclasses or same-shaped dicts.
  142. RuleFn = Callable[[Any, list[Any], list[Any], int, dict], list[Diagnostic]]
  143. def _aux_slot_explicit(slot: Any) -> bool:
  144. """True if the aux slot was user-configured: provider other than "auto",
  145. or any of model/base_url/api_key set (the default falls through to the
  146. main model)."""
  147. if not isinstance(slot, dict):
  148. return False
  149. provider = str(slot.get("provider") or "").strip().lower()
  150. if provider and provider != "auto":
  151. return True
  152. return any(str(slot.get(key) or "").strip() for key in ("model", "base_url", "api_key"))
  153. def _main_model_visible(raw_config: Any) -> bool:
  154. """Best-effort "a main model is configured" from the raw config dict (the
  155. dashboard process may not share CLI runtime state). Unprovable => False,
  156. which errs toward NOT firing the diagnostic."""
  157. if not isinstance(raw_config, dict):
  158. return False
  159. model_cfg = raw_config.get("model")
  160. if isinstance(model_cfg, dict):
  161. provider = str(model_cfg.get("provider") or "").strip()
  162. model = str(
  163. model_cfg.get("default") or model_cfg.get("model") or model_cfg.get("name") or ""
  164. ).strip()
  165. return bool(provider and model)
  166. return bool(str(model_cfg or "").strip())
  167. def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
  168. """Report whether the triage aux paths look configured: ``{auto_decompose,
  169. decomposer_explicit, specifier_explicit, main_model_visible}``. ``None``
  170. when no config context is present (keeps low-level callers/tests silent)."""
  171. if not isinstance(config, dict):
  172. return None
  173. explicit = config.get("triage_aux_status")
  174. if isinstance(explicit, dict):
  175. return explicit
  176. aux = config.get("auxiliary")
  177. kanban_cfg = config.get("kanban") if isinstance(config.get("kanban"), dict) else {}
  178. # No auxiliary/kanban/model keys at all => a low-level caller passing {}.
  179. if not isinstance(aux, dict) and not kanban_cfg and "model" not in config:
  180. return None
  181. aux = aux if isinstance(aux, dict) else {}
  182. return {
  183. # ``auto_decompose`` defaults to True per kanban DEFAULT_CONFIG.
  184. "auto_decompose": bool(kanban_cfg["auto_decompose"]) if "auto_decompose" in kanban_cfg else True,
  185. "decomposer_explicit": _aux_slot_explicit(aux.get("kanban_decomposer")),
  186. "specifier_explicit": _aux_slot_explicit(aux.get("triage_specifier")),
  187. "main_model_visible": _main_model_visible(config),
  188. }
  189. def _positive_int(value: Any, default: int) -> int:
  190. try:
  191. parsed = int(value)
  192. except (TypeError, ValueError):
  193. return default
  194. return parsed if parsed >= 1 else default
  195. def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
  196. """A worker's kanban_complete named created_cards that don't exist / weren't
  197. its own; the completion was blocked. Clears on a later completion/edit."""
  198. hits = _active_hallucination_events(events, "completion_blocked_hallucination")
  199. if not hits:
  200. return []
  201. actions = [DiagnosticAction(kind="comment", label="Add a comment explaining what to do",
  202. suggested=False)]
  203. actions += _generic_recovery_actions(task, running=_is_running(task))
  204. return [Diagnostic(
  205. kind="hallucinated_cards", severity="error",
  206. title="Worker claimed cards that don't exist",
  207. detail="The completing worker declared created_cards that either didn't exist or weren't "
  208. "created by its profile. The completion was blocked and the task stayed in its prior "
  209. "state. Usually means the worker hallucinated ids instead of capturing return values "
  210. "from kanban_create.",
  211. actions=actions,
  212. first_seen_at=_event_ts(hits[0]), last_seen_at=_event_ts(hits[-1]), count=len(hits),
  213. data={"phantom_ids": _unique_payload_ids(hits, "phantom_cards")},
  214. )]
  215. # (primary_slot, fallback_slot, primary_desc, detail_path) keyed by auto_decompose.
  216. _TRIAGE_SLOTS = {
  217. True: (
  218. "auxiliary.kanban_decomposer", "auxiliary.triage_specifier", "decomposer",
  219. "Auto-decompose is on, so the dispatcher needs auxiliary.kanban_decomposer (with "
  220. "auxiliary.triage_specifier as a fallback for non-fan-out tasks).",
  221. ),
  222. False: (
  223. "auxiliary.triage_specifier", "auxiliary.kanban_decomposer", "specifier",
  224. "Auto-decompose is off, so triage tasks need "
  225. "`hermes kanban specify`, which uses auxiliary.triage_specifier.",
  226. ),
  227. }
  228. def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnostic]:
  229. """A triage task can't leave triage without a usable aux model. With
  230. auto-decompose on the primary slot is ``auxiliary.kanban_decomposer``
  231. (specifier as fallback); off, it is ``auxiliary.triage_specifier``. The
  232. default ``provider: auto`` falls back to the main model, so this fires only
  233. when the slot isn't explicit AND no main model is visible. Requires config
  234. context ({} keeps it silent)."""
  235. if _task_field(task, "status") != "triage":
  236. return []
  237. status = triage_aux_status(cfg)
  238. if status is None:
  239. return []
  240. auto_decompose = bool(status.get("auto_decompose"))
  241. main_visible = bool(status.get("main_model_visible"))
  242. decomposer_explicit = bool(status.get("decomposer_explicit"))
  243. specifier_explicit = bool(status.get("specifier_explicit"))
  244. primary_slot, fallback_slot, primary_desc, detail_path = _TRIAGE_SLOTS[auto_decompose]
  245. primary_explicit, fallback_explicit = (
  246. (decomposer_explicit, specifier_explicit) if auto_decompose
  247. else (specifier_explicit, decomposer_explicit)
  248. )
  249. if primary_explicit or main_visible:
  250. return []
  251. task_id = _task_field(task, "id") or "<task_id>"
  252. actions = [_cli_hint(
  253. f"Configure {primary_slot}", f"hermes config set {primary_slot}.provider auto", suggested=True,
  254. )]
  255. if not fallback_explicit and not main_visible:
  256. actions.append(_cli_hint(
  257. f"Or configure fallback {fallback_slot}", f"hermes config set {fallback_slot}.provider auto",
  258. ))
  259. if not auto_decompose:
  260. cmd = f"hermes kanban specify {task_id}"
  261. actions.append(_cli_hint(f"Specify manually: {cmd}", cmd))
  262. return [Diagnostic(
  263. kind="triage_aux_unavailable", severity="warning",
  264. title=f"Triage {primary_desc} has no usable model",
  265. detail=f"This task is still in triage and no working auxiliary model is visible to the "
  266. f"dispatcher. {detail_path} The default slot uses `provider: auto` which falls back to "
  267. f"the main model, but no main model is configured either. Configure the slot directly "
  268. f"or set a main model so the auto fallback can take over.",
  269. actions=actions,
  270. first_seen_at=now, last_seen_at=now, count=1,
  271. data={"task_id": task_id, "auto_decompose": auto_decompose,
  272. "primary_slot": primary_slot, "main_model_visible": main_visible},
  273. )]
  274. def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
  275. """Advisory: the completion summary mentions ``t_<hex>`` ids that don't
  276. resolve. Warning only; clears on a later clean completion."""
  277. hits = _active_hallucination_events(events, "suspected_hallucinated_references")
  278. if not hits:
  279. return []
  280. return [Diagnostic(
  281. kind="prose_phantom_refs", severity="warning",
  282. title="Completion summary references unknown task ids",
  283. detail="The completion summary mentions task ids that don't resolve in this board's database. "
  284. "The completion itself succeeded, but downstream consumers parsing the summary may be "
  285. "pointed at cards that never existed.",
  286. actions=_generic_recovery_actions(task, running=_is_running(task)),
  287. first_seen_at=_event_ts(hits[0]), last_seen_at=_event_ts(hits[-1]), count=len(hits),
  288. data={"phantom_refs": _unique_payload_ids(hits, "phantom_refs")},
  289. )]
  290. def _failure_threshold(cfg: dict) -> Any:
  291. """``failure_threshold`` with the legacy ``spawn_failure_threshold`` alias."""
  292. return cfg.get("failure_threshold", cfg.get("spawn_failure_threshold", 3))
  293. _OUTCOME_LABELS = {"spawn_failed": "spawn", "timed_out": "timeout", "crashed": "crash"}
  294. def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
  295. """``consecutive_failures`` >= cfg["failure_threshold"] (legacy key
  296. ``spawn_failure_threshold``), regardless of failure mode — the kernel keeps
  297. retrying and the operator must intervene. Runtime callers derive the
  298. threshold from ``kanban.failure_limit`` so it doesn't lag the breaker.
  299. Exempt: done/archived (a manual done ends no run, so the streak is history)
  300. and running (a retry in flight must not read as a current failure; re-fires
  301. if it fails too)."""
  302. if _task_field(task, "status") in ("done", "archived", "running"):
  303. return []
  304. threshold = _positive_int(_failure_threshold(cfg), 3)
  305. failure_limit = _positive_int(cfg.get("failure_limit"), threshold)
  306. failures = _first_field(task, "consecutive_failures", "spawn_failures", 0)
  307. # A terminal provider error (credential revoked, model gone) blocks the card after ONE
  308. # attempt, below any threshold; it still needs an operator, so diagnose it now.
  309. terminal_trip = _latest_gave_up_is_terminal_provider(events)
  310. if not terminal_trip and (failures is None or failures < threshold):
  311. return []
  312. failures = failures or 0
  313. last_err = _first_field(task, "last_failure_error", "last_spawn_error")
  314. assignee = _task_field(task, "assignee")
  315. # Most recent failure outcome makes the title/action specific.
  316. most_recent_outcome = next(
  317. (oc for oc in (_task_field(r, "outcome") for r in _runs_newest_first(runs))
  318. if oc in {"spawn_failed", "timed_out", "crashed"}),
  319. None,
  320. )
  321. actions: list[DiagnosticAction] = []
  322. if most_recent_outcome == "spawn_failed" and assignee and assignee != "default":
  323. # Spawn is failing specifically — profile setup issue.
  324. doctor, auth = f"hermes -p {assignee} doctor", f"hermes -p {assignee} auth"
  325. actions.append(_cli_hint(f"Verify profile: {doctor}", doctor, suggested=True))
  326. actions.append(_cli_hint(f"Fix profile auth: {auth}", auth))
  327. elif most_recent_outcome in {"timed_out", "crashed"}:
  328. # Worker got off the ground but died: logs diagnose, reclaim/reassign recover.
  329. task_id = _task_field(task, "id")
  330. if task_id:
  331. actions.append(_log_hint_action(task_id))
  332. actions.extend(_generic_recovery_actions(task, running=_is_running(task)))
  333. severity = "critical" if failures >= threshold * 2 else "error"
  334. err_snippet = _error_snippet(last_err)
  335. outcome_label = _OUTCOME_LABELS.get(most_recent_outcome or "", "failure")
  336. if terminal_trip:
  337. title = "Provider rejected this profile's credential or model — blocked after one attempt"
  338. detail = (
  339. f"The worker's provider call failed with an error a retry cannot fix (revoked or invalid "
  340. f"API key, model not found), so the dispatcher blocked the task instead of spending the "
  341. f"{failure_limit}-attempt retry budget on it. Full last error:\n\n{err_snippet}\n\n"
  342. f"Fix the assignee profile's provider credentials/model, then unblock the task."
  343. )
  344. elif err_snippet:
  345. title = f"Agent {outcome_label} x{failures}: {err_snippet.splitlines()[0][:160]}"
  346. detail = (
  347. f"This task has failed {failures} times in a row (most recent: {outcome_label}). Full "
  348. f"last error:\n\n{err_snippet}\n\nThe dispatcher circuit breaker is configured for "
  349. f"{failure_limit} consecutive non-success attempts. Fix the root cause and reclaim or "
  350. f"unblock the task to retry."
  351. )
  352. else:
  353. title = f"Agent {outcome_label} x{failures} (no error recorded)"
  354. detail = (
  355. f"This task has failed {failures} times in a row (most recent: {outcome_label}) but no "
  356. f"error text was captured. Check the suggested command or the worker log."
  357. )
  358. return [Diagnostic(
  359. kind="repeated_failures", severity=severity,
  360. title=title, detail=detail, actions=actions,
  361. first_seen_at=now, last_seen_at=now, count=failures,
  362. data={
  363. "consecutive_failures": failures,
  364. "most_recent_outcome": most_recent_outcome,
  365. "last_error": last_err,
  366. "failure_threshold": threshold,
  367. "failure_limit": failure_limit,
  368. },
  369. )]
  370. def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
  371. """Trailing run outcomes show >= cfg["crash_threshold"] (default 2)
  372. consecutive ``crashed`` with no ``completed``/``reclaimed`` between. Fires
  373. earlier than ``repeated_failures`` for a crash-specific heads-up and
  374. suppresses itself when the unified rule is about to fire.
  375. Exempt: done/archived (a manual done appends no completed run, so the
  376. streak would be permanent) and running (an in-flight run has no outcome
  377. and wouldn't break the scan)."""
  378. if _task_field(task, "status") in ("done", "archived", "running"):
  379. return []
  380. # Unified rule will catch this — let it handle to avoid double fire.
  381. if (_task_field(task, "consecutive_failures", 0) or 0) >= int(_failure_threshold(cfg)):
  382. return []
  383. threshold = int(cfg.get("crash_threshold", 2))
  384. # Count trailing consecutive 'crashed' outcomes; a success (or manual
  385. # reclaim) breaks the streak, other outcomes neither count nor break it.
  386. consecutive = 0
  387. last_err = None
  388. for r in _runs_newest_first(runs):
  389. outcome = _task_field(r, "outcome")
  390. if outcome == "crashed":
  391. consecutive += 1
  392. if last_err is None:
  393. last_err = _task_field(r, "error")
  394. elif outcome in {"completed", "reclaimed"}:
  395. break
  396. if consecutive < threshold:
  397. return []
  398. task_id = _task_field(task, "id")
  399. actions: list[DiagnosticAction] = []
  400. if task_id:
  401. actions.append(_log_hint_action(task_id))
  402. actions.extend(_generic_recovery_actions(task, running=_is_running(task)))
  403. severity = "critical" if consecutive >= threshold * 2 else "error"
  404. # Error up-front so operators see WHAT broke without opening the logs.
  405. err_snippet = _error_snippet(last_err)
  406. if err_snippet:
  407. title = f"Agent crashed {consecutive}x: {err_snippet.splitlines()[0][:160]}"
  408. detail = (
  409. f"The last {consecutive} runs ended with outcome=crashed. "
  410. f"Full last error:\n\n{err_snippet}"
  411. )
  412. else:
  413. title = f"Agent crashed {consecutive}x (no error recorded)"
  414. detail = (
  415. f"The last {consecutive} runs ended with outcome=crashed but "
  416. f"no error text was captured. Check the worker log for more."
  417. )
  418. return [Diagnostic(
  419. kind="repeated_crashes", severity=severity,
  420. title=title, detail=detail, actions=actions,
  421. first_seen_at=now, last_seen_at=now, count=consecutive,
  422. data={"consecutive_crashes": consecutive, "last_error": last_err},
  423. )]
  424. def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagnostic]:
  425. """Legacy review handoff starving children: the implementation is
  426. sticky-blocked with a ``review-required:`` reason while todo children wait
  427. for it to be terminal. Graph-aware; deliberately mutates nothing."""
  428. if _task_field(task, "status") != "blocked":
  429. return []
  430. latest_block = next((ev for ev in reversed(list(events)) if _event_kind(ev) == "blocked"), None)
  431. if latest_block is None:
  432. return []
  433. reason = str(_parse_payload(latest_block).get("reason") or "").strip()
  434. if not reason.lower().startswith("review-required:"):
  435. return []
  436. graph = cfg.get("_graph")
  437. if not isinstance(graph, dict):
  438. return []
  439. waiting_children = [
  440. child for child in (graph.get("children") or [])
  441. if isinstance(child, dict) and child.get("status") == "todo"
  442. ]
  443. if not waiting_children:
  444. return []
  445. task_id = str(_task_field(task, "id") or "")
  446. child_ids = [str(child.get("id")) for child in waiting_children if child.get("id")]
  447. actions: list[DiagnosticAction] = []
  448. if task_id:
  449. actions.append(_cli_hint(
  450. "Complete the finished implementation phase", f"hermes kanban complete {task_id}",
  451. suggested=True,
  452. ))
  453. if task_id and child_ids:
  454. actions.append(_cli_hint(
  455. "Or unlink the incorrectly gated reviewer", f"hermes kanban unlink {task_id} {child_ids[0]}",
  456. ))
  457. blocked_at = _event_ts(latest_block) or now
  458. return [Diagnostic(
  459. kind="review_dependency_deadlock", severity="error",
  460. title=f"Review handoff blocks {len(child_ids)} dependent task(s)",
  461. detail="This implementation is sticky-blocked for review while its downstream task(s) require "
  462. "the implementation to be done or archived before they can run. Complete the finished "
  463. "phase, unlink the incorrect dependency, or migrate this workflow to the first-class "
  464. "review lifecycle.",
  465. actions=actions,
  466. first_seen_at=blocked_at, last_seen_at=blocked_at, count=len(child_ids),
  467. data={"blocked_parent_id": task_id, "waiting_child_ids": child_ids, "block_reason": reason},
  468. )]
  469. def _rule_running_with_open_parents(task, events, runs, now, cfg) -> list[Diagnostic]:
  470. """A ``running`` card with a direct parent that is not ``done``/``archived``:
  471. the dependency gate is not holding it (the parent reopened mid-run, or the
  472. edge predates the running-child refusal) and ``kanban_complete`` will be
  473. refused until the parents finish. Graph-aware; mutates nothing."""
  474. if _task_field(task, "status") != "running":
  475. return []
  476. graph = cfg.get("_graph")
  477. if not isinstance(graph, dict):
  478. return []
  479. open_parents = [
  480. parent for parent in (graph.get("parents") or [])
  481. if isinstance(parent, dict) and parent.get("id")
  482. and parent.get("status") not in ("done", "archived")
  483. ]
  484. if not open_parents:
  485. return []
  486. task_id = str(_task_field(task, "id") or "")
  487. parent_ids = [str(parent["id"]) for parent in open_parents]
  488. seen_at = int(_task_field(task, "started_at", default=0) or 0) or now
  489. return [Diagnostic(
  490. kind="running_with_open_parents", severity="warning",
  491. title=f"Running while {len(parent_ids)} parent(s) are not done",
  492. detail="This card is running concurrently with a parent it declares a dependency on, so the "
  493. "parent's work is not serialised ahead of it and completion will be refused until every "
  494. "parent is done or archived. Finish the parent, or unlink the edge if it was never meant "
  495. "to gate this run.",
  496. actions=[_cli_hint("Unlink the parent that should not gate this run",
  497. f"hermes kanban unlink {parent_ids[0]} {task_id}")],
  498. first_seen_at=seen_at, last_seen_at=now, count=len(parent_ids),
  499. data={"open_parents": [{"id": p["id"], "status": p.get("status")} for p in open_parents]},
  500. )]
  501. def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
  502. """Blocked for >= cfg["blocked_stale_hours"] (default 24) with no comment
  503. or unblock since the last ``blocked`` event."""
  504. hours = float(cfg.get("blocked_stale_hours", 24))
  505. if _task_field(task, "status") != "blocked":
  506. return []
  507. last_blocked_ts = _latest_event_ts(events, {"blocked"})
  508. if last_blocked_ts == 0:
  509. return []
  510. age_hours = (now - last_blocked_ts) / 3600.0
  511. if age_hours < hours:
  512. return []
  513. # Any comment / unblock after the block breaks the "stale" signal.
  514. if any(_event_kind(ev) in {"commented", "unblocked"} and _event_ts(ev) > last_blocked_ts
  515. for ev in events):
  516. return []
  517. return [Diagnostic(
  518. kind="stuck_in_blocked", severity="warning",
  519. title=f"Task has been blocked for {int(age_hours)}h",
  520. detail=f"This task transitioned to blocked {int(age_hours)}h ago and has had no comments or "
  521. f"unblock attempts since. Blocked tasks are waiting for human input — check the block "
  522. f"reason and either unblock with feedback or answer with a comment.",
  523. actions=[DiagnosticAction(kind="comment", label="Add a comment / unblock the task",
  524. suggested=True)],
  525. first_seen_at=last_blocked_ts, last_seen_at=last_blocked_ts, count=1,
  526. data={"blocked_at": last_blocked_ts, "age_hours": round(age_hours, 1)},
  527. )]
  528. def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic]:
  529. """>= cfg["block_cycle_threshold"] (default 3) blocked-after-unblocked
  530. cycles within cfg["block_cycle_window_seconds"] (default 24h). Complements
  531. ``_rule_stuck_in_blocked``, whose timer any unblock resets, so fast cyclers
  532. are invisible to it.
  533. ``_rule_stuck_in_blocked`` resets its timer on any ``commented`` / ``unblocked`` event, so a task that
  534. cycles every few minutes is invisible to it regardless of how many times it cycles (#29747 gap 1). This
  535. rule complements that one by counting block→unblock cycles in a sliding window.
  536. """
  537. threshold = _positive_int(cfg.get("block_cycle_threshold"), 3)
  538. window_seconds = float(cfg.get("block_cycle_window_seconds", 24 * 3600))
  539. cycle_cutoff = now - window_seconds
  540. # Walk in id (arrival) order — created_at alone can't order events that
  541. # share a second. A blocked event after >= 1 unblocked since the last
  542. # counted cycle is a new cycle.
  543. cycles = 0
  544. seen_unblock_since_last_cycle = False
  545. initial_blocked_ts = 0
  546. last_cycle_blocked_ts = 0
  547. for ev in events:
  548. ts = _event_ts(ev)
  549. if ts < cycle_cutoff:
  550. continue
  551. kind = _event_kind(ev)
  552. if kind == "blocked":
  553. if initial_blocked_ts == 0:
  554. initial_blocked_ts = ts
  555. if seen_unblock_since_last_cycle:
  556. cycles += 1
  557. last_cycle_blocked_ts = ts
  558. seen_unblock_since_last_cycle = False
  559. elif kind == "unblocked":
  560. seen_unblock_since_last_cycle = True
  561. if cycles < threshold:
  562. return []
  563. task_id = _task_field(task, "id")
  564. actions: list[DiagnosticAction] = []
  565. if task_id:
  566. cmd = f"hermes kanban events {task_id}"
  567. actions.append(_cli_hint(f"Check block reasons: {cmd}", cmd, suggested=True))
  568. return [Diagnostic(
  569. kind="block_unblock_cycling", severity="warning",
  570. title=f"Task block→unblock cycled {cycles}x in {int(window_seconds/3600)}h",
  571. detail=f"This task has been blocked {cycles} times after being unblocked, suggesting the "
  572. f"unblock is not addressing the root cause and the worker keeps hitting the same wall. "
  573. f"Review the block reasons in the event history; a different intervention (reassign, "
  574. f"change scope, archive) may be needed.",
  575. actions=actions,
  576. first_seen_at=int(initial_blocked_ts) if initial_blocked_ts else int(now),
  577. last_seen_at=int(last_cycle_blocked_ts) if last_cycle_blocked_ts else int(now),
  578. count=cycles,
  579. data={"cycles": cycles, "window_seconds": int(window_seconds)},
  580. )]
  581. def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
  582. """Assigned, unclaimed, ``ready`` for >= cfg["stranded_threshold_seconds"]
  583. (default 30 min). Deliberately age-based and identity-agnostic so it
  584. catches typo'd assignees, deleted profiles, and down external worker
  585. pools alike without a registry to curate. Unassigned tasks are excluded —
  586. the dispatcher's ``skipped_unassigned`` already covers them."""
  587. threshold_seconds = float(cfg.get("stranded_threshold_seconds", 30 * 60))
  588. if _task_field(task, "status") != "ready":
  589. return []
  590. # A live claim means it's being worked on even without progress yet.
  591. if _task_field(task, "claim_lock"):
  592. return []
  593. assignee = _task_field(task, "assignee") or ""
  594. if not assignee.strip():
  595. return []
  596. # Most recent event that put the task into ready; with none (old task /
  597. # truncated events) fall back to created_at — over-flagging an ancient
  598. # task beats missing a stranded one.
  599. last_ready_ts = _latest_event_ts(events, {"created", "promoted", "reclaimed", "unblocked"})
  600. if last_ready_ts == 0:
  601. last_ready_ts = int(_task_field(task, "created_at", default=0) or 0)
  602. if last_ready_ts == 0:
  603. return []
  604. age_seconds = now - last_ready_ts
  605. if age_seconds < threshold_seconds:
  606. return []
  607. age_str = f"{age_seconds / 3600:.1f}h" if age_seconds >= 3600 else f"{int(age_seconds / 60)}m"
  608. # Escalate with age: <2x threshold warning, 2x-6x error, >6x critical.
  609. if age_seconds >= threshold_seconds * 6:
  610. severity = "critical"
  611. elif age_seconds >= threshold_seconds * 2:
  612. severity = "error"
  613. else:
  614. severity = "warning"
  615. actions = [
  616. DiagnosticAction(kind="reassign", label="Reassign to a different worker",
  617. payload={"current_assignee": assignee}),
  618. _cli_hint("Check dispatcher status", "hermes kanban diagnostics"),
  619. ]
  620. return [Diagnostic(
  621. kind="stranded_in_ready", severity=severity,
  622. title=f"Ready for {age_str} with no worker",
  623. detail=f"This task has been ready for {age_str} but nothing has claimed it. Common causes: "
  624. f"assignee {assignee!r} is misspelled, the profile was deleted, or the external worker "
  625. f"pool for this lane is down. Confirm the assignee is correct and that a worker is "
  626. f"actually polling for it.",
  627. actions=actions,
  628. first_seen_at=last_ready_ts, last_seen_at=last_ready_ts, count=1,
  629. data={"ready_since": last_ready_ts, "age_seconds": int(age_seconds),
  630. "assignee": assignee, "threshold_seconds": int(threshold_seconds)},
  631. )]
  632. def _rule_stale_running(task, events, runs, now, cfg) -> list[Diagnostic]:
  633. """A ``running`` card whose claim expired >= cfg["stale_running_grace_seconds"]
  634. (default 1h) ago with no live recorded worker.
  635. Nothing in the dispatcher is going to free this card — its reclaim paths can decline a
  636. card whose owner receipt never resolved, and until they could not, the failure was
  637. invisible: an operator saw only the cards queued behind the profile's worker cap. So the
  638. jam itself is the signal. Silent in every shape that can still progress: a live claim
  639. (the owner is still working), a heartbeat inside the dispatcher's max-stale window, a
  640. foreign host's lock (its pids say nothing here), or a live recorded worker. Read-only.
  641. """
  642. grace_seconds = float(cfg.get("stale_running_grace_seconds", 60 * 60))
  643. if _task_field(task, "status") != "running":
  644. return []
  645. from hermes_cli import kanban_db as kb
  646. expires = kb._to_epoch(_task_field(task, "claim_expires", None))
  647. if expires is None or (now - expires) < grace_seconds:
  648. return []
  649. lock = str(_task_field(task, "claim_lock", "") or "")
  650. if not lock.startswith(kb._host_prefix()):
  651. return []
  652. hb = kb._to_epoch(_task_field(task, "last_heartbeat_at", None))
  653. if hb is not None and (now - hb) <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:
  654. return []
  655. from hermes_cli.kanban_db_dispatch import _worker_alive
  656. recorded = [(_task_field(task, "worker_pid", None),
  657. _task_field(task, "worker_started_at", None))]
  658. # Only this attempt's pids speak for it: an earlier run's worker that is still alive
  659. # (or whose number was reused) says nothing about whether the current run is executing.
  660. run_id = _task_field(task, "current_run_id", None)
  661. for ev in events:
  662. if run_id is not None and _task_field(ev, "run_id", None) != run_id:
  663. continue
  664. if _event_kind(ev) in {"spawned", "worker_bound"}:
  665. payload = _parse_payload(ev)
  666. recorded.append((payload.get("pid"), payload.get("started_at")))
  667. pids = sorted({int(pid) for pid, _ in recorded if pid})
  668. if any(_worker_alive(int(pid), started_at) for pid, started_at in recorded if pid):
  669. return []
  670. expired_seconds = now - expires
  671. age_str = (f"{expired_seconds / 3600:.1f}h" if expired_seconds >= 3600
  672. else f"{int(expired_seconds / 60)}m")
  673. # Escalate with age: <2x grace warning, 2x-6x error, >6x critical.
  674. if expired_seconds >= grace_seconds * 6:
  675. severity = "critical"
  676. elif expired_seconds >= grace_seconds * 2:
  677. severity = "error"
  678. else:
  679. severity = "warning"
  680. task_id = str(_task_field(task, "id") or "")
  681. return [Diagnostic(
  682. kind="stale_running", severity=severity,
  683. title=f"Running {age_str} past its expired claim with no live worker",
  684. detail=f"Nothing is executing this card and nothing has reclaimed it: the claim expired "
  685. f"{age_str} ago, no heartbeat arrived inside the last hour, and no pid recorded for "
  686. f"the run ({', '.join(str(p) for p in pids) or 'none'}) is alive. The dispatcher "
  687. f"declines a reclaim when the run's owner receipt never resolved, so the card can "
  688. f"hold its profile's worker slot indefinitely while every card queued behind it "
  689. f"stays deferred. Check the card's log, then reclaim it.",
  690. actions=[DiagnosticAction(kind="reclaim", label="Reclaim the dead claim", suggested=True),
  691. _cli_hint("Reclaim from the CLI", f"hermes kanban reclaim {task_id}")],
  692. first_seen_at=expires, last_seen_at=expires, count=1,
  693. data={"claim_expires": expires, "expired_seconds": int(expired_seconds),
  694. "recorded_pids": pids, "last_heartbeat_at": hb,
  695. "grace_seconds": int(grace_seconds)},
  696. )]
  697. # Order matters: earlier rules render first on severity ties.
  698. _RULES: list[RuleFn] = [
  699. _rule_hallucinated_cards,
  700. _rule_triage_aux_unavailable,
  701. _rule_prose_phantom_refs,
  702. _rule_repeated_failures,
  703. _rule_repeated_crashes,
  704. _rule_review_dependency_deadlock,
  705. _rule_running_with_open_parents,
  706. _rule_stuck_in_blocked,
  707. _rule_block_unblock_cycling,
  708. _rule_stranded_in_ready,
  709. _rule_stale_running,
  710. ]
  711. DEFAULT_CONFIG = {
  712. # Match the dispatcher default (kanban.failure_limit) so repeated-failure
  713. # diagnostics do not lag behind the default auto-block threshold.
  714. "failure_threshold": 2,
  715. # Legacy alias accepted at read time by _rule_repeated_failures.
  716. "spawn_failure_threshold": 2,
  717. "crash_threshold": 2,
  718. "blocked_stale_hours": 24,
  719. # Below 30 min the signal is dominated by tasks about to be claimed on
  720. # the next dispatcher tick.
  721. "stranded_threshold_seconds": 30 * 60,
  722. # The dispatcher reclaims an expired claim on its own next tick, so a card
  723. # still `running` an hour past its expiry means every reclaim path declined
  724. # it. Same hour the heartbeat backstop uses.
  725. "stale_running_grace_seconds": 60 * 60,
  726. }
  727. def _has_explicit_threshold(cfg: dict) -> bool:
  728. return "failure_threshold" in cfg or "spawn_failure_threshold" in cfg
  729. def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
  730. """Diagnostics config from the ``kanban`` section. ``kanban.diagnostics.
  731. failure_threshold`` is an explicit override; otherwise the threshold is
  732. ``kanban.failure_limit`` so diagnostics match the dispatcher's breaker."""
  733. kanban_cfg = kanban_cfg or {}
  734. diag_cfg = dict(kanban_cfg.get("diagnostics") or {})
  735. diag_cfg.setdefault(
  736. "failure_limit", kanban_cfg.get("failure_limit", DEFAULT_CONFIG["failure_threshold"]),
  737. )
  738. if not _has_explicit_threshold(diag_cfg):
  739. diag_cfg["failure_threshold"] = diag_cfg["failure_limit"]
  740. return diag_cfg
  741. def config_from_runtime_config(raw_config: Optional[dict]) -> dict:
  742. """Diagnostics config from the full runtime config: folds ``kanban`` through
  743. ``config_from_kanban_config`` and carries ``kanban``/``auxiliary``/``model``
  744. through for the triage-aware rules."""
  745. raw_config = raw_config or {}
  746. if not isinstance(raw_config, dict):
  747. return {}
  748. cfg: dict = {}
  749. kanban_cfg = raw_config.get("kanban")
  750. if isinstance(kanban_cfg, dict):
  751. cfg.update(config_from_kanban_config(kanban_cfg))
  752. cfg["kanban"] = kanban_cfg
  753. for key in ("auxiliary", "model"):
  754. value = raw_config.get(key)
  755. if value is not None:
  756. cfg[key] = value
  757. return cfg
  758. def compute_task_diagnostics(
  759. task,
  760. events: list,
  761. runs: list,
  762. *,
  763. now: Optional[int] = None,
  764. config: Optional[dict] = None,
  765. graph: Optional[dict] = None,
  766. ) -> list[Diagnostic]:
  767. """Run every rule for one task; critical first, then error, warning; ties
  768. broken by most-recent ``last_seen_at``."""
  769. now_ts = int(now if now is not None else time.time())
  770. config = config or {}
  771. cfg = {**DEFAULT_CONFIG, **config}
  772. if graph is not None:
  773. cfg["_graph"] = graph
  774. if not _has_explicit_threshold(config) and "failure_limit" in config:
  775. cfg["failure_threshold"] = _positive_int(
  776. config.get("failure_limit"), DEFAULT_CONFIG["failure_threshold"],
  777. )
  778. out: list[Diagnostic] = []
  779. for rule in _RULES:
  780. try:
  781. out.extend(rule(task, events, runs, now_ts, cfg))
  782. except Exception:
  783. # A broken rule must never 500 a whole /board request.
  784. continue
  785. severity_idx = {s: i for i, s in enumerate(SEVERITY_ORDER)}
  786. out.sort(key=lambda d: (-severity_idx.get(d.severity, -1), -(d.last_seen_at or 0)))
  787. return out