| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910 |
- """Kanban diagnostics — structured, actionable distress signals for tasks.
- A ``Diagnostic`` carries a **kind** (canonical code the UI/tests match on), a
- **severity**, title/detail text, and **actions** the dashboard renders as
- buttons and the CLI as hints. Rules are stateless and read-only over
- (task, events, runs, optional graph); callers compute on demand. Only
- operator-fixable signals (not a one-off provider 502); every diagnostic has a
- recovery action and auto-clears when the failure mode resolves.
- """
- from __future__ import annotations
- from dataclasses import asdict, dataclass, field
- from typing import Any, Callable, Iterable, Optional
- import json
- import time
- # Least → most urgent; sorted outputs put critical first.
- SEVERITY_ORDER = ("warning", "error", "critical")
- def severity_at_or_above(severity: Optional[str], threshold: Optional[str]) -> bool:
- """Return True when ``severity`` meets or exceeds ``threshold``."""
- if threshold is None:
- return True
- if severity not in SEVERITY_ORDER or threshold not in SEVERITY_ORDER:
- return False
- return SEVERITY_ORDER.index(severity) >= SEVERITY_ORDER.index(threshold)
- @dataclass
- class DiagnosticAction:
- """A recovery action. ``kind`` drives rendering: ``reclaim``/``reassign``
- POST to /tasks/:id/*; ``unblock`` PATCHes status to ready; ``cli_hint``
- shows ``payload.command``; ``open_docs`` links ``payload.url``; ``comment``
- nudges the operator. ``suggested=True`` = recommended first step."""
- kind: str
- label: str
- payload: dict = field(default_factory=dict)
- suggested: bool = False
- def to_dict(self) -> dict:
- return asdict(self)
- @dataclass
- class Diagnostic:
- """One active distress signal on a task."""
- kind: str
- severity: str # "warning" | "error" | "critical"
- title: str
- detail: str
- actions: list[DiagnosticAction] = field(default_factory=list)
- first_seen_at: int = 0
- last_seen_at: int = 0
- count: int = 1
- run_id: Optional[int] = None # None = task-wide
- data: dict = field(default_factory=dict) # structured payload for the UI
- def to_dict(self) -> dict:
- return asdict(self)
- # --- Rule helpers ---
- def _task_field(task, name, default=None):
- """Read a field from a sqlite3.Row, a kanban_db.Task dataclass, or a dict."""
- if task is None:
- return default
- try:
- if hasattr(task, "keys") and name in task.keys():
- return task[name]
- except Exception:
- pass
- if isinstance(task, dict):
- return task.get(name, default)
- return getattr(task, name, default)
- def _parse_payload(ev) -> dict:
- """Tolerate event.payload being either a dict or a JSON string."""
- p = _task_field(ev, "payload", None)
- if isinstance(p, dict):
- return p
- if isinstance(p, str):
- try:
- return json.loads(p) or {}
- except Exception:
- return {}
- return {}
- def _event_kind(ev) -> str:
- return _task_field(ev, "kind", "") or ""
- def _event_ts(ev) -> int:
- return int(_task_field(ev, "created_at", 0) or 0)
- def _first_field(task, primary: str, legacy: str, default=None):
- """``task[primary]`` unless it is None, else ``task[legacy]`` (old DB rows)."""
- v = _task_field(task, primary, None)
- return v if v is not None else _task_field(task, legacy, default)
- def _latest_event_ts(events: Iterable[Any], kinds: set[str]) -> int:
- """Max ``created_at`` over events whose kind is in ``kinds`` (0 if none)."""
- return max([0, *(_event_ts(ev) for ev in events if _event_kind(ev) in kinds)])
- def _latest_gave_up_is_terminal_provider(events: Iterable[Any]) -> bool:
- """True when the most recent breaker trip was a terminal provider error (credential
- revoked, model gone) and nothing has resumed the task since."""
- for ev in reversed(list(events)):
- kind = _event_kind(ev)
- if kind == "gave_up":
- return bool(_parse_payload(ev).get("terminal_provider"))
- if kind in {"unblocked", "promoted", "completed", "claimed"}:
- return False
- return False
- def _cli_hint(label: str, command: str, *, suggested: bool = False) -> DiagnosticAction:
- return DiagnosticAction(kind="cli_hint", label=label, payload={"command": command},
- suggested=suggested)
- def _log_hint_action(task_id: str) -> DiagnosticAction:
- cmd = f"hermes kanban log {task_id}"
- return _cli_hint(f"Check logs: {cmd}", cmd, suggested=True)
- def _error_snippet(last_err) -> str:
- """First 500 chars of the error (with ellipsis), or "" when absent."""
- err_text = (last_err or "").strip() if last_err else ""
- return err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
- def _active_hallucination_events(events: Iterable[Any], kind: str) -> list[Any]:
- """Events of ``kind`` with no ``completed``/``edited`` event strictly after
- them. Requires id-sorted (arrival-order) input, which the DB provides."""
- active: list[Any] = []
- for ev in events:
- k = _event_kind(ev)
- if k in {"completed", "edited"}:
- active.clear()
- elif k == kind:
- active.append(ev)
- return active
- def _unique_payload_ids(hits: list[Any], key: str) -> list[str]:
- """Ordered, de-duplicated ``payload[key]`` entries across ``hits``."""
- out: list[str] = []
- for ev in hits:
- for pid in _parse_payload(ev).get(key, []) or []:
- if pid not in out:
- out.append(pid)
- return out
- def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAction]:
- """Baseline recovery primitives every diagnostic can fall back on."""
- out: list[DiagnosticAction] = []
- if running:
- out.append(DiagnosticAction(kind="reclaim", label="Reclaim task", payload={}))
- out.append(DiagnosticAction(
- kind="reassign", label="Reassign to different profile", payload={"reclaim_first": running},
- ))
- return out
- def _is_running(task) -> bool:
- return _task_field(task, "status") == "running"
- def _runs_newest_first(runs) -> list[Any]:
- # reversed(sorted()) not sorted(reverse=True): equal ids must keep the
- # last-listed run first.
- return list(reversed(sorted(runs, key=lambda r: _task_field(r, "id", 0))))
- # --- Rule implementations ---
- # Each rule: (task, events, runs, now_ts, config) -> list[Diagnostic].
- # ``events``/``runs`` are kanban_db rows/dataclasses or same-shaped dicts.
- RuleFn = Callable[[Any, list[Any], list[Any], int, dict], list[Diagnostic]]
- def _aux_slot_explicit(slot: Any) -> bool:
- """True if the aux slot was user-configured: provider other than "auto",
- or any of model/base_url/api_key set (the default falls through to the
- main model)."""
- if not isinstance(slot, dict):
- return False
- provider = str(slot.get("provider") or "").strip().lower()
- if provider and provider != "auto":
- return True
- return any(str(slot.get(key) or "").strip() for key in ("model", "base_url", "api_key"))
- def _main_model_visible(raw_config: Any) -> bool:
- """Best-effort "a main model is configured" from the raw config dict (the
- dashboard process may not share CLI runtime state). Unprovable => False,
- which errs toward NOT firing the diagnostic."""
- if not isinstance(raw_config, dict):
- return False
- model_cfg = raw_config.get("model")
- if isinstance(model_cfg, dict):
- provider = str(model_cfg.get("provider") or "").strip()
- model = str(
- model_cfg.get("default") or model_cfg.get("model") or model_cfg.get("name") or ""
- ).strip()
- return bool(provider and model)
- return bool(str(model_cfg or "").strip())
- def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
- """Report whether the triage aux paths look configured: ``{auto_decompose,
- decomposer_explicit, specifier_explicit, main_model_visible}``. ``None``
- when no config context is present (keeps low-level callers/tests silent)."""
- if not isinstance(config, dict):
- return None
- explicit = config.get("triage_aux_status")
- if isinstance(explicit, dict):
- return explicit
- aux = config.get("auxiliary")
- kanban_cfg = config.get("kanban") if isinstance(config.get("kanban"), dict) else {}
- # No auxiliary/kanban/model keys at all => a low-level caller passing {}.
- if not isinstance(aux, dict) and not kanban_cfg and "model" not in config:
- return None
- aux = aux if isinstance(aux, dict) else {}
- return {
- # ``auto_decompose`` defaults to True per kanban DEFAULT_CONFIG.
- "auto_decompose": bool(kanban_cfg["auto_decompose"]) if "auto_decompose" in kanban_cfg else True,
- "decomposer_explicit": _aux_slot_explicit(aux.get("kanban_decomposer")),
- "specifier_explicit": _aux_slot_explicit(aux.get("triage_specifier")),
- "main_model_visible": _main_model_visible(config),
- }
- def _positive_int(value: Any, default: int) -> int:
- try:
- parsed = int(value)
- except (TypeError, ValueError):
- return default
- return parsed if parsed >= 1 else default
- def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
- """A worker's kanban_complete named created_cards that don't exist / weren't
- its own; the completion was blocked. Clears on a later completion/edit."""
- hits = _active_hallucination_events(events, "completion_blocked_hallucination")
- if not hits:
- return []
- actions = [DiagnosticAction(kind="comment", label="Add a comment explaining what to do",
- suggested=False)]
- actions += _generic_recovery_actions(task, running=_is_running(task))
- return [Diagnostic(
- kind="hallucinated_cards", severity="error",
- title="Worker claimed cards that don't exist",
- detail="The completing worker declared created_cards that either didn't exist or weren't "
- "created by its profile. The completion was blocked and the task stayed in its prior "
- "state. Usually means the worker hallucinated ids instead of capturing return values "
- "from kanban_create.",
- actions=actions,
- first_seen_at=_event_ts(hits[0]), last_seen_at=_event_ts(hits[-1]), count=len(hits),
- data={"phantom_ids": _unique_payload_ids(hits, "phantom_cards")},
- )]
- # (primary_slot, fallback_slot, primary_desc, detail_path) keyed by auto_decompose.
- _TRIAGE_SLOTS = {
- True: (
- "auxiliary.kanban_decomposer", "auxiliary.triage_specifier", "decomposer",
- "Auto-decompose is on, so the dispatcher needs auxiliary.kanban_decomposer (with "
- "auxiliary.triage_specifier as a fallback for non-fan-out tasks).",
- ),
- False: (
- "auxiliary.triage_specifier", "auxiliary.kanban_decomposer", "specifier",
- "Auto-decompose is off, so triage tasks need "
- "`hermes kanban specify`, which uses auxiliary.triage_specifier.",
- ),
- }
- def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnostic]:
- """A triage task can't leave triage without a usable aux model. With
- auto-decompose on the primary slot is ``auxiliary.kanban_decomposer``
- (specifier as fallback); off, it is ``auxiliary.triage_specifier``. The
- default ``provider: auto`` falls back to the main model, so this fires only
- when the slot isn't explicit AND no main model is visible. Requires config
- context ({} keeps it silent)."""
- if _task_field(task, "status") != "triage":
- return []
- status = triage_aux_status(cfg)
- if status is None:
- return []
- auto_decompose = bool(status.get("auto_decompose"))
- main_visible = bool(status.get("main_model_visible"))
- decomposer_explicit = bool(status.get("decomposer_explicit"))
- specifier_explicit = bool(status.get("specifier_explicit"))
- primary_slot, fallback_slot, primary_desc, detail_path = _TRIAGE_SLOTS[auto_decompose]
- primary_explicit, fallback_explicit = (
- (decomposer_explicit, specifier_explicit) if auto_decompose
- else (specifier_explicit, decomposer_explicit)
- )
- if primary_explicit or main_visible:
- return []
- task_id = _task_field(task, "id") or "<task_id>"
- actions = [_cli_hint(
- f"Configure {primary_slot}", f"hermes config set {primary_slot}.provider auto", suggested=True,
- )]
- if not fallback_explicit and not main_visible:
- actions.append(_cli_hint(
- f"Or configure fallback {fallback_slot}", f"hermes config set {fallback_slot}.provider auto",
- ))
- if not auto_decompose:
- cmd = f"hermes kanban specify {task_id}"
- actions.append(_cli_hint(f"Specify manually: {cmd}", cmd))
- return [Diagnostic(
- kind="triage_aux_unavailable", severity="warning",
- title=f"Triage {primary_desc} has no usable model",
- detail=f"This task is still in triage and no working auxiliary model is visible to the "
- f"dispatcher. {detail_path} The default slot uses `provider: auto` which falls back to "
- f"the main model, but no main model is configured either. Configure the slot directly "
- f"or set a main model so the auto fallback can take over.",
- actions=actions,
- first_seen_at=now, last_seen_at=now, count=1,
- data={"task_id": task_id, "auto_decompose": auto_decompose,
- "primary_slot": primary_slot, "main_model_visible": main_visible},
- )]
- def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
- """Advisory: the completion summary mentions ``t_<hex>`` ids that don't
- resolve. Warning only; clears on a later clean completion."""
- hits = _active_hallucination_events(events, "suspected_hallucinated_references")
- if not hits:
- return []
- return [Diagnostic(
- kind="prose_phantom_refs", severity="warning",
- title="Completion summary references unknown task ids",
- detail="The completion summary mentions task ids that don't resolve in this board's database. "
- "The completion itself succeeded, but downstream consumers parsing the summary may be "
- "pointed at cards that never existed.",
- actions=_generic_recovery_actions(task, running=_is_running(task)),
- first_seen_at=_event_ts(hits[0]), last_seen_at=_event_ts(hits[-1]), count=len(hits),
- data={"phantom_refs": _unique_payload_ids(hits, "phantom_refs")},
- )]
- def _failure_threshold(cfg: dict) -> Any:
- """``failure_threshold`` with the legacy ``spawn_failure_threshold`` alias."""
- return cfg.get("failure_threshold", cfg.get("spawn_failure_threshold", 3))
- _OUTCOME_LABELS = {"spawn_failed": "spawn", "timed_out": "timeout", "crashed": "crash"}
- def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
- """``consecutive_failures`` >= cfg["failure_threshold"] (legacy key
- ``spawn_failure_threshold``), regardless of failure mode — the kernel keeps
- retrying and the operator must intervene. Runtime callers derive the
- threshold from ``kanban.failure_limit`` so it doesn't lag the breaker.
- Exempt: done/archived (a manual done ends no run, so the streak is history)
- and running (a retry in flight must not read as a current failure; re-fires
- if it fails too)."""
- if _task_field(task, "status") in ("done", "archived", "running"):
- return []
- threshold = _positive_int(_failure_threshold(cfg), 3)
- failure_limit = _positive_int(cfg.get("failure_limit"), threshold)
- failures = _first_field(task, "consecutive_failures", "spawn_failures", 0)
- # A terminal provider error (credential revoked, model gone) blocks the card after ONE
- # attempt, below any threshold; it still needs an operator, so diagnose it now.
- terminal_trip = _latest_gave_up_is_terminal_provider(events)
- if not terminal_trip and (failures is None or failures < threshold):
- return []
- failures = failures or 0
- last_err = _first_field(task, "last_failure_error", "last_spawn_error")
- assignee = _task_field(task, "assignee")
- # Most recent failure outcome makes the title/action specific.
- most_recent_outcome = next(
- (oc for oc in (_task_field(r, "outcome") for r in _runs_newest_first(runs))
- if oc in {"spawn_failed", "timed_out", "crashed"}),
- None,
- )
- actions: list[DiagnosticAction] = []
- if most_recent_outcome == "spawn_failed" and assignee and assignee != "default":
- # Spawn is failing specifically — profile setup issue.
- doctor, auth = f"hermes -p {assignee} doctor", f"hermes -p {assignee} auth"
- actions.append(_cli_hint(f"Verify profile: {doctor}", doctor, suggested=True))
- actions.append(_cli_hint(f"Fix profile auth: {auth}", auth))
- elif most_recent_outcome in {"timed_out", "crashed"}:
- # Worker got off the ground but died: logs diagnose, reclaim/reassign recover.
- task_id = _task_field(task, "id")
- if task_id:
- actions.append(_log_hint_action(task_id))
- actions.extend(_generic_recovery_actions(task, running=_is_running(task)))
- severity = "critical" if failures >= threshold * 2 else "error"
- err_snippet = _error_snippet(last_err)
- outcome_label = _OUTCOME_LABELS.get(most_recent_outcome or "", "failure")
- if terminal_trip:
- title = "Provider rejected this profile's credential or model — blocked after one attempt"
- detail = (
- f"The worker's provider call failed with an error a retry cannot fix (revoked or invalid "
- f"API key, model not found), so the dispatcher blocked the task instead of spending the "
- f"{failure_limit}-attempt retry budget on it. Full last error:\n\n{err_snippet}\n\n"
- f"Fix the assignee profile's provider credentials/model, then unblock the task."
- )
- elif err_snippet:
- title = f"Agent {outcome_label} x{failures}: {err_snippet.splitlines()[0][:160]}"
- detail = (
- f"This task has failed {failures} times in a row (most recent: {outcome_label}). Full "
- f"last error:\n\n{err_snippet}\n\nThe dispatcher circuit breaker is configured for "
- f"{failure_limit} consecutive non-success attempts. Fix the root cause and reclaim or "
- f"unblock the task to retry."
- )
- else:
- title = f"Agent {outcome_label} x{failures} (no error recorded)"
- detail = (
- f"This task has failed {failures} times in a row (most recent: {outcome_label}) but no "
- f"error text was captured. Check the suggested command or the worker log."
- )
- return [Diagnostic(
- kind="repeated_failures", severity=severity,
- title=title, detail=detail, actions=actions,
- first_seen_at=now, last_seen_at=now, count=failures,
- data={
- "consecutive_failures": failures,
- "most_recent_outcome": most_recent_outcome,
- "last_error": last_err,
- "failure_threshold": threshold,
- "failure_limit": failure_limit,
- },
- )]
- def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
- """Trailing run outcomes show >= cfg["crash_threshold"] (default 2)
- consecutive ``crashed`` with no ``completed``/``reclaimed`` between. Fires
- earlier than ``repeated_failures`` for a crash-specific heads-up and
- suppresses itself when the unified rule is about to fire.
- Exempt: done/archived (a manual done appends no completed run, so the
- streak would be permanent) and running (an in-flight run has no outcome
- and wouldn't break the scan)."""
- if _task_field(task, "status") in ("done", "archived", "running"):
- return []
- # Unified rule will catch this — let it handle to avoid double fire.
- if (_task_field(task, "consecutive_failures", 0) or 0) >= int(_failure_threshold(cfg)):
- return []
- threshold = int(cfg.get("crash_threshold", 2))
- # Count trailing consecutive 'crashed' outcomes; a success (or manual
- # reclaim) breaks the streak, other outcomes neither count nor break it.
- consecutive = 0
- last_err = None
- for r in _runs_newest_first(runs):
- outcome = _task_field(r, "outcome")
- if outcome == "crashed":
- consecutive += 1
- if last_err is None:
- last_err = _task_field(r, "error")
- elif outcome in {"completed", "reclaimed"}:
- break
- if consecutive < threshold:
- return []
- task_id = _task_field(task, "id")
- actions: list[DiagnosticAction] = []
- if task_id:
- actions.append(_log_hint_action(task_id))
- actions.extend(_generic_recovery_actions(task, running=_is_running(task)))
- severity = "critical" if consecutive >= threshold * 2 else "error"
- # Error up-front so operators see WHAT broke without opening the logs.
- err_snippet = _error_snippet(last_err)
- if err_snippet:
- title = f"Agent crashed {consecutive}x: {err_snippet.splitlines()[0][:160]}"
- detail = (
- f"The last {consecutive} runs ended with outcome=crashed. "
- f"Full last error:\n\n{err_snippet}"
- )
- else:
- title = f"Agent crashed {consecutive}x (no error recorded)"
- detail = (
- f"The last {consecutive} runs ended with outcome=crashed but "
- f"no error text was captured. Check the worker log for more."
- )
- return [Diagnostic(
- kind="repeated_crashes", severity=severity,
- title=title, detail=detail, actions=actions,
- first_seen_at=now, last_seen_at=now, count=consecutive,
- data={"consecutive_crashes": consecutive, "last_error": last_err},
- )]
- def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagnostic]:
- """Legacy review handoff starving children: the implementation is
- sticky-blocked with a ``review-required:`` reason while todo children wait
- for it to be terminal. Graph-aware; deliberately mutates nothing."""
- if _task_field(task, "status") != "blocked":
- return []
- latest_block = next((ev for ev in reversed(list(events)) if _event_kind(ev) == "blocked"), None)
- if latest_block is None:
- return []
- reason = str(_parse_payload(latest_block).get("reason") or "").strip()
- if not reason.lower().startswith("review-required:"):
- return []
- graph = cfg.get("_graph")
- if not isinstance(graph, dict):
- return []
- waiting_children = [
- child for child in (graph.get("children") or [])
- if isinstance(child, dict) and child.get("status") == "todo"
- ]
- if not waiting_children:
- return []
- task_id = str(_task_field(task, "id") or "")
- child_ids = [str(child.get("id")) for child in waiting_children if child.get("id")]
- actions: list[DiagnosticAction] = []
- if task_id:
- actions.append(_cli_hint(
- "Complete the finished implementation phase", f"hermes kanban complete {task_id}",
- suggested=True,
- ))
- if task_id and child_ids:
- actions.append(_cli_hint(
- "Or unlink the incorrectly gated reviewer", f"hermes kanban unlink {task_id} {child_ids[0]}",
- ))
- blocked_at = _event_ts(latest_block) or now
- return [Diagnostic(
- kind="review_dependency_deadlock", severity="error",
- title=f"Review handoff blocks {len(child_ids)} dependent task(s)",
- detail="This implementation is sticky-blocked for review while its downstream task(s) require "
- "the implementation to be done or archived before they can run. Complete the finished "
- "phase, unlink the incorrect dependency, or migrate this workflow to the first-class "
- "review lifecycle.",
- actions=actions,
- first_seen_at=blocked_at, last_seen_at=blocked_at, count=len(child_ids),
- data={"blocked_parent_id": task_id, "waiting_child_ids": child_ids, "block_reason": reason},
- )]
- def _rule_running_with_open_parents(task, events, runs, now, cfg) -> list[Diagnostic]:
- """A ``running`` card with a direct parent that is not ``done``/``archived``:
- the dependency gate is not holding it (the parent reopened mid-run, or the
- edge predates the running-child refusal) and ``kanban_complete`` will be
- refused until the parents finish. Graph-aware; mutates nothing."""
- if _task_field(task, "status") != "running":
- return []
- graph = cfg.get("_graph")
- if not isinstance(graph, dict):
- return []
- open_parents = [
- parent for parent in (graph.get("parents") or [])
- if isinstance(parent, dict) and parent.get("id")
- and parent.get("status") not in ("done", "archived")
- ]
- if not open_parents:
- return []
- task_id = str(_task_field(task, "id") or "")
- parent_ids = [str(parent["id"]) for parent in open_parents]
- seen_at = int(_task_field(task, "started_at", default=0) or 0) or now
- return [Diagnostic(
- kind="running_with_open_parents", severity="warning",
- title=f"Running while {len(parent_ids)} parent(s) are not done",
- detail="This card is running concurrently with a parent it declares a dependency on, so the "
- "parent's work is not serialised ahead of it and completion will be refused until every "
- "parent is done or archived. Finish the parent, or unlink the edge if it was never meant "
- "to gate this run.",
- actions=[_cli_hint("Unlink the parent that should not gate this run",
- f"hermes kanban unlink {parent_ids[0]} {task_id}")],
- first_seen_at=seen_at, last_seen_at=now, count=len(parent_ids),
- data={"open_parents": [{"id": p["id"], "status": p.get("status")} for p in open_parents]},
- )]
- def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
- """Blocked for >= cfg["blocked_stale_hours"] (default 24) with no comment
- or unblock since the last ``blocked`` event."""
- hours = float(cfg.get("blocked_stale_hours", 24))
- if _task_field(task, "status") != "blocked":
- return []
- last_blocked_ts = _latest_event_ts(events, {"blocked"})
- if last_blocked_ts == 0:
- return []
- age_hours = (now - last_blocked_ts) / 3600.0
- if age_hours < hours:
- return []
- # Any comment / unblock after the block breaks the "stale" signal.
- if any(_event_kind(ev) in {"commented", "unblocked"} and _event_ts(ev) > last_blocked_ts
- for ev in events):
- return []
- return [Diagnostic(
- kind="stuck_in_blocked", severity="warning",
- title=f"Task has been blocked for {int(age_hours)}h",
- detail=f"This task transitioned to blocked {int(age_hours)}h ago and has had no comments or "
- f"unblock attempts since. Blocked tasks are waiting for human input — check the block "
- f"reason and either unblock with feedback or answer with a comment.",
- actions=[DiagnosticAction(kind="comment", label="Add a comment / unblock the task",
- suggested=True)],
- first_seen_at=last_blocked_ts, last_seen_at=last_blocked_ts, count=1,
- data={"blocked_at": last_blocked_ts, "age_hours": round(age_hours, 1)},
- )]
- def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic]:
- """>= cfg["block_cycle_threshold"] (default 3) blocked-after-unblocked
- cycles within cfg["block_cycle_window_seconds"] (default 24h). Complements
- ``_rule_stuck_in_blocked``, whose timer any unblock resets, so fast cyclers
- are invisible to it.
- ``_rule_stuck_in_blocked`` resets its timer on any ``commented`` / ``unblocked`` event, so a task that
- cycles every few minutes is invisible to it regardless of how many times it cycles (#29747 gap 1). This
- rule complements that one by counting block→unblock cycles in a sliding window.
- """
- threshold = _positive_int(cfg.get("block_cycle_threshold"), 3)
- window_seconds = float(cfg.get("block_cycle_window_seconds", 24 * 3600))
- cycle_cutoff = now - window_seconds
- # Walk in id (arrival) order — created_at alone can't order events that
- # share a second. A blocked event after >= 1 unblocked since the last
- # counted cycle is a new cycle.
- cycles = 0
- seen_unblock_since_last_cycle = False
- initial_blocked_ts = 0
- last_cycle_blocked_ts = 0
- for ev in events:
- ts = _event_ts(ev)
- if ts < cycle_cutoff:
- continue
- kind = _event_kind(ev)
- if kind == "blocked":
- if initial_blocked_ts == 0:
- initial_blocked_ts = ts
- if seen_unblock_since_last_cycle:
- cycles += 1
- last_cycle_blocked_ts = ts
- seen_unblock_since_last_cycle = False
- elif kind == "unblocked":
- seen_unblock_since_last_cycle = True
- if cycles < threshold:
- return []
- task_id = _task_field(task, "id")
- actions: list[DiagnosticAction] = []
- if task_id:
- cmd = f"hermes kanban events {task_id}"
- actions.append(_cli_hint(f"Check block reasons: {cmd}", cmd, suggested=True))
- return [Diagnostic(
- kind="block_unblock_cycling", severity="warning",
- title=f"Task block→unblock cycled {cycles}x in {int(window_seconds/3600)}h",
- detail=f"This task has been blocked {cycles} times after being unblocked, suggesting the "
- f"unblock is not addressing the root cause and the worker keeps hitting the same wall. "
- f"Review the block reasons in the event history; a different intervention (reassign, "
- f"change scope, archive) may be needed.",
- actions=actions,
- first_seen_at=int(initial_blocked_ts) if initial_blocked_ts else int(now),
- last_seen_at=int(last_cycle_blocked_ts) if last_cycle_blocked_ts else int(now),
- count=cycles,
- data={"cycles": cycles, "window_seconds": int(window_seconds)},
- )]
- def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
- """Assigned, unclaimed, ``ready`` for >= cfg["stranded_threshold_seconds"]
- (default 30 min). Deliberately age-based and identity-agnostic so it
- catches typo'd assignees, deleted profiles, and down external worker
- pools alike without a registry to curate. Unassigned tasks are excluded —
- the dispatcher's ``skipped_unassigned`` already covers them."""
- threshold_seconds = float(cfg.get("stranded_threshold_seconds", 30 * 60))
- if _task_field(task, "status") != "ready":
- return []
- # A live claim means it's being worked on even without progress yet.
- if _task_field(task, "claim_lock"):
- return []
- assignee = _task_field(task, "assignee") or ""
- if not assignee.strip():
- return []
- # Most recent event that put the task into ready; with none (old task /
- # truncated events) fall back to created_at — over-flagging an ancient
- # task beats missing a stranded one.
- last_ready_ts = _latest_event_ts(events, {"created", "promoted", "reclaimed", "unblocked"})
- if last_ready_ts == 0:
- last_ready_ts = int(_task_field(task, "created_at", default=0) or 0)
- if last_ready_ts == 0:
- return []
- age_seconds = now - last_ready_ts
- if age_seconds < threshold_seconds:
- return []
- age_str = f"{age_seconds / 3600:.1f}h" if age_seconds >= 3600 else f"{int(age_seconds / 60)}m"
- # Escalate with age: <2x threshold warning, 2x-6x error, >6x critical.
- if age_seconds >= threshold_seconds * 6:
- severity = "critical"
- elif age_seconds >= threshold_seconds * 2:
- severity = "error"
- else:
- severity = "warning"
- actions = [
- DiagnosticAction(kind="reassign", label="Reassign to a different worker",
- payload={"current_assignee": assignee}),
- _cli_hint("Check dispatcher status", "hermes kanban diagnostics"),
- ]
- return [Diagnostic(
- kind="stranded_in_ready", severity=severity,
- title=f"Ready for {age_str} with no worker",
- detail=f"This task has been ready for {age_str} but nothing has claimed it. Common causes: "
- f"assignee {assignee!r} is misspelled, the profile was deleted, or the external worker "
- f"pool for this lane is down. Confirm the assignee is correct and that a worker is "
- f"actually polling for it.",
- actions=actions,
- first_seen_at=last_ready_ts, last_seen_at=last_ready_ts, count=1,
- data={"ready_since": last_ready_ts, "age_seconds": int(age_seconds),
- "assignee": assignee, "threshold_seconds": int(threshold_seconds)},
- )]
- def _rule_stale_running(task, events, runs, now, cfg) -> list[Diagnostic]:
- """A ``running`` card whose claim expired >= cfg["stale_running_grace_seconds"]
- (default 1h) ago with no live recorded worker.
- Nothing in the dispatcher is going to free this card — its reclaim paths can decline a
- card whose owner receipt never resolved, and until they could not, the failure was
- invisible: an operator saw only the cards queued behind the profile's worker cap. So the
- jam itself is the signal. Silent in every shape that can still progress: a live claim
- (the owner is still working), a heartbeat inside the dispatcher's max-stale window, a
- foreign host's lock (its pids say nothing here), or a live recorded worker. Read-only.
- """
- grace_seconds = float(cfg.get("stale_running_grace_seconds", 60 * 60))
- if _task_field(task, "status") != "running":
- return []
- from hermes_cli import kanban_db as kb
- expires = kb._to_epoch(_task_field(task, "claim_expires", None))
- if expires is None or (now - expires) < grace_seconds:
- return []
- lock = str(_task_field(task, "claim_lock", "") or "")
- if not lock.startswith(kb._host_prefix()):
- return []
- hb = kb._to_epoch(_task_field(task, "last_heartbeat_at", None))
- if hb is not None and (now - hb) <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:
- return []
- from hermes_cli.kanban_db_dispatch import _worker_alive
- recorded = [(_task_field(task, "worker_pid", None),
- _task_field(task, "worker_started_at", None))]
- # Only this attempt's pids speak for it: an earlier run's worker that is still alive
- # (or whose number was reused) says nothing about whether the current run is executing.
- run_id = _task_field(task, "current_run_id", None)
- for ev in events:
- if run_id is not None and _task_field(ev, "run_id", None) != run_id:
- continue
- if _event_kind(ev) in {"spawned", "worker_bound"}:
- payload = _parse_payload(ev)
- recorded.append((payload.get("pid"), payload.get("started_at")))
- pids = sorted({int(pid) for pid, _ in recorded if pid})
- if any(_worker_alive(int(pid), started_at) for pid, started_at in recorded if pid):
- return []
- expired_seconds = now - expires
- age_str = (f"{expired_seconds / 3600:.1f}h" if expired_seconds >= 3600
- else f"{int(expired_seconds / 60)}m")
- # Escalate with age: <2x grace warning, 2x-6x error, >6x critical.
- if expired_seconds >= grace_seconds * 6:
- severity = "critical"
- elif expired_seconds >= grace_seconds * 2:
- severity = "error"
- else:
- severity = "warning"
- task_id = str(_task_field(task, "id") or "")
- return [Diagnostic(
- kind="stale_running", severity=severity,
- title=f"Running {age_str} past its expired claim with no live worker",
- detail=f"Nothing is executing this card and nothing has reclaimed it: the claim expired "
- f"{age_str} ago, no heartbeat arrived inside the last hour, and no pid recorded for "
- f"the run ({', '.join(str(p) for p in pids) or 'none'}) is alive. The dispatcher "
- f"declines a reclaim when the run's owner receipt never resolved, so the card can "
- f"hold its profile's worker slot indefinitely while every card queued behind it "
- f"stays deferred. Check the card's log, then reclaim it.",
- actions=[DiagnosticAction(kind="reclaim", label="Reclaim the dead claim", suggested=True),
- _cli_hint("Reclaim from the CLI", f"hermes kanban reclaim {task_id}")],
- first_seen_at=expires, last_seen_at=expires, count=1,
- data={"claim_expires": expires, "expired_seconds": int(expired_seconds),
- "recorded_pids": pids, "last_heartbeat_at": hb,
- "grace_seconds": int(grace_seconds)},
- )]
- # Order matters: earlier rules render first on severity ties.
- _RULES: list[RuleFn] = [
- _rule_hallucinated_cards,
- _rule_triage_aux_unavailable,
- _rule_prose_phantom_refs,
- _rule_repeated_failures,
- _rule_repeated_crashes,
- _rule_review_dependency_deadlock,
- _rule_running_with_open_parents,
- _rule_stuck_in_blocked,
- _rule_block_unblock_cycling,
- _rule_stranded_in_ready,
- _rule_stale_running,
- ]
- DEFAULT_CONFIG = {
- # Match the dispatcher default (kanban.failure_limit) so repeated-failure
- # diagnostics do not lag behind the default auto-block threshold.
- "failure_threshold": 2,
- # Legacy alias accepted at read time by _rule_repeated_failures.
- "spawn_failure_threshold": 2,
- "crash_threshold": 2,
- "blocked_stale_hours": 24,
- # Below 30 min the signal is dominated by tasks about to be claimed on
- # the next dispatcher tick.
- "stranded_threshold_seconds": 30 * 60,
- # The dispatcher reclaims an expired claim on its own next tick, so a card
- # still `running` an hour past its expiry means every reclaim path declined
- # it. Same hour the heartbeat backstop uses.
- "stale_running_grace_seconds": 60 * 60,
- }
- def _has_explicit_threshold(cfg: dict) -> bool:
- return "failure_threshold" in cfg or "spawn_failure_threshold" in cfg
- def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
- """Diagnostics config from the ``kanban`` section. ``kanban.diagnostics.
- failure_threshold`` is an explicit override; otherwise the threshold is
- ``kanban.failure_limit`` so diagnostics match the dispatcher's breaker."""
- kanban_cfg = kanban_cfg or {}
- diag_cfg = dict(kanban_cfg.get("diagnostics") or {})
- diag_cfg.setdefault(
- "failure_limit", kanban_cfg.get("failure_limit", DEFAULT_CONFIG["failure_threshold"]),
- )
- if not _has_explicit_threshold(diag_cfg):
- diag_cfg["failure_threshold"] = diag_cfg["failure_limit"]
- return diag_cfg
- def config_from_runtime_config(raw_config: Optional[dict]) -> dict:
- """Diagnostics config from the full runtime config: folds ``kanban`` through
- ``config_from_kanban_config`` and carries ``kanban``/``auxiliary``/``model``
- through for the triage-aware rules."""
- raw_config = raw_config or {}
- if not isinstance(raw_config, dict):
- return {}
- cfg: dict = {}
- kanban_cfg = raw_config.get("kanban")
- if isinstance(kanban_cfg, dict):
- cfg.update(config_from_kanban_config(kanban_cfg))
- cfg["kanban"] = kanban_cfg
- for key in ("auxiliary", "model"):
- value = raw_config.get(key)
- if value is not None:
- cfg[key] = value
- return cfg
- def compute_task_diagnostics(
- task,
- events: list,
- runs: list,
- *,
- now: Optional[int] = None,
- config: Optional[dict] = None,
- graph: Optional[dict] = None,
- ) -> list[Diagnostic]:
- """Run every rule for one task; critical first, then error, warning; ties
- broken by most-recent ``last_seen_at``."""
- now_ts = int(now if now is not None else time.time())
- config = config or {}
- cfg = {**DEFAULT_CONFIG, **config}
- if graph is not None:
- cfg["_graph"] = graph
- if not _has_explicit_threshold(config) and "failure_limit" in config:
- cfg["failure_threshold"] = _positive_int(
- config.get("failure_limit"), DEFAULT_CONFIG["failure_threshold"],
- )
- out: list[Diagnostic] = []
- for rule in _RULES:
- try:
- out.extend(rule(task, events, runs, now_ts, cfg))
- except Exception:
- # A broken rule must never 500 a whole /board request.
- continue
- severity_idx = {s: i for i, s in enumerate(SEVERITY_ORDER)}
- out.sort(key=lambda d: (-severity_idx.get(d.severity, -1), -(d.last_seen_at or 0)))
- return out
|