kanban_ops.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410
  1. """Dispatcher and maintenance verbs for ``hermes kanban``: ``dispatch``,
  2. ``daemon`` (deprecated standalone loop), ``tail``/``watch`` event streaming,
  3. ``gc`` and ``repair``.
  4. """
  5. from __future__ import annotations
  6. import argparse
  7. import json
  8. import os
  9. import sys
  10. import time
  11. from pathlib import Path
  12. from hermes_cli import kanban_db as kb
  13. from hermes_cli import kanban_db_connect as kbc
  14. from hermes_cli import kanban_db_dispatch as kbd
  15. from hermes_cli import kanban_db_workspace as kbw
  16. from hermes_cli.kanban_output import _err, _fmt_ts, _print_json
  17. def _kanban_config() -> dict:
  18. """``config.yaml`` ``kanban:`` section, or ``{}`` when config can't be loaded."""
  19. try:
  20. from hermes_cli.config import load_config
  21. cfg = load_config()
  22. return (cfg.get("kanban", {}) if isinstance(cfg, dict) else {}) or {}
  23. except Exception:
  24. return {}
  25. def _poll_loop(interval: float, tick) -> int:
  26. """Run ``tick()`` every ``interval`` seconds (floor 0.1) until Ctrl-C."""
  27. try:
  28. while True:
  29. tick()
  30. time.sleep(max(0.1, interval))
  31. except KeyboardInterrupt:
  32. print("\n(stopped)")
  33. return 0
  34. def _cmd_tail(args: argparse.Namespace) -> int:
  35. last_id = 0
  36. print(f"Tailing events for {args.task_id}. Ctrl-C to stop.")
  37. def tick():
  38. nonlocal last_id
  39. with kbc.connect_closing() as conn:
  40. events = kb.list_events(conn, args.task_id)
  41. for e in events:
  42. if e.id > last_id:
  43. pl = f" {e.payload}" if e.payload else ""
  44. print(f"[{_fmt_ts(e.created_at)}] {e.kind}{pl}", flush=True)
  45. last_id = e.id
  46. return _poll_loop(args.interval, tick)
  47. def _cmd_dispatch(args: argparse.Namespace) -> int:
  48. # Honour kanban.default_assignee, kanban.max_in_progress,
  49. # kanban.max_in_progress_per_profile and kanban.max_spawn with the same
  50. # semantics as the gateway dispatch path.
  51. try:
  52. from hermes_cli.config import load_config
  53. _cfg = load_config()
  54. _kanban_cfg = _cfg.get("kanban", {}) if isinstance(_cfg, dict) else {}
  55. default_assignee = (_kanban_cfg.get("default_assignee") or "").strip() or None
  56. max_in_progress_per_profile = kbd._positive_int(
  57. _kanban_cfg.get("max_in_progress_per_profile"), None
  58. )
  59. # Memory-derived default when unset — same fallback the gateway applies.
  60. max_in_progress = kbd.resolve_max_in_progress(
  61. kbd._positive_int(_kanban_cfg.get("max_in_progress"), None)
  62. )
  63. # CLI --max is the more explicit signal, so it wins over kanban.max_spawn.
  64. cli_max = getattr(args, "max", None)
  65. max_spawn = (
  66. cli_max if cli_max is not None else kbd._positive_int(_kanban_cfg.get("max_spawn"), None)
  67. )
  68. except Exception:
  69. default_assignee = max_in_progress_per_profile = max_in_progress = None
  70. max_spawn = getattr(args, "max", None)
  71. with kbc.connect_closing() as conn:
  72. res = kbd.dispatch_once(
  73. conn,
  74. dry_run=args.dry_run,
  75. max_spawn=max_spawn,
  76. max_in_progress=max_in_progress,
  77. failure_limit=getattr(args, "failure_limit", kbd.DEFAULT_FAILURE_LIMIT),
  78. default_assignee=default_assignee,
  79. max_in_progress_per_profile=max_in_progress_per_profile,
  80. )
  81. if getattr(args, "json", False):
  82. _print_json({
  83. **{k: getattr(res, k)
  84. for k in ("reclaimed", "crashed", "timed_out", "stale", "auto_blocked", "promoted",
  85. "reaped_terminal_workers")},
  86. "spawned": [
  87. {"task_id": tid, "assignee": who, "workspace": ws} for (tid, who, ws) in res.spawned
  88. ],
  89. "skipped_unassigned": res.skipped_unassigned,
  90. "skipped_nonspawnable": res.skipped_nonspawnable,
  91. "skipped_per_profile_capped": [
  92. {"task_id": tid, "assignee": who, "current": current}
  93. for (tid, who, current) in res.skipped_per_profile_capped
  94. ],
  95. "auto_assigned_default": res.auto_assigned_default,
  96. "respawn_guarded": [
  97. {"task_id": tid, "reason": reason}
  98. for (tid, reason) in res.respawn_guarded
  99. ],
  100. "rate_limited": res.rate_limited,
  101. "skipped_locked": res.skipped_locked,
  102. "memory_pressure": res.memory_pressure,
  103. }, ascii=True)
  104. return 0
  105. print(f"Reclaimed: {res.reclaimed}")
  106. if res.reaped_terminal_workers:
  107. print(f"Reaped workers of finished tasks: {', '.join(res.reaped_terminal_workers)}")
  108. for label, items in (
  109. ("Crashed: ", res.crashed),
  110. ("Timed out: ", res.timed_out),
  111. ("Stale: ", res.stale),
  112. ("Auto-blocked:", res.auto_blocked),
  113. ):
  114. print(f"{label} {len(items)}")
  115. if items:
  116. print(f" {', '.join(items)}")
  117. print(f"Promoted: {res.promoted}")
  118. print(f"Spawned: {len(res.spawned)}")
  119. tag = " (dry)" if args.dry_run else ""
  120. for tid, who, ws in res.spawned:
  121. print(f" - {tid} -> {who} @ {ws or '-'}{tag}")
  122. if res.auto_assigned_default:
  123. print(
  124. f"Auto-assigned to kanban.default_assignee={default_assignee!r}: "
  125. f"{', '.join(res.auto_assigned_default)}"
  126. )
  127. if res.skipped_unassigned:
  128. print(f"Skipped (unassigned): {', '.join(res.skipped_unassigned)}")
  129. for tid, who, current in res.skipped_per_profile_capped:
  130. print(f"Deferred ({who} at per-profile cap, {current} running): {tid}")
  131. if res.skipped_nonspawnable:
  132. print(
  133. f"Skipped (non-spawnable assignee — terminal lane, OK): "
  134. f"{', '.join(res.skipped_nonspawnable)}"
  135. )
  136. for tid, reason in res.respawn_guarded:
  137. print(f"Guarded ({reason}): {tid}")
  138. if res.rate_limited:
  139. print(f"Rate-limited (released to ready, no failure counted): {', '.join(res.rate_limited)}")
  140. if res.skipped_locked:
  141. print("Skipped: another dispatcher holds this board's lock (no writes this tick)")
  142. if res.memory_pressure:
  143. print(f"Memory pressure {res.memory_pressure}: new workers restricted this tick")
  144. return 0
  145. _DAEMON_DEPRECATED = (
  146. "hermes kanban daemon: DEPRECATED — the dispatcher now runs\ninside the gateway. To use "
  147. "kanban:\n\n hermes gateway start # starts the gateway + embedded dispatcher\n\nReady "
  148. "tasks will be picked up on the next dispatcher tick\n(default: every 60 seconds). Configure "
  149. "via config.yaml:\n\n kanban:\n dispatch_in_gateway: true # default\n "
  150. "dispatch_interval_seconds: 60\n failure_limit: 2 # consecutive non-success "
  151. "attempts before auto-block\n\nRunning both the gateway AND this standalone daemon will\nrace "
  152. "for claims. If you truly need the old standalone\ndaemon (no gateway available), rerun with "
  153. "--force."
  154. )
  155. def _cmd_daemon(args: argparse.Namespace) -> int:
  156. """Deprecated — the dispatcher now runs inside the gateway. Kept so old
  157. scripts/systemd units get a clear migration message; ``--force`` (hidden
  158. from --help) keeps the standalone loop for hosts that truly cannot run the
  159. gateway. The default path exits 2 so nobody accidentally runs two
  160. dispatchers against the same kanban.db."""
  161. if not getattr(args, "force", False):
  162. return _err(_DAEMON_DEPRECATED, 2)
  163. # Init before printing "started" so the DB path is right and init errors
  164. # surface immediately.
  165. kb.init_db()
  166. pidfile = getattr(args, "pidfile", None)
  167. if pidfile:
  168. try:
  169. Path(pidfile).parent.mkdir(parents=True, exist_ok=True)
  170. Path(pidfile).write_text(str(os.getpid()), encoding="utf-8")
  171. except OSError as exc:
  172. print(f"warning: could not write pidfile {pidfile}: {exc}", file=sys.stderr)
  173. verbose = bool(getattr(args, "verbose", False))
  174. print(
  175. f"Kanban dispatcher running STANDALONE via --force (interval={args.interval}s, "
  176. f"pid={os.getpid()}). Ctrl-C to stop. NOTE: if a gateway is also running with "
  177. f"dispatch_in_gateway=true (default), you have two dispatchers racing for claims.",
  178. file=sys.stderr,
  179. )
  180. # Health telemetry: warn when every tick finds ready work but spawns
  181. # nothing (broken profile, PATH drift, missing venv, credential loss) —
  182. # the per-task breaker auto-blocks quietly, so the operator needs a signal.
  183. HEALTH_WINDOW = 6 # ticks (default 30s at interval=5)
  184. health_state = {"bad_ticks": 0, "last_warn_at": 0}
  185. def _ready_queue_nonempty() -> bool:
  186. """Is there a ready+assigned+unclaimed task the dispatcher would spawn for?
  187. Control-plane lanes pulled via ``claim_task`` are correctly idle, not stuck."""
  188. try:
  189. with kbc.connect_closing() as conn:
  190. return kbd.has_spawnable_ready(conn)
  191. except Exception:
  192. return False
  193. def _on_tick(res):
  194. ready_pending = bool(res.skipped_unassigned) or _ready_queue_nonempty()
  195. if ready_pending and not res.spawned:
  196. health_state["bad_ticks"] += 1
  197. else:
  198. health_state["bad_ticks"] = 0
  199. # Warn once per HEALTH_WINDOW bad ticks, at most every 5 minutes.
  200. if health_state["bad_ticks"] >= HEALTH_WINDOW:
  201. now = int(time.time())
  202. if now - health_state["last_warn_at"] >= 300:
  203. held = kbd.describe_suppression([res])
  204. held = f" Last tick held back: {held}." if held else ""
  205. print(
  206. f"[{_fmt_ts(now)}] WARN dispatcher stuck: ready queue non-empty for "
  207. f"{health_state['bad_ticks']} consecutive ticks but 0 workers spawned "
  208. f"successfully.{held} Check profile health (venv, PATH, credentials) and `hermes "
  209. f"kanban list --status ready` / `hermes kanban list --status blocked` for "
  210. f"recent spawn_failed tasks.",
  211. file=sys.stderr, flush=True,
  212. )
  213. health_state["last_warn_at"] = now
  214. if not verbose:
  215. return
  216. did_work = (
  217. res.reclaimed or res.crashed or res.timed_out or res.promoted
  218. or res.spawned or res.auto_blocked or res.stale
  219. )
  220. if did_work:
  221. print(
  222. f"[{_fmt_ts(int(time.time()))}] reclaimed={res.reclaimed} "
  223. f"crashed={len(res.crashed)} timed_out={len(res.timed_out)} stale={len(res.stale)} "
  224. f"promoted={res.promoted} spawned={len(res.spawned)} "
  225. f"auto_blocked={len(res.auto_blocked)}",
  226. flush=True,
  227. )
  228. try:
  229. kbd.run_daemon(
  230. interval=args.interval,
  231. max_spawn=args.max,
  232. failure_limit=getattr(args, "failure_limit", kbd.DEFAULT_FAILURE_LIMIT),
  233. on_tick=_on_tick,
  234. )
  235. finally:
  236. if pidfile:
  237. try:
  238. Path(pidfile).unlink()
  239. except OSError:
  240. pass
  241. print("(dispatcher stopped)")
  242. return 0
  243. def _cmd_watch(args: argparse.Namespace) -> int:
  244. """Live-stream task_events to the terminal."""
  245. kinds = {k.strip() for k in args.kinds.split(",") if k.strip()} if args.kinds else None
  246. print(f"Watching kanban events (initial board '{kb.get_current_board()}'). Ctrl-C to stop.", flush=True)
  247. # Seed cursor at the latest id so we don't replay history.
  248. with kbc.connect_closing() as conn:
  249. cursor = int(conn.execute("SELECT COALESCE(MAX(id), 0) AS m FROM task_events").fetchone()["m"])
  250. def tick():
  251. nonlocal cursor
  252. with kbc.connect_closing() as conn:
  253. rows = conn.execute(
  254. "SELECT e.id, e.task_id, e.kind, e.payload, e.created_at, t.assignee, "
  255. "t.tenant FROM task_events e LEFT JOIN tasks t ON t.id = e.task_id WHERE e.id > ? "
  256. "ORDER BY e.id ASC LIMIT 200",
  257. (cursor,),
  258. ).fetchall()
  259. for r in rows:
  260. cursor = max(cursor, int(r["id"]))
  261. if (kinds and r["kind"] not in kinds) or (args.assignee and r["assignee"] != args.assignee) \
  262. or (args.tenant and r["tenant"] != args.tenant):
  263. continue
  264. try:
  265. payload = json.loads(r["payload"]) if r["payload"] else None
  266. except Exception:
  267. payload = None
  268. pl = f" {payload}" if payload else ""
  269. print(
  270. f"[{_fmt_ts(kb._to_epoch(r['created_at']))}] {r['task_id']:10s} "
  271. f"{r['kind']:18s} (@{r['assignee'] or '-'}){pl}",
  272. flush=True,
  273. )
  274. return _poll_loop(args.interval, tick)
  275. def _cmd_gc(args: argparse.Namespace) -> int:
  276. """Remove archived tasks' scratch workspaces, old events, and old worker logs."""
  277. import shutil
  278. event_days = getattr(args, "event_retention_days", 30)
  279. log_days = getattr(args, "log_retention_days", 30)
  280. if event_days < 0 or log_days < 0:
  281. return _err("kanban gc: retention days must be >= 0 (0 disables that sweep)", 2)
  282. scratch_root = kb.workspaces_root()
  283. removed_ws = 0
  284. with kbc.connect_closing() as conn:
  285. rows = conn.execute(
  286. "SELECT id, workspace_kind, workspace_path, branch_name FROM tasks "
  287. "WHERE status = 'archived'"
  288. ).fetchall()
  289. for row in rows:
  290. if row["workspace_kind"] == "worktree":
  291. # Backstop for worktrees that escaped the completion/archive hook.
  292. # Same safety predicate: only clean, fully-pushed worktrees go.
  293. wt_path = row["workspace_path"]
  294. if wt_path and Path(wt_path).is_dir():
  295. if kbw._defer_shared_worktree_cleanup(conn, row["id"], wt_path):
  296. continue
  297. kbw._cleanup_worktree_workspace(row["id"], wt_path, row["branch_name"])
  298. if not Path(wt_path).is_dir():
  299. removed_ws += 1
  300. continue
  301. if row["workspace_kind"] != "scratch":
  302. continue
  303. path = Path(row["workspace_path"] or (scratch_root / row["id"]))
  304. # Same containment predicate as completion cleanup (#28818): strictly below a
  305. # managed root, never the root itself (which holds every task's scratch dir).
  306. # Cheap existence/symlink check first: most rows were already cleaned at
  307. # completion, and rmtree refuses a symlink (so it must not be counted).
  308. if not path.is_dir() or path.is_symlink() or not kbw._is_managed_scratch_path(path):
  309. continue
  310. if kbw._defer_shared_workspace_cleanup(conn, row["id"], path):
  311. continue
  312. shutil.rmtree(path, ignore_errors=True)
  313. if not path.exists():
  314. removed_ws += 1
  315. removed_events = 0
  316. if event_days:
  317. with kbc.connect_closing() as conn:
  318. removed_events = kb.gc_events(conn, older_than_seconds=event_days * 24 * 3600)
  319. removed_logs = kb.gc_worker_logs(older_than_seconds=log_days * 24 * 3600) if log_days else 0
  320. print(f"GC complete: {removed_ws} workspace(s), "
  321. f"{removed_events} event row(s), {removed_logs} log file(s) removed")
  322. return 0
  323. def _cmd_repair(args: argparse.Namespace) -> int:
  324. """Integrity check + narrow index-REINDEX auto-repair. Dispatched BEFORE
  325. the auto ``kb.init_db()`` (init refuses corrupt DBs). Exit 0 = healthy /
  326. repaired / no DB file, 1 = still corrupt."""
  327. try:
  328. report = kbc.repair_db()
  329. except Exception as exc: # locked/busy probe, unexpected I/O
  330. return _err(f"kanban repair: {exc}")
  331. if getattr(args, "json", False):
  332. _print_json({
  333. "status": report.status,
  334. "db_path": str(report.db_path),
  335. "messages": report.messages,
  336. "post_repair_messages": report.post_repair_messages,
  337. "backup_path": str(report.backup_path) if report.backup_path else None,
  338. "reindexed": report.reindexed,
  339. }, ascii=True)
  340. return 0 if report.status in {"ok", "repaired", "missing"} else 1
  341. if report.status == "missing":
  342. print(f"No kanban DB at {report.db_path} — nothing to repair.")
  343. return 0
  344. if report.status == "ok":
  345. print(f"{report.db_path}: integrity_check ok — no repair needed.")
  346. return 0
  347. if report.status == "repaired":
  348. print(f"{report.db_path}: repaired.")
  349. print(f" reindexed: {', '.join(report.reindexed)}")
  350. if report.backup_path:
  351. print(f" pre-repair backup: {report.backup_path}")
  352. print(" integrity_check now ok.")
  353. return 0
  354. # still corrupt
  355. def err(line: str) -> None:
  356. print(line, file=sys.stderr)
  357. err(f"{report.db_path}: CORRUPT.")
  358. for line in (report.messages or [])[:10]:
  359. err(f" {line}")
  360. if report.reindexed:
  361. err(f" REINDEX ({', '.join(report.reindexed)}) attempted but integrity_check is still failing:")
  362. for line in (report.post_repair_messages or [])[:10]:
  363. err(f" {line}")
  364. else:
  365. err(" Not an index-only failure — automatic REINDEX repair does not apply (fail-closed).")
  366. if report.backup_path:
  367. err(f" corrupt copy quarantined at: {report.backup_path}")
  368. err(
  369. " Recover manually (copy kanban.db aside FIRST, then run "
  370. "`sqlite3 <copy> \".recover\"` into a fresh file — never against "
  371. "the live path, a WAL-reset-vulnerable sqlite3 CLI can corrupt it "
  372. "further) or move the file aside to start a new board."
  373. )
  374. return 1