kanban_swarm.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. """Kanban Swarm v1: thin swarm topology helpers on top of Kanban.
  2. Deliberately no second scheduler — a small task graph written into the
  3. existing Kanban kernel:
  4. planning root (completed immediately)
  5. ├─ parallel specialist workers (ready)
  6. └─ verifier (todo until all workers done)
  7. └─ synthesizer (todo until verifier done)
  8. The shared blackboard is structured JSON comments on the root task, so all
  9. state lives in existing task_comments/task_events rows and the dashboard,
  10. notifier, slash command and dispatcher keep working without a new service.
  11. """
  12. from __future__ import annotations
  13. from dataclasses import asdict, dataclass, field
  14. import json
  15. import sqlite3
  16. import time
  17. from typing import Any, Iterable, Optional
  18. from hermes_cli import kanban_db as kb
  19. BLACKBOARD_PREFIX = "[swarm:blackboard] "
  20. @dataclass(frozen=True)
  21. class SwarmWorkerSpec:
  22. """A single parallel worker card in a swarm."""
  23. profile: str
  24. title: str
  25. body: str
  26. skills: list[str] = field(default_factory=list)
  27. priority: int = 0
  28. max_runtime_seconds: Optional[int] = None
  29. @dataclass(frozen=True)
  30. class SwarmCreated:
  31. """IDs produced by :func:`create_swarm`."""
  32. root_id: str
  33. worker_ids: list[str]
  34. verifier_id: str
  35. synthesizer_id: str
  36. def as_dict(self) -> dict[str, Any]:
  37. return asdict(self)
  38. def _require_text(value: str, field_name: str) -> str:
  39. text = (value or "").strip()
  40. if not text:
  41. raise ValueError(f"{field_name} is required")
  42. return text
  43. def _swarm_context(root_id: str, goal: str) -> str:
  44. return (
  45. f"\n\n## Swarm protocol\n- Swarm root / shared blackboard: `{root_id}`.\n- Read "
  46. f"sibling/parent handoffs from Kanban context before working.\n- Put machine-readable "
  47. f"facts in completion metadata.\n- Put cross-worker notes on the root task using "
  48. f"structured comments.\n- Goal: {goal.strip()}\n"
  49. )
  50. def _activate_root_inline(
  51. conn: sqlite3.Connection,
  52. root_id: str,
  53. *,
  54. summary: str,
  55. metadata: dict[str, Any],
  56. ) -> bool:
  57. """Inline blocked→done CAS flip + event insert for the swarm root.
  58. Runs INSIDE create_swarm's write_txn, so it must not call
  59. ``kb.complete_task`` (own transaction + post-commit side effects that
  60. would run while the outer txn can still roll back). The caller runs
  61. ``recompute_ready`` after the outer commit.
  62. """
  63. cur = conn.execute(
  64. """
  65. UPDATE tasks
  66. SET status = 'done',
  67. completed_at = ?,
  68. claim_lock = NULL,
  69. claim_expires= NULL,
  70. worker_pid = NULL
  71. WHERE id = ?
  72. AND status = 'blocked'
  73. """,
  74. (int(time.time()), root_id),
  75. )
  76. if cur.rowcount != 1:
  77. return False
  78. run_id = kb._synthesize_ended_run(conn, root_id, outcome="completed", summary=summary, metadata=metadata)
  79. kb._append_event(
  80. conn, root_id, "completed", {"result_len": 0, "summary": summary[:400] or None}, run_id=run_id,
  81. )
  82. return True
  83. def create_swarm(
  84. conn: sqlite3.Connection,
  85. *,
  86. goal: str,
  87. workers: Iterable[SwarmWorkerSpec],
  88. verifier_assignee: str,
  89. synthesizer_assignee: str,
  90. root_title: Optional[str] = None,
  91. verifier_title: str = "Verify swarm outputs",
  92. synthesizer_title: str = "Synthesize swarm outputs",
  93. tenant: Optional[str] = None,
  94. created_by: str = "swarm-orchestrator",
  95. workspace_kind: Optional[str] = None,
  96. workspace_path: Optional[str] = None,
  97. priority: int = 0,
  98. idempotency_key: Optional[str] = None,
  99. ) -> SwarmCreated:
  100. """Atomically create a durable, immediately dispatchable Kanban swarm."""
  101. activation_summary = "Swarm topology planned; root remains the shared blackboard."
  102. activated = False
  103. with kb.write_txn(conn):
  104. created = _create_swarm_uncommitted(
  105. conn, goal=goal, workers=workers, verifier_assignee=verifier_assignee,
  106. synthesizer_assignee=synthesizer_assignee, root_title=root_title,
  107. verifier_title=verifier_title, synthesizer_title=synthesizer_title, tenant=tenant,
  108. created_by=created_by, workspace_kind=workspace_kind, workspace_path=workspace_path,
  109. priority=priority, idempotency_key=idempotency_key,
  110. )
  111. root = kb.get_task(conn, created.root_id)
  112. if root is not None and root.status == "blocked":
  113. if not _activate_root_inline(
  114. conn,
  115. created.root_id,
  116. summary=activation_summary,
  117. metadata={
  118. "kind": "kanban_swarm_v1",
  119. "goal": goal.strip(),
  120. "worker_count": len(created.worker_ids),
  121. },
  122. ):
  123. raise RuntimeError("could not activate the completed swarm topology")
  124. activated = True
  125. if activated:
  126. # After commit: recompute_ready opens its own txn and must never run
  127. # under an open write_txn.
  128. kb.recompute_ready(conn)
  129. root = kb.get_task(conn, created.root_id)
  130. run = kb.latest_run(conn, created.root_id)
  131. kb._fire_kanban_lifecycle_hook(
  132. "kanban_task_completed",
  133. created.root_id,
  134. board=kb.get_current_board(),
  135. assignee=root.assignee if root else None,
  136. run_id=run.id if run else None,
  137. summary=activation_summary,
  138. )
  139. return created
  140. def _create_swarm_uncommitted(
  141. conn: sqlite3.Connection, *, goal: str, workers: Iterable[SwarmWorkerSpec],
  142. verifier_assignee: str, synthesizer_assignee: str, root_title: Optional[str],
  143. verifier_title: str, synthesizer_title: str, tenant: Optional[str], created_by: str,
  144. workspace_kind: Optional[str], workspace_path: Optional[str], priority: int, idempotency_key: Optional[str],
  145. ) -> SwarmCreated:
  146. """Create the swarm graph inside the caller's transaction: planning root
  147. (``blocked`` until the caller activates it), parallel workers, a verifier
  148. waiting on every worker, and a synthesizer waiting on the verifier."""
  149. goal = _require_text(goal, "goal")
  150. verifier_assignee = _require_text(verifier_assignee, "verifier_assignee")
  151. synthesizer_assignee = _require_text(synthesizer_assignee, "synthesizer_assignee")
  152. worker_specs = list(workers)
  153. if not worker_specs:
  154. raise ValueError("at least one worker is required")
  155. for i, spec in enumerate(worker_specs, start=1):
  156. _require_text(spec.profile, f"workers[{i}].profile")
  157. _require_text(spec.title, f"workers[{i}].title")
  158. common = dict(
  159. created_by=created_by, tenant=tenant,
  160. workspace_kind=workspace_kind, workspace_path=workspace_path,
  161. )
  162. root = kb.create_task(
  163. conn,
  164. title=root_title or f"Swarm: {goal.splitlines()[0][:80]}",
  165. body="Kanban Swarm v1 planning/root card. This card is completed "
  166. "immediately so parallel workers can start while it remains the "
  167. f"shared blackboard and audit anchor.\n\nGoal:\n{goal}",
  168. assignee=created_by,
  169. priority=priority,
  170. idempotency_key=idempotency_key,
  171. initial_status="blocked",
  172. **common,
  173. )
  174. # Idempotency may return an existing root: recover its topology from the
  175. # blackboard instead of duplicating the graph.
  176. existing = latest_blackboard(conn, root).get("topology")
  177. if isinstance(existing, dict):
  178. worker_ids = [str(x) for x in existing.get("worker_ids", []) if x]
  179. verifier_id = existing.get("verifier_id")
  180. synthesizer_id = existing.get("synthesizer_id")
  181. if worker_ids and verifier_id and synthesizer_id:
  182. return SwarmCreated(root, worker_ids, str(verifier_id), str(synthesizer_id))
  183. context_suffix = _swarm_context(root, goal)
  184. worker_ids = [
  185. kb.create_task(
  186. conn,
  187. title=spec.title,
  188. body=(spec.body or "") + context_suffix,
  189. assignee=spec.profile,
  190. parents=[root],
  191. priority=spec.priority or priority,
  192. skills=spec.skills or None,
  193. max_runtime_seconds=spec.max_runtime_seconds,
  194. **common,
  195. )
  196. for spec in worker_specs
  197. ]
  198. verifier = kb.create_task(
  199. conn,
  200. title=verifier_title,
  201. body=(
  202. "Review every worker handoff and blackboard update. Gate the swarm: "
  203. "complete only with metadata {\"gate\": \"pass\"} when evidence is "
  204. "sufficient; otherwise block with exact missing work."
  205. + context_suffix
  206. ),
  207. assignee=verifier_assignee,
  208. parents=worker_ids,
  209. priority=priority,
  210. skills=["requesting-code-review"],
  211. **common,
  212. )
  213. synthesizer = kb.create_task(
  214. conn,
  215. title=synthesizer_title,
  216. body=(
  217. "Synthesize the verified worker outputs into the final deliverable. "
  218. "Do not start until the verifier has passed the gate."
  219. + context_suffix
  220. ),
  221. assignee=synthesizer_assignee,
  222. parents=[verifier],
  223. priority=priority,
  224. skills=["humanizer"],
  225. **common,
  226. )
  227. created = SwarmCreated(root, worker_ids, verifier, synthesizer)
  228. post_blackboard_update(conn, root, author=created_by, key="topology", value=created.as_dict() | {"goal": goal})
  229. return created
  230. def post_blackboard_update(conn: sqlite3.Connection, root_id: str, *, author: str, key: str, value: Any) -> int:
  231. """Append one structured update to the swarm root blackboard."""
  232. _require_text(root_id, "root_id")
  233. author = _require_text(author, "author")
  234. key = _require_text(key, "key")
  235. payload = json.dumps({"key": key, "value": value}, ensure_ascii=False, sort_keys=True)
  236. return kb.add_comment(conn, root_id, author=author, body=BLACKBOARD_PREFIX + payload)
  237. def latest_blackboard(conn: sqlite3.Connection, root_id: str) -> dict[str, Any]:
  238. """Merge structured blackboard comments on a root card. Later comments
  239. replace earlier values for the same key; ``_authors`` records the author
  240. of the winning value for traceability."""
  241. merged: dict[str, Any] = {}
  242. authors: dict[str, str] = {}
  243. for comment in kb.list_comments(conn, root_id):
  244. body = comment.body or ""
  245. if not body.startswith(BLACKBOARD_PREFIX):
  246. continue
  247. try:
  248. payload = json.loads(body[len(BLACKBOARD_PREFIX):])
  249. except json.JSONDecodeError:
  250. continue
  251. key = payload.get("key")
  252. if not isinstance(key, str) or not key:
  253. continue
  254. merged[key] = payload.get("value")
  255. authors[key] = comment.author
  256. if authors:
  257. merged["_authors"] = authors
  258. return merged
  259. def parse_worker_arg(raw: str) -> SwarmWorkerSpec:
  260. """Parse CLI ``--worker profile:title[:skill,skill]`` values."""
  261. parts = [p.strip() for p in raw.split(":", 2)]
  262. if len(parts) < 2:
  263. raise ValueError("worker must be profile:title or profile:title:skill,skill")
  264. skills = [s.strip() for s in parts[2].split(",") if s.strip()] if len(parts) == 3 and parts[2] else []
  265. return SwarmWorkerSpec(profile=parts[0], title=parts[1], body=parts[1], skills=skills)