kanban_transfer.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371
  1. """Kanban board export / import — move a whole board between machines.
  2. Backs ``hermes kanban export|import``, the ``/boards/{slug}/export`` and
  3. ``/boards/import`` REST endpoints, and the desktop board switcher. Archive
  4. layout (``<slug>.tar.gz``, one top-level dir named for the source slug):
  5. ``manifest.json`` (format/version/provenance/counts), ``board.json`` (display
  6. metadata, machine-local fields stripped), ``kanban.db`` (consistent snapshot),
  7. ``attachments/<task>/…`` (unless --no-attachments), ``logs/<task>.log`` (only
  8. with --include-logs).
  9. Two things make this more than ``tar czf`` of the board directory: the DB is
  10. live (WAL mode, dispatcher may be mid-write) so export uses SQLite's online
  11. backup instead of a file copy that would miss the ``-wal`` sidecar; and rows
  12. carry machine-local state (claims, PIDs, heartbeats, absolute paths, gateway
  13. chat subscriptions, session ids) that would import a stranger's claims or push
  14. events into a stranger's Telegram thread — scrubbed on export and re-scrubbed
  15. on import (an archive is untrusted); see :func:`_scrub_local_state` and
  16. :func:`_relocate_imported_rows`. Imports always land as a **new** board (slug
  17. auto-suffixes on collision), never ``default``, so the importer can ignore the
  18. default board's split on-disk layout.
  19. """
  20. from __future__ import annotations
  21. import contextlib
  22. import json
  23. import shutil
  24. import sqlite3
  25. import tempfile
  26. import time
  27. from pathlib import Path
  28. from typing import Any, Optional
  29. from hermes_cli import kanban_db as kb
  30. from hermes_cli import kanban_db_connect as kbc
  31. from hermes_cli.archive_safe import (
  32. archive_root_dirs,
  33. copy_regular_files,
  34. make_targz,
  35. safe_extract_targz,
  36. )
  37. ARCHIVE_FORMAT = "hermes-kanban-board"
  38. ARCHIVE_FORMAT_VERSION = 1
  39. # Statuses from which the dispatcher can still act on a task. A task whose
  40. # workspace cannot be rebuilt on this machine is parked in ``triage`` only
  41. # if it is in one of these — terminal and already-parked tasks are left
  42. # alone rather than having their history rewritten.
  43. _DISPATCHABLE_STATUSES = ("ready", "running", "todo", "scheduled")
  44. _COUNTED_TABLES = ("tasks", "task_links", "task_comments", "task_events", "task_runs", "task_attachments")
  45. def _placeholders(items) -> str:
  46. return ", ".join("?" * len(items))
  47. # ---------------------------------------------------------------------------
  48. # Export
  49. # ---------------------------------------------------------------------------
  50. def _snapshot_db(source: Path, target: Path) -> None:
  51. """Consistent copy of ``source`` via the online-backup API (a file copy
  52. would miss pages still in the ``-wal`` sidecar and could tear)."""
  53. with contextlib.closing(sqlite3.connect(str(source))) as src, \
  54. contextlib.closing(sqlite3.connect(str(target))) as dst:
  55. src.backup(dst)
  56. def _scrub_local_state(conn: sqlite3.Connection) -> None:
  57. """Strip machine-local runtime state (claims, PIDs, and above all the
  58. gateway chat ids subscribed to task events). Caller owns the transaction.
  59. Run on export and again on import (an archive is untrusted input)."""
  60. conn.execute("DELETE FROM kanban_notify_subs")
  61. conn.execute(
  62. """
  63. UPDATE tasks
  64. SET claim_lock = NULL,
  65. claim_expires = NULL,
  66. worker_pid = NULL,
  67. current_run_id = NULL,
  68. last_heartbeat_at = NULL,
  69. session_id = NULL,
  70. project_id = NULL,
  71. consecutive_failures = 0,
  72. last_failure_error = NULL
  73. """
  74. )
  75. # A task caught mid-run is not running anywhere the importer can see.
  76. # Send it back to the queue rather than shipping a phantom claim.
  77. conn.execute("UPDATE tasks SET status = 'ready' WHERE status = 'running'")
  78. conn.execute(
  79. """
  80. UPDATE task_runs
  81. SET status = 'released',
  82. outcome = COALESCE(outcome, 'reclaimed'),
  83. ended_at = COALESCE(ended_at, ?),
  84. last_heartbeat_at = NULL
  85. WHERE status = 'running'
  86. """,
  87. (int(time.time()),),
  88. )
  89. conn.execute("UPDATE task_runs SET claim_lock = NULL, worker_pid = NULL")
  90. def _write_json(path: Path, payload: dict[str, Any]) -> None:
  91. path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
  92. def _count_rows(conn: sqlite3.Connection) -> dict[str, int]:
  93. return {t: int(conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]) for t in _COUNTED_TABLES}
  94. def export_board(
  95. board: Optional[str],
  96. output_path: str,
  97. *,
  98. include_attachments: bool = True,
  99. include_logs: bool = False,
  100. ) -> dict[str, Any]:
  101. """Export ``board`` to a ``tar.gz`` (suffix optional on ``output_path``);
  102. returns a summary dict. Workspaces are never included — large,
  103. machine-local, rebuilt on demand."""
  104. slug = kb._normalize_board_slug(board) or kb.get_current_board()
  105. if not kb.board_exists(slug):
  106. raise ValueError(f"board {slug!r} does not exist")
  107. db_path = kb.kanban_db_path(slug)
  108. if not db_path.exists():
  109. raise FileNotFoundError(f"board {slug!r} has no database at {db_path}")
  110. base = str(Path(output_path).expanduser()).removesuffix(".tar.gz").removesuffix(".tgz")
  111. Path(base).parent.mkdir(parents=True, exist_ok=True)
  112. with tempfile.TemporaryDirectory() as tmpdir:
  113. staged = Path(tmpdir) / slug
  114. staged.mkdir(parents=True)
  115. _snapshot_db(db_path, staged / "kanban.db")
  116. # The snapshot is a private file with no other writers, so plain
  117. # commit/close is enough — no need for the board DB's WAL dance.
  118. with contextlib.closing(sqlite3.connect(str(staged / "kanban.db"))) as snapshot:
  119. _scrub_local_state(snapshot)
  120. snapshot.commit()
  121. counts = _count_rows(snapshot)
  122. meta = kb.read_board_metadata(slug)
  123. # Both name a location on the exporting machine; the importer
  124. # resolves its own.
  125. meta.pop("db_path", None)
  126. meta["default_workdir"] = None
  127. meta["project_id"] = None
  128. _write_json(staged / "board.json", meta)
  129. attachments = copy_regular_files(kb.attachments_root(slug), staged / "attachments") if include_attachments else 0
  130. logs = copy_regular_files(kb.worker_logs_dir(slug), staged / "logs") if include_logs else 0
  131. from hermes_cli.version_info import get_version_info
  132. manifest = {
  133. "format": ARCHIVE_FORMAT,
  134. "format_version": ARCHIVE_FORMAT_VERSION,
  135. "board": slug,
  136. "board_name": meta.get("name") or slug,
  137. "exported_at": int(time.time()),
  138. "hermes_version": get_version_info().base_version,
  139. "includes": {"attachments": bool(include_attachments), "logs": bool(include_logs)},
  140. "counts": {**counts, "attachment_files": attachments, "log_files": logs},
  141. }
  142. _write_json(staged / "manifest.json", manifest)
  143. archive = make_targz(base, tmpdir, slug)
  144. return {
  145. "board": slug,
  146. "archive": archive,
  147. "size": Path(archive).stat().st_size,
  148. "counts": manifest["counts"],
  149. }
  150. # ---------------------------------------------------------------------------
  151. # Import
  152. # ---------------------------------------------------------------------------
  153. def _available_slug(preferred: str) -> str:
  154. """``preferred`` or the first free ``<preferred>-N``. ``default`` always
  155. exists, so a default-board export lands as ``default-2``."""
  156. if not kb.board_exists(preferred):
  157. return preferred
  158. # Leave headroom for the suffix inside the 64-char slug limit.
  159. stem = preferred[:58].rstrip("-_") or "board"
  160. n = 2
  161. while kb.board_exists(f"{stem}-{n}"):
  162. n += 1
  163. return f"{stem}-{n}"
  164. def _read_manifest(root: Path) -> dict[str, Any]:
  165. path = root / "manifest.json"
  166. if not path.exists():
  167. raise ValueError("archive is not a Hermes kanban board export (no manifest.json)")
  168. try:
  169. manifest = json.loads(path.read_text(encoding="utf-8-sig"))
  170. except json.JSONDecodeError as exc:
  171. raise ValueError(f"archive manifest is not valid JSON: {exc}") from exc
  172. if not isinstance(manifest, dict) or manifest.get("format") != ARCHIVE_FORMAT:
  173. raise ValueError(
  174. "archive is not a Hermes kanban board export "
  175. f"(format={manifest.get('format') if isinstance(manifest, dict) else None!r})"
  176. )
  177. version = manifest.get("format_version")
  178. if not isinstance(version, int) or version > ARCHIVE_FORMAT_VERSION:
  179. raise ValueError(
  180. f"archive format version {version!r} is newer than this Hermes "
  181. f"understands (max {ARCHIVE_FORMAT_VERSION}) — update Hermes and retry"
  182. )
  183. return manifest
  184. def _read_board_metadata(path: Path) -> dict[str, Any]:
  185. """Read an archive's ``board.json``, tolerating a missing/broken file."""
  186. try:
  187. raw = json.loads(path.read_text(encoding="utf-8-sig"))
  188. except (OSError, json.JSONDecodeError):
  189. return {}
  190. return raw if isinstance(raw, dict) else {}
  191. def _relocate_imported_rows(conn: sqlite3.Connection, slug: str) -> tuple[dict[str, int], list[str]]:
  192. """Re-anchor an imported board's rows to this machine; returns ``(stats, warnings)``.
  193. * Attachment rows are repointed at this board's tree; rows whose blob
  194. did not travel (``--no-attachments``) are dropped, since a dangling row
  195. breaks download in every UI.
  196. * Workspace paths are cleared. ``scratch`` regenerates on next claim;
  197. dispatchable ``dir``/``worktree`` tasks are parked in ``triage``,
  198. otherwise the dispatcher claims them, fails to build a workspace, and
  199. burns them into the failure breaker.
  200. * Runtime state is scrubbed again (untrusted input, one UPDATE).
  201. """
  202. warnings: list[str] = []
  203. now = int(time.time())
  204. attachments_dir = kb.attachments_root(slug)
  205. with kb.write_txn(conn):
  206. _scrub_local_state(conn)
  207. dropped = rehomed = 0
  208. for row in conn.execute("SELECT id, task_id, stored_path FROM task_attachments").fetchall():
  209. landed = attachments_dir / row["task_id"] / Path(row["stored_path"]).name
  210. if landed.is_file():
  211. conn.execute("UPDATE task_attachments SET stored_path = ? WHERE id = ?", (str(landed), row["id"]))
  212. rehomed += 1
  213. else:
  214. conn.execute("DELETE FROM task_attachments WHERE id = ?", (row["id"],))
  215. dropped += 1
  216. if dropped:
  217. warnings.append(f"{dropped} attachment record(s) dropped — the files were not in the archive")
  218. parked = [
  219. r["id"]
  220. for r in conn.execute(
  221. "SELECT id FROM tasks WHERE workspace_kind IN ('dir', 'worktree') "
  222. f"AND status IN ({_placeholders(_DISPATCHABLE_STATUSES)})",
  223. _DISPATCHABLE_STATUSES,
  224. ).fetchall()
  225. ]
  226. conn.execute("UPDATE tasks SET workspace_path = NULL, branch_name = NULL")
  227. if parked:
  228. conn.execute(f"UPDATE tasks SET status = 'triage' WHERE id IN ({_placeholders(parked)})", parked)
  229. warnings.append(
  230. f"{len(parked)} task(s) moved to triage — their workspace was a directory or git "
  231. f"worktree on the exporting machine and needs to be pointed somewhere on this one"
  232. )
  233. for row in conn.execute("SELECT id FROM tasks").fetchall():
  234. conn.execute(
  235. "INSERT INTO task_events (task_id, run_id, kind, payload, created_at) "
  236. "VALUES (?, NULL, 'imported', ?, ?)",
  237. (row["id"], json.dumps({"board": slug, "parked": row["id"] in parked}, ensure_ascii=False), now),
  238. )
  239. return {"attachments": rehomed, "parked": len(parked)}, warnings
  240. def import_board(
  241. archive_path: str,
  242. slug: Optional[str] = None,
  243. *,
  244. activate: bool = False,
  245. ) -> dict[str, Any]:
  246. """Import an archive as a NEW board (``slug`` overrides the archive's;
  247. either way it auto-suffixes if taken). Returns a summary dict."""
  248. archive = Path(archive_path).expanduser()
  249. if not archive.exists():
  250. raise FileNotFoundError(f"archive not found: {archive}")
  251. roots = archive_root_dirs(archive)
  252. if len(roots) != 1:
  253. raise ValueError("a kanban board archive must contain exactly one top-level directory")
  254. archive_root = roots.pop()
  255. with tempfile.TemporaryDirectory() as tmpdir:
  256. staging = Path(tmpdir)
  257. safe_extract_targz(archive, staging)
  258. extracted = staging / archive_root
  259. manifest = _read_manifest(extracted)
  260. staged_db = extracted / "kanban.db"
  261. if not staged_db.is_file():
  262. raise ValueError("archive is missing kanban.db")
  263. requested = kb._normalize_board_slug(slug or manifest.get("board") or archive_root)
  264. if not requested:
  265. raise ValueError(
  266. "cannot determine a board name from the archive — pass one "
  267. "explicitly with --as <slug>"
  268. )
  269. target = _available_slug(requested)
  270. staged_meta = _read_board_metadata(extracted / "board.json")
  271. board_root = kb.board_dir(target)
  272. board_root.mkdir(parents=True, exist_ok=True)
  273. shutil.move(str(staged_db), str(board_root / "kanban.db"))
  274. for tree in ("attachments", "logs"):
  275. src = extracted / tree
  276. if src.is_dir():
  277. shutil.move(str(src), str(board_root / tree))
  278. # Rewritten rather than moved across: the archive's copy names a slug
  279. # and a workdir that belong to the exporting machine.
  280. name = str(staged_meta.get("name") or manifest.get("board_name") or target)
  281. kb.write_board_metadata(
  282. target,
  283. name=name,
  284. description=str(staged_meta.get("description") or ""),
  285. icon=str(staged_meta.get("icon") or ""),
  286. color=str(staged_meta.get("color") or ""),
  287. archived=False,
  288. )
  289. # Bring the imported schema up to this install's version before the
  290. # relocation pass writes to it.
  291. kb.init_db(board=target)
  292. with kbc.connect_closing(board=target) as conn:
  293. stats, warnings = _relocate_imported_rows(conn, target)
  294. counts = _count_rows(conn)
  295. if activate:
  296. kb.set_current_board(target)
  297. return {
  298. "board": target,
  299. "requested_board": requested,
  300. "renamed": target != requested,
  301. "name": name,
  302. "path": str(kb.board_dir(target)),
  303. "db_path": str(kb.kanban_db_path(target)),
  304. "source": {
  305. "board": manifest.get("board"),
  306. "exported_at": manifest.get("exported_at"),
  307. "hermes_version": manifest.get("hermes_version"),
  308. },
  309. "counts": counts,
  310. "attachments_restored": stats["attachments"],
  311. "tasks_parked": stats["parked"],
  312. "warnings": warnings,
  313. "activated": bool(activate),
  314. }