worktree_archive.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. """Restorable snapshots of git worktrees: ``refs/hermes/archive/<tree>``.
  2. A snapshot is two commits built with a TEMPORARY index (the tree's real index and working files
  3. are only read): ``I`` = the staged state (tree of a copy of the real index, parent ``HEAD``) and
  4. ``W`` = the whole working state (tracked + non-ignored untracked files, parents ``HEAD`` and
  5. ``I``), the same shape ``git stash -u`` produces. ``W`` carries the metadata (original path,
  6. branch, HEAD, reason, skipped files) as JSON in its message, and the ref keeps every object
  7. reachable, so the tree and even its branch can be deleted without losing work.
  8. Bytes are stored raw: every tracked/untracked regular file is re-hashed with
  9. ``hash-object --no-filters`` and restored by writing the blob bytes directly, so no attribute
  10. source (``.gitattributes``, ``$GIT_COMMON_DIR/info/attributes``, ``core.attributesFile``) and no
  11. clean/smudge/eol conversion can change them; a restore writes back exactly the archived bytes. Untracked files over a size cap are skipped and listed
  12. (so ``.git`` never balloons with build artefacts the ignore rules missed), and a tree with a
  13. skipped file is never removed; an untracked embedded repository cannot be captured at all and
  14. fails the snapshot, which keeps the tree. Ignored files are not captured and are deleted with the
  15. tree; an embedded repository inside an ignored directory keeps it (``worktree_retention``).
  16. """
  17. from __future__ import annotations
  18. import json
  19. import logging
  20. import os
  21. import re
  22. import shutil
  23. import subprocess
  24. import time
  25. import uuid
  26. from dataclasses import dataclass, field
  27. from pathlib import Path
  28. from typing import Optional
  29. logger = logging.getLogger(__name__)
  30. ARCHIVE_PREFIX = "refs/hermes/archive/"
  31. _SUBJECT = "hermes worktree archive: "
  32. _IDENTITY = {
  33. "GIT_AUTHOR_NAME": "Hermes worktree archive", "GIT_AUTHOR_EMAIL": "hermes-archive@localhost",
  34. "GIT_COMMITTER_NAME": "Hermes worktree archive", "GIT_COMMITTER_EMAIL": "hermes-archive@localhost",
  35. }
  36. # Raw bytes in, raw bytes out: no line-ending conversion on either side of the round trip.
  37. _RAW_CONFIG = (("core.autocrlf", "false"), ("core.safecrlf", "false"), ("core.eol", "lf"))
  38. class ArchiveError(RuntimeError):
  39. """The snapshot (or restore) could not be completed; callers must keep the tree."""
  40. @dataclass
  41. class Archive:
  42. ref: str
  43. name: str
  44. sha: str
  45. created: float
  46. meta: dict = field(default_factory=dict)
  47. staged: str = "" # staged_state() of exactly the index this archive captured
  48. def _env(cwd, index: Optional[str] = None) -> dict:
  49. from hermes_cli._subprocess_compat import noninteractive_repo_git_env
  50. env = noninteractive_repo_git_env(cwd)
  51. if env is None:
  52. raise ArchiveError("git filter discovery failed")
  53. env.update(_IDENTITY)
  54. start = int(env["GIT_CONFIG_COUNT"])
  55. for offset, (key, value) in enumerate(_RAW_CONFIG):
  56. env[f"GIT_CONFIG_KEY_{start + offset}"] = key
  57. env[f"GIT_CONFIG_VALUE_{start + offset}"] = value
  58. env["GIT_CONFIG_COUNT"] = str(start + len(_RAW_CONFIG))
  59. if index:
  60. env["GIT_INDEX_FILE"] = index
  61. return env
  62. def _run(args, cwd, env, timeout: float = 300, stdin_text: Optional[str] = None):
  63. return subprocess.run(["git", *args], cwd=str(cwd), env=env, capture_output=True, text=True, # windows-footgun: ok -- encoding passed on the next line
  64. encoding="utf-8", errors="surrogateescape", timeout=timeout,
  65. input=stdin_text, stdin=None if stdin_text is not None else subprocess.DEVNULL,
  66. check=False)
  67. def _ok(args, cwd, env, **kw) -> str:
  68. result = _run(args, cwd, env, **kw)
  69. if result.returncode != 0:
  70. raise ArchiveError(f"git {args[0]} failed: {result.stderr.strip()[-300:]}")
  71. return result.stdout
  72. def _raw_attrs(args, cwd, env, empty_tree: str, **kw) -> str:
  73. """Run a content-converting command with attributes read from the empty tree (git >= 2.40);
  74. older git lacks ``--attr-source`` and falls back to the checkout's own attributes."""
  75. result = _run(["--attr-source", empty_tree, *args], cwd, env, **kw)
  76. if result.returncode != 0 and "attr-source" in result.stderr:
  77. result = _run(args, cwd, env, **kw)
  78. if result.returncode != 0:
  79. raise ArchiveError(f"git {args[0]} failed: {result.stderr.strip()[-300:]}")
  80. return result.stdout
  81. def _run_bytes(args, cwd, env, data: bytes, timeout: float = 300) -> bytes:
  82. result = subprocess.run(["git", *args], cwd=str(cwd), env=env, capture_output=True, input=data,
  83. timeout=timeout, check=False)
  84. if result.returncode != 0:
  85. raise ArchiveError(f"git {args[0]} failed: {result.stderr.decode('utf-8', 'replace').strip()[-300:]}")
  86. return result.stdout
  87. def _regular_entries(cwd, env) -> list[tuple[str, str, str]]:
  88. """``(mode, sha, path)`` for every regular-file entry of the index ``env`` points at."""
  89. out = _ok(["ls-files", "-s", "-z"], cwd, env)
  90. entries = []
  91. for record in out.split("\0"):
  92. meta, _, path = record.partition("\t")
  93. parts = meta.split()
  94. if path and len(parts) == 3 and parts[0] in ("100644", "100755"):
  95. entries.append((parts[0], parts[1], path))
  96. return entries
  97. def _rehash_raw(wt: Path, tenv) -> None:
  98. """Replace every regular-file blob in the temporary index with the file's exact bytes.
  99. ``add`` runs content conversion from every attribute source, and ``--attr-source`` only
  100. replaces ``.gitattributes``: ``info/attributes`` and ``core.attributesFile`` still apply.
  101. ``hash-object --no-filters`` skips all of it.
  102. """
  103. entries = [e for e in _regular_entries(wt, tenv) if os.path.isfile(wt / e[2]) and not os.path.islink(wt / e[2])]
  104. if not entries:
  105. return
  106. shas: list[str] = []
  107. # --stdin-paths reads one path per line and C-unquotes a leading '"': hash those one by one.
  108. batch = [p for _m, _s, p in entries if "\n" not in p and not p.startswith('"')]
  109. if batch:
  110. data = "".join(p + "\n" for p in batch).encode("utf-8", "surrogateescape")
  111. shas = _run_bytes(["hash-object", "-w", "--no-filters", "--stdin-paths"], wt, tenv, data).decode().split()
  112. by_path = dict(zip(batch, shas))
  113. for _m, _s, path in entries:
  114. if path not in by_path:
  115. raw = (wt / path).read_bytes()
  116. by_path[path] = _run_bytes(["hash-object", "-w", "--no-filters", "--stdin"], wt, tenv, raw).decode().strip()
  117. info = "".join(f"{mode} {by_path[path]}\t{path}\0" for mode, _s, path in entries)
  118. _run_bytes(["update-index", "-z", "--index-info"], wt, tenv, info.encode("utf-8", "surrogateescape"))
  119. def _write_raw(dest: Path, env) -> None:
  120. """Overwrite every regular file of the index with its blob's exact bytes (no smudge/eol)."""
  121. entries = _regular_entries(dest, env)
  122. if not entries:
  123. return
  124. out = _run_bytes(["cat-file", "--batch"], dest, env, "".join(f"{s}\n" for _m, s, _p in entries).encode())
  125. pos = 0
  126. for _mode, _sha, path in entries:
  127. header_end = out.index(b"\n", pos)
  128. size = int(out[pos:header_end].split()[2])
  129. body = out[header_end + 1:header_end + 1 + size]
  130. pos = header_end + 1 + size + 1
  131. target = dest / path
  132. if target.is_symlink():
  133. target.unlink()
  134. target.parent.mkdir(parents=True, exist_ok=True)
  135. mode = target.stat().st_mode if target.exists() else None
  136. target.write_bytes(body)
  137. if mode is not None:
  138. os.chmod(target, mode)
  139. def ref_safe(name: str) -> str:
  140. """A tree name as a single valid ref component."""
  141. safe = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("-.") or "tree"
  142. return safe.replace("..", "-").removesuffix(".lock")
  143. def _untracked(wt: Path, env) -> list[str]:
  144. out = _ok(["ls-files", "--others", "--exclude-standard", "-z"], wt, env)
  145. return [p for p in out.split("\0") if p]
  146. SUBMODULE_REASON = "submodule "
  147. def submodule_work(wt_path, *, gitlinks: bool = True) -> Optional[str]:
  148. """Why a submodule keeps *wt_path*, or None. Read-only.
  149. An archive records a submodule only as its ``160000`` gitlink, and removing the tree deletes
  150. its per-worktree ``modules/`` repositories (kept by ``git submodule deinit``, private branches
  151. included). Until submodules are archived recursively: no tree with a gitlink is archived
  152. (*gitlinks*), and no tree whose ``modules/`` holds a repository or whose gitlink is checked out
  153. is removed at all.
  154. """
  155. wt = Path(wt_path)
  156. env = {**_env(wt), "GIT_OPTIONAL_LOCKS": "0"}
  157. modules = Path(_ok(["rev-parse", "--absolute-git-dir"], wt, env).strip()) / "modules"
  158. if modules.is_dir() and any(modules.iterdir()):
  159. return f"{SUBMODULE_REASON}repository in {modules} would be deleted with the tree"
  160. staged = _ok(["ls-files", "--stage", "-z"], wt, env)
  161. links = [e.split("\t", 1)[1] for e in staged.split("\0") if e.startswith("160000 ") and "\t" in e]
  162. kept = [p for p in links if gitlinks or (wt / p / ".git").exists()]
  163. return f"{SUBMODULE_REASON}{kept[0]} cannot be archived (gitlink only)" if kept else None
  164. def staged_state(wt_path) -> str:
  165. """Digest of the semantic staged entries (mode, blob id, stage, path) of *wt_path*.
  166. A stat-only index refresh (``git status``) leaves it unchanged; any ``git add``/``apply
  167. --cached``/``rm --cached`` changes it. Raises :class:`ArchiveError` when git fails.
  168. """
  169. wt = Path(wt_path)
  170. return _digest_index(wt, _env(wt))
  171. def _digest_index(wt: Path, env) -> str:
  172. import hashlib
  173. env = {**env, "GIT_OPTIONAL_LOCKS": "0"}
  174. return hashlib.sha256(_ok(["ls-files", "--stage", "-z"], wt, env).encode("utf-8", "surrogateescape")).hexdigest()
  175. def archive_preflight(wt_path, max_untracked_mb: float = 50) -> Optional[tuple[str, str]]:
  176. """``(keep-action, reason)`` when archiving and removing *wt_path* would lose something
  177. (``worktree_retention.removal_blocker``: submodule, embedded repository, per-worktree ref,
  178. an untracked file over the size cap), else None. Read-only."""
  179. from hermes_cli.worktree_retention import KEEP_UNARCHIVABLE, removal_blocker
  180. try:
  181. wt = Path(wt_path)
  182. skipped = _split_oversized(wt, _untracked(wt, _env(wt)), max_untracked_mb)
  183. except (ArchiveError, OSError, subprocess.SubprocessError) as exc:
  184. return KEEP_UNARCHIVABLE, str(exc)
  185. return removal_blocker(wt, skipped=skipped, gitlinks=True)
  186. def _split_oversized(wt: Path, untracked: list[str], max_mb: float) -> list[dict]:
  187. if max_mb <= 0:
  188. return []
  189. cap = max_mb * 1024 * 1024
  190. skipped = []
  191. for rel in untracked:
  192. try:
  193. size = os.lstat(wt / rel).st_size
  194. except OSError:
  195. continue
  196. if size > cap:
  197. skipped.append({"path": rel, "bytes": size})
  198. return skipped
  199. def _create_ref(repo_root: str, name: str, sha: str, env) -> str:
  200. base = ARCHIVE_PREFIX + ref_safe(name)
  201. stamp = time.strftime("%Y%m%dT%H%M%S")
  202. for ref in (base, f"{base}--{stamp}", *(f"{base}--{stamp}-{i}" for i in range(2, 50))):
  203. # Empty old value = create only: two racing sweeps never overwrite each other's archive.
  204. if _run(["update-ref", "-m", "hermes worktree archive", ref, sha, ""], repo_root, env).returncode == 0:
  205. return ref
  206. raise ArchiveError(f"could not create an archive ref for {name}")
  207. def snapshot_worktree(repo_root: str, wt_path, *, reason: str, max_untracked_mb: float = 50) -> Archive:
  208. """Archive the full working state of *wt_path*; raises :class:`ArchiveError` on any failure.
  209. Never writes the tree's real index or working files (temporary ``GIT_INDEX_FILE`` only).
  210. """
  211. wt = Path(wt_path)
  212. env = _env(wt)
  213. # Shared refusal point (sweep, exit-time Archive, subagent finalization).
  214. unsafe = submodule_work(wt)
  215. if unsafe:
  216. raise ArchiveError(unsafe)
  217. head = _ok(["rev-parse", "--verify", "HEAD^{commit}"], wt, env).strip()
  218. branch = _run(["symbolic-ref", "--short", "-q", "HEAD"], wt, env).stdout.strip() or None
  219. git_dir = Path(_ok(["rev-parse", "--absolute-git-dir"], wt, env).strip())
  220. untracked = _untracked(wt, env)
  221. embedded = [p for p in untracked if p.endswith("/")]
  222. if embedded:
  223. raise ArchiveError(f"untracked embedded repository {embedded[0].rstrip('/')} cannot be archived")
  224. skipped = _split_oversized(wt, untracked, max_untracked_mb)
  225. token = f"hermes-archive-{os.getpid()}-{uuid.uuid4().hex[:8]}"
  226. tmp_index, pathspec = git_dir / f"{token}.index", git_dir / f"{token}.pathspec"
  227. try:
  228. tenv = _env(wt, index=str(tmp_index))
  229. if (git_dir / "index").is_file():
  230. shutil.copyfile(git_dir / "index", tmp_index)
  231. else:
  232. _ok(["read-tree", head], wt, tenv)
  233. staged = _digest_index(wt, tenv)
  234. # Staged state first (fails on an unmerged index: a conflicted tree is never archived).
  235. index_tree = _ok(["write-tree"], wt, tenv).strip()
  236. empty_tree = _ok(["mktree"], wt, env, stdin_text="").strip()
  237. specs = [":(top)"] + [f":(top,exclude,literal){s['path']}" for s in skipped]
  238. pathspec.write_bytes(("\0".join(specs) + "\0").encode("utf-8", "surrogateescape"))
  239. _raw_attrs(["add", "-A", "--pathspec-from-file", str(pathspec), "--pathspec-file-nul"],
  240. wt, tenv, empty_tree)
  241. # Every blob from the exact bytes on disk (add converted them per attributes).
  242. _rehash_raw(wt, tenv)
  243. work_tree = _ok(["write-tree"], wt, tenv).strip()
  244. index_commit = _ok(["commit-tree", index_tree, "-p", head, "-m", f"index of {wt.name}"], wt, env).strip()
  245. meta = {"version": 1, "name": wt.name, "path": str(wt), "repo_root": str(repo_root),
  246. "branch": branch, "head": head, "index_commit": index_commit,
  247. "created": int(time.time()), "reason": reason, "skipped_untracked": skipped}
  248. message = f"{_SUBJECT}{wt.name}\n\n{json.dumps(meta, indent=1, ensure_ascii=False)}\n"
  249. sha = _ok(["commit-tree", work_tree, "-p", head, "-p", index_commit], wt, env,
  250. stdin_text=message).strip()
  251. finally:
  252. for leftover in (tmp_index, tmp_index.with_name(tmp_index.name + ".lock"), pathspec):
  253. leftover.unlink(missing_ok=True)
  254. ref = _create_ref(repo_root, wt.name, sha, env)
  255. logger.info("Archived worktree %s -> %s (%s)", wt, ref, reason)
  256. return Archive(ref=ref, name=ref[len(ARCHIVE_PREFIX):], sha=sha, created=meta["created"], meta=meta,
  257. staged=staged)
  258. def _parse_meta(body: str) -> dict:
  259. start = body.find("{")
  260. try:
  261. meta = json.loads(body[start:]) if start >= 0 else {}
  262. except ValueError:
  263. meta = {}
  264. return meta if isinstance(meta, dict) else {}
  265. def list_archives(repo_root: str) -> list[Archive]:
  266. """Every archive in *repo_root*, newest first. Read-only; [] when git fails."""
  267. try:
  268. env = _env(repo_root)
  269. out = _ok(["for-each-ref", "--sort=-creatordate",
  270. "--format=%(refname)%00%(objectname)%00%(creatordate:unix)%00%(contents:body)%00",
  271. ARCHIVE_PREFIX], repo_root, env, timeout=60)
  272. except (ArchiveError, OSError, subprocess.SubprocessError):
  273. return []
  274. archives = []
  275. for record in out.split("\0\n"):
  276. parts = record.split("\0")
  277. if len(parts) < 4 or not parts[0].startswith(ARCHIVE_PREFIX):
  278. continue
  279. meta = _parse_meta(parts[3])
  280. created = float(meta.get("created") or parts[2] or 0)
  281. archives.append(Archive(ref=parts[0], name=parts[0][len(ARCHIVE_PREFIX):], sha=parts[1],
  282. created=created, meta=meta))
  283. return archives
  284. def find_archive(repo_root: str, name: str) -> Optional[Archive]:
  285. """Exact archive name/ref first, else the newest archive of the tree called *name*."""
  286. archives = list_archives(repo_root)
  287. wanted = name.removeprefix(ARCHIVE_PREFIX)
  288. for arc in archives:
  289. if arc.name == wanted:
  290. return arc
  291. return next((a for a in archives if a.meta.get("name") == name or a.name == ref_safe(name)), None)
  292. def prune_expired_archives(repo_root: str, retention_days: float, *, now: Optional[float] = None,
  293. dry_run: bool = False) -> list[str]:
  294. """Delete archive refs older than *retention_days* (0 keeps them forever)."""
  295. if not retention_days or retention_days <= 0:
  296. return []
  297. cutoff = (now or time.time()) - retention_days * 86400
  298. expired = [a for a in list_archives(repo_root) if a.created and a.created < cutoff]
  299. if dry_run or not expired:
  300. return [a.ref for a in expired]
  301. env = _env(repo_root)
  302. deleted = []
  303. for arc in expired:
  304. if _run(["update-ref", "-d", arc.ref, arc.sha], repo_root, env).returncode == 0:
  305. deleted.append(arc.ref)
  306. return deleted
  307. def _branch_tip(repo_root: str, branch: str, env) -> Optional[str]:
  308. result = _run(["rev-parse", "--verify", "-q", f"refs/heads/{branch}^{{commit}}"], repo_root, env)
  309. return result.stdout.strip() if result.returncode == 0 else None
  310. def _checked_out_branches(repo_root: str, env) -> set:
  311. out = _run(["worktree", "list", "--porcelain"], repo_root, env).stdout
  312. return {line[len("branch refs/heads/"):] for line in out.splitlines() if line.startswith("branch refs/heads/")}
  313. def _add_args(repo_root: str, dest: Path, branch: Optional[str], head: str, env) -> tuple[list, bool]:
  314. """``worktree add`` argv restoring *branch* when that is safe -> (argv, detached)."""
  315. base = ["worktree", "add", "--no-checkout"]
  316. if branch:
  317. tip = _branch_tip(repo_root, branch, env)
  318. if tip is None:
  319. return [*base, "-b", branch, str(dest), head], False
  320. if tip == head and branch not in _checked_out_branches(repo_root, env):
  321. return [*base, str(dest), branch], False
  322. return [*base, "--detach", str(dest), head], True
  323. def restore_archive(repo_root: str, name: str, target=None) -> dict:
  324. """Recreate the archived tree at its original path (or *target*); raises ArchiveError."""
  325. arc = find_archive(repo_root, name)
  326. if arc is None:
  327. raise ArchiveError(f"no archive named {name}")
  328. meta = arc.meta
  329. head = meta.get("head") or f"{arc.sha}^1"
  330. index_commit = meta.get("index_commit") or f"{arc.sha}^2"
  331. dest = Path(target or meta.get("path") or Path(repo_root) / ".worktrees" / arc.name).absolute()
  332. if dest.exists() and (not dest.is_dir() or any(dest.iterdir())):
  333. raise ArchiveError(f"{dest} already exists; pass a different path")
  334. env = _env(repo_root)
  335. args, detached = _add_args(repo_root, dest, meta.get("branch"), head, env)
  336. dest.parent.mkdir(parents=True, exist_ok=True)
  337. _ok(args, repo_root, env)
  338. wenv = _env(dest)
  339. empty_tree = _ok(["mktree"], dest, wenv, stdin_text="").strip()
  340. _ok(["read-tree", f"{arc.sha}^{{tree}}"], dest, wenv)
  341. _raw_attrs(["checkout-index", "-a", "-f"], dest, wenv, empty_tree)
  342. _write_raw(dest, wenv) # checkout-index still converted per info/attributes
  343. _ok(["read-tree", f"{index_commit}^{{tree}}"], dest, wenv)
  344. _run(["update-index", "-q", "--refresh"], dest, wenv)
  345. from hermes_cli.worktree_registry import record_worktree_repo
  346. record_worktree_repo(repo_root)
  347. return {"path": str(dest), "branch": meta.get("branch"), "detached": detached, "head": head,
  348. "ref": arc.ref, "skipped_untracked": meta.get("skipped_untracked") or []}