worktree_retention.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. """What a worktree holds that neither git nor an archive can bring back.
  2. Every path that deletes a tree (the sweep, the classic clean-tree pruner, exit-time Archive,
  3. subagent finalization, the attended ``hermes worktree prune``) asks :func:`removal_blocker`
  4. first; anything it names keeps the tree. The rule: a tree may only be removed when everything
  5. that is not reproducible is in git or in the archive. Not reproducible:
  6. * an embedded repository anywhere below the tree, ignored or not (``build/tool/.git``, a nested
  7. worktree): its private commits live only on disk (``keep-unarchivable``);
  8. * a commit held only by a per-worktree ref (``refs/worktree/*``, ``refs/bisect/*``,
  9. ``refs/rewritten/*``), which dies with the tree's admin dir (``keep-worktree-refs``);
  10. * an untracked file the snapshot skipped for size (``keep-large-untracked``).
  11. Ignored files (``.env`` copies, caches, build and test output) are disposable: they are not
  12. archived and are deleted with the tree. Only an embedded repository inside an ignored directory
  13. keeps it, because its commits may exist nowhere else.
  14. :func:`hidden_index_edits` finds skip-worktree / assume-unchanged files whose bytes differ from
  15. the index: ``git status`` never reports them, so dirty checks must add them.
  16. """
  17. from __future__ import annotations
  18. import os
  19. import subprocess
  20. from dataclasses import dataclass, field
  21. from pathlib import Path
  22. from typing import Optional
  23. # Dependency stores and caches: never descended, not even to look for an embedded repository.
  24. _OPAQUE_DIRS = frozenset({
  25. "node_modules", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
  26. ".tox", ".next", ".turbo", ".cache", ".gradle",
  27. })
  28. _PER_WORKTREE_REFS = ("refs/worktree/", "refs/bisect/", "refs/rewritten/")
  29. _MAX_ENTRIES = 200_000 # per tree; past this an embedded repository cannot be ruled out -> keep
  30. _NAMED = 5 # offenders named in a reason
  31. KEEP_UNARCHIVABLE = "keep-unarchivable"
  32. KEEP_WORKTREE_REFS = "keep-worktree-refs"
  33. KEEP_LARGE_UNTRACKED = "keep-large-untracked"
  34. KEEP_SUBMODULE_WORK = "keep-submodule-work"
  35. def _env(wt: Path) -> dict:
  36. from hermes_cli.worktree_archive import _env as archive_env
  37. return {**archive_env(wt), "GIT_OPTIONAL_LOCKS": "0"}
  38. def _git(args, wt: Path, env, timeout: float = 120) -> str:
  39. from hermes_cli.worktree_archive import _ok
  40. return _ok(args, wt, env, timeout=timeout)
  41. @dataclass
  42. class _Scan:
  43. embedded: list = field(default_factory=list)
  44. entries: int = 0
  45. truncated: bool = False
  46. def _walk(scan: _Scan, wt: Path, rel_dir: str) -> None:
  47. """Below ignored *rel_dir*: record nested ``.git`` entries. Symlinks are not followed."""
  48. stack = [rel_dir]
  49. while stack and not scan.truncated:
  50. rel = stack.pop()
  51. try:
  52. entries = list(os.scandir(wt / rel))
  53. except OSError:
  54. continue
  55. for entry in entries:
  56. scan.entries += 1
  57. if scan.entries > _MAX_ENTRIES:
  58. scan.truncated = True
  59. return
  60. child = f"{rel}/{entry.name}"
  61. if entry.name == ".git":
  62. scan.embedded.append(rel)
  63. continue
  64. if entry.is_dir(follow_symlinks=False) and entry.name not in _OPAQUE_DIRS:
  65. stack.append(child)
  66. def _ignored_entries(wt: Path, env) -> list[str]:
  67. """Ignored paths, whole ignored directories collapsed to ``dir/``. ``ls-files --directory``
  68. hits an internal error on some trees (git 2.53: "directory entry not superset of prefix");
  69. ``status --ignored`` reports the same set."""
  70. from hermes_cli.worktree_archive import ArchiveError
  71. try:
  72. out = _git(["ls-files", "--others", "--ignored", "--exclude-standard", "--directory",
  73. "--no-empty-directory", "-z"], wt, env)
  74. return [p for p in out.split("\0") if p]
  75. except ArchiveError:
  76. out = _git(["status", "--porcelain", "-z", "--ignored=traditional", "--untracked-files=normal",
  77. "--no-renames"], wt, env)
  78. return [e[3:] for e in out.split("\0") if e.startswith("!! ")]
  79. def scan_tree(wt_path) -> _Scan:
  80. """Embedded repositories of *wt_path*, ignored or not. Read-only.
  81. Untracked non-ignored repositories are reported by ``ls-files`` itself (``dir/``); ignored
  82. ones hide inside collapsed ignored directories, so those are walked (bounded; dependency
  83. stores are never descended). Raises ``ArchiveError`` when git fails.
  84. """
  85. wt = Path(wt_path)
  86. env = _env(wt)
  87. scan = _Scan()
  88. # Without --directory git descends untracked dirs and lists each embedded repository as "dir/".
  89. untracked = _git(["ls-files", "--others", "--exclude-standard", "-z"], wt, env)
  90. scan.embedded += [p.rstrip("/") for p in untracked.split("\0") if p.endswith("/")]
  91. for rel in _ignored_entries(wt, env):
  92. rel_clean = rel.rstrip("/")
  93. path = wt / rel_clean
  94. if not rel.endswith("/") or path.is_symlink() or rel_clean.rsplit("/", 1)[-1] in _OPAQUE_DIRS:
  95. continue
  96. if (path / ".git").exists():
  97. scan.embedded.append(rel_clean)
  98. continue
  99. _walk(scan, wt, rel_clean)
  100. if scan.truncated:
  101. break
  102. return scan
  103. def hidden_index_edits(wt_path, env=None) -> list[str]:
  104. """Skip-worktree / assume-unchanged paths whose disk state differs from the index.
  105. ``git status`` trusts both flags and reports nothing. A skip-worktree file missing from disk
  106. is a sparse checkout, not an edit; a missing assume-unchanged file is a deletion. Raises
  107. ``ArchiveError`` when git fails.
  108. """
  109. wt = Path(wt_path)
  110. env = env or _env(wt)
  111. flagged = []
  112. for record in _git(["ls-files", "-v", "-s", "-z"], wt, env).split("\0"):
  113. meta, _, rel = record.partition("\t")
  114. parts = meta.split()
  115. if not rel or len(parts) != 4 or not (parts[0] == "S" or parts[0].islower()):
  116. continue
  117. path = wt / rel
  118. if not os.path.lexists(path):
  119. if parts[0] != "S":
  120. flagged.append(rel) # assume-unchanged file deleted
  121. continue
  122. flagged.append((rel, parts[1], parts[2]))
  123. edits = [f for f in flagged if isinstance(f, str)]
  124. for rel, mode, sha in (f for f in flagged if not isinstance(f, str)):
  125. path = wt / rel
  126. if mode == "120000":
  127. data = os.readlink(path).encode("utf-8", "surrogateescape") if path.is_symlink() else None
  128. else:
  129. data = None if path.is_symlink() or not path.is_file() else path.read_bytes()
  130. if data is None:
  131. edits.append(rel) # type changed
  132. continue
  133. if _hash(wt, env, data, rel, raw=True) != sha and _hash(wt, env, data, rel, raw=False) != sha:
  134. edits.append(rel)
  135. return edits
  136. def _hash(wt: Path, env, data: bytes, rel: str, *, raw: bool) -> str:
  137. args = ["hash-object", "--stdin"] + (["--no-filters"] if raw else [f"--path={rel}"])
  138. result = subprocess.run(["git", *args], cwd=str(wt), env=env, input=data, capture_output=True,
  139. timeout=60, check=False)
  140. return result.stdout.decode("ascii", "replace").strip() if result.returncode == 0 else ""
  141. def private_worktree_refs(wt_path, env=None) -> list[str]:
  142. """Per-worktree refs of *wt_path* holding commits no branch, tag, remote or archive reaches.
  143. They live in the tree's admin dir and vanish with it. Raises ``ArchiveError`` when git fails.
  144. """
  145. wt = Path(wt_path)
  146. env = env or _env(wt)
  147. out = _git(["for-each-ref", "--format=%(refname)%00%(objectname)", *_PER_WORKTREE_REFS], wt, env)
  148. private = []
  149. for line in out.splitlines():
  150. name, _, sha = line.partition("\0")
  151. if sha and _git(["rev-list", "--max-count=1", sha, "--not", "--branches", "--tags", "--remotes",
  152. "--glob=refs/hermes/archive/*"], wt, env).strip():
  153. private.append(name)
  154. return private
  155. def _named(paths: list, more: bool = False) -> str:
  156. extra = len(paths) - _NAMED
  157. tail = f" (+{extra} more)" if extra > 0 else (" (+more)" if more else "")
  158. return ", ".join(paths[:_NAMED]) + tail
  159. def removal_blocker(wt_path, *, skipped: Optional[list] = None, gitlinks: bool = False
  160. ) -> Optional[tuple[str, str]]:
  161. """``(keep-action, reason)`` when removing *wt_path* would lose something, else None.
  162. *skipped* is the snapshot's ``skipped_untracked`` list (or the oversize files a preflight
  163. would skip). *gitlinks* refuses any submodule (archive paths); without it only a checked-out
  164. or retained module repository blocks. Any failure to verify blocks. Read-only.
  165. """
  166. from hermes_cli.worktree_archive import ArchiveError, submodule_work
  167. try:
  168. why = submodule_work(wt_path, gitlinks=gitlinks)
  169. if why:
  170. return KEEP_SUBMODULE_WORK, why
  171. scan = scan_tree(wt_path)
  172. if scan.embedded:
  173. return KEEP_UNARCHIVABLE, f"embedded repository {scan.embedded[0]} cannot be archived"
  174. if scan.truncated:
  175. return KEEP_UNARCHIVABLE, f"over {_MAX_ENTRIES} ignored entries; cannot rule out an embedded repository"
  176. refs = private_worktree_refs(wt_path)
  177. if refs:
  178. return KEEP_WORKTREE_REFS, f"per-worktree ref {_named(refs)} holds commits no branch has"
  179. if skipped:
  180. return KEEP_LARGE_UNTRACKED, f"untracked file(s) over the archive size cap: {_named([s['path'] for s in skipped])}"
  181. except (ArchiveError, OSError, subprocess.SubprocessError) as exc:
  182. return KEEP_UNARCHIVABLE, f"could not verify the tree is safe to remove: {exc}"
  183. return None