git_pack_tidy.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. """Progressive, time-boxed cleanup of a partial clone's promisor packfiles (#129712).
  2. A partial clone downloads objects when git first needs them, and git asks for them without telling
  3. the server what it already has, so the same trees (and, for history downloads, commits) land again
  4. and again, each download as its own packfile that nothing removes: ``.git`` reached 39 GiB on a
  5. ~1 GiB repository. A full ``git gc`` over that is all-or-nothing and on a big checkout runs for tens
  6. of minutes, so ``hermes update`` instead spends at most ``TIDY_BUDGET_SECONDS`` per run and every
  7. unit of work it finishes is kept:
  8. 1. Erase promisor packs whose every object is also in another promisor pack. Nothing is lost, and
  9. nothing depends on the remote still serving it: the rule is proven against the local packs alone.
  10. 2. While more than ``PACK_COUNT_TARGET`` promisor packs remain, merge the smallest few into one
  11. with ``git pack-objects --stdin-packs``; the merged pack holds each object once.
  12. Each erase and each merge is atomic for git (a pack exists for git only while both its ``.idx`` and
  13. ``.pack`` do), so an update that runs out of budget, fails or is killed leaves the rest for the next.
  14. """
  15. from __future__ import annotations
  16. import contextlib
  17. import json
  18. import logging
  19. import mmap
  20. import os
  21. import shutil
  22. import struct
  23. import subprocess
  24. import time
  25. from dataclasses import dataclass
  26. from pathlib import Path
  27. from typing import Dict, Iterable, Iterator, List, Optional
  28. from hermes_cli._subprocess_compat import (
  29. NO_LAZY_FETCH_ENV,
  30. noninteractive_git_env,
  31. windows_hide_flags,
  32. )
  33. logger = logging.getLogger(__name__)
  34. TIDY_BUDGET_SECONDS = 60
  35. PACK_COUNT_TARGET = 50
  36. # A pack this young may still be in use by the git that just fetched it; index-pack also writes a
  37. # new pack's .promisor/.keep before its .pack, so a young marker without a payload may be a live fetch.
  38. _MIN_PACK_AGE_SECONDS = 60 * 60
  39. _MERGE_BATCH_BYTES = 256 * 1024 * 1024
  40. # Kept in .git/: packs proven to hold an object no other pack has (valid until a new pack appears,
  41. # since removing packs never adds copies) and the merge batch size, halved after a merge that ran out
  42. # of time. Remembering both is what guarantees progress: no update repeats work an earlier one finished.
  43. _STATE_FILE = "hermes-pack-tidy.json"
  44. _STATE_VERSION = 2 # v1 could record a pack as unique after comparing it with only some packs
  45. # Two tidies at once could each erase a pack because the other still has its copies.
  46. _LOCK_FILE = "hermes-pack-tidy.lock"
  47. # Payload first: git loads a pack only when both .idx and .pack exist, so once .pack is gone the pack is
  48. # gone for git, and a failure on .pack itself (a reader still maps it on Windows) changes nothing.
  49. _PACK_SUFFIXES = (".pack", ".idx", ".rev", ".bitmap", ".mtimes", ".promisor", ".keep")
  50. _IDX_V2 = b"\xfftOc\x00\x00\x00\x02"
  51. _FANOUT = 8
  52. _NAMES = _FANOUT + 256 * 4
  53. @dataclass
  54. class TidyResult:
  55. erased: int = 0
  56. freed_bytes: int = 0
  57. merged: int = 0
  58. packs_left: int = 0
  59. out_of_time: bool = False
  60. class _Index:
  61. """A version-2 SHA-1 pack index (what a GitHub clone writes), searched in place."""
  62. def __init__(self, idx: Path) -> None:
  63. with open(idx, "rb") as fh:
  64. self._map = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
  65. if self._map[:8] != _IDX_V2:
  66. self.close()
  67. raise ValueError(f"{idx.name}: not a v2 SHA-1 pack index")
  68. self.count = struct.unpack_from(">I", self._map, _FANOUT + 255 * 4)[0]
  69. def oids(self) -> Iterator[bytes]:
  70. for i in range(self.count): # lazily: a big index must not run past the deadline before its first check
  71. yield self._map[_NAMES + i * 20: _NAMES + i * 20 + 20]
  72. def __contains__(self, oid: bytes) -> bool:
  73. lo = struct.unpack_from(">I", self._map, _FANOUT + (oid[0] - 1) * 4)[0] if oid[0] else 0
  74. hi = struct.unpack_from(">I", self._map, _FANOUT + oid[0] * 4)[0]
  75. while lo < hi:
  76. mid = (lo + hi) // 2
  77. name = self._map[_NAMES + mid * 20: _NAMES + mid * 20 + 20]
  78. if name == oid:
  79. return True
  80. if name < oid:
  81. lo = mid + 1
  82. else:
  83. hi = mid
  84. return False
  85. def close(self) -> None:
  86. self._map.close() # Windows refuses to unlink a mapped file
  87. def _git_env() -> dict[str, str]:
  88. return {**noninteractive_git_env(), **NO_LAZY_FETCH_ENV}
  89. def _promisor_packs(pack_dir: Path) -> list[Path]:
  90. return [p for p in pack_dir.glob("pack-*.pack")
  91. if p.with_suffix(".promisor").exists() and p.with_suffix(".idx").exists()]
  92. def _pinned(pack: Path, cutoff: float) -> bool:
  93. """Whether a ``.keep`` pins the pack. fetch-pack and index-pack write ``<cmd> <pid> on <host>`` and
  94. delete it once the fetch has updated its refs, so an old one of theirs belongs to a fetch that was
  95. killed; it pins nothing, and git's own repack never touches the pack while it exists."""
  96. keep = pack.with_suffix(".keep")
  97. try:
  98. return not (keep.read_text(encoding="utf-8-sig", errors="replace").startswith(("fetch-pack ", "index-pack "))
  99. and keep.stat().st_mtime < cutoff)
  100. except FileNotFoundError:
  101. return False
  102. except OSError:
  103. return True
  104. def _unlink(part: Path) -> None:
  105. if os.name == "nt":
  106. os.chmod(part, 0o666) # git writes packs read-only; Windows refuses to unlink those
  107. part.unlink()
  108. def _drop_midx(pack_dir: Path) -> None:
  109. """A multi-pack-index names the packs it covers; dropping it before any pack goes means no crash
  110. can leave one naming a deleted pack. git rebuilds it on its own maintenance.
  111. The incremental layout (``multi-pack-index.d/``) is read through its chain file, so once the chain
  112. is gone git ignores the layers and removing them is only tidying."""
  113. (pack_dir / "multi-pack-index").unlink(missing_ok=True)
  114. layers = pack_dir / "multi-pack-index.d"
  115. (layers / "multi-pack-index-chain").unlink(missing_ok=True)
  116. for layer in layers.glob("*"):
  117. with contextlib.suppress(OSError): # a layer a reader still maps goes on a later run
  118. _unlink(layer)
  119. with contextlib.suppress(OSError):
  120. layers.rmdir()
  121. def _remove_pack(pack: Path) -> int:
  122. """Delete one pack's files, payload first; returns the bytes freed.
  123. Raises on the first part that will not go. Leftovers after the payload are invisible to git and
  124. :func:`_sweep_remnants` retries them on a later run."""
  125. _drop_midx(pack.parent)
  126. freed = 0
  127. for suffix in _PACK_SUFFIXES:
  128. part = pack.with_suffix(suffix)
  129. try:
  130. size = part.stat().st_size
  131. _unlink(part)
  132. freed += size
  133. except FileNotFoundError:
  134. continue
  135. return freed
  136. def _sweep_remnants(pack_dir: Path) -> None:
  137. """Finish deletions an earlier run could not complete: pack files whose payload is already gone.
  138. Aborted-transfer ``tmp_*`` files are gitlock.clear_stale_tmp_packs's job, earlier in the update."""
  139. cutoff = time.time() - _MIN_PACK_AGE_SECONDS
  140. for part in pack_dir.glob("pack-*.*"):
  141. try:
  142. if part.suffix in _PACK_SUFFIXES[1:] and not part.with_suffix(".pack").exists() \
  143. and part.stat().st_mtime < cutoff:
  144. _remove_pack(part)
  145. except OSError:
  146. logger.debug("pack remnant %s still in use", part.name, exc_info=True)
  147. def _load_state(pack_dir: Path) -> dict:
  148. try:
  149. state = json.loads((pack_dir.parent.parent / _STATE_FILE).read_text(encoding="utf-8-sig"))
  150. if state.get("v") != _STATE_VERSION:
  151. raise ValueError("pack tidy state from another version")
  152. kept, seen = set(state.get("kept", [])), set(state.get("seen", []))
  153. merge_bytes = int(state.get("merge_bytes", _MERGE_BATCH_BYTES))
  154. except (OSError, ValueError, TypeError, AttributeError):
  155. kept, seen, merge_bytes = set(), set(), _MERGE_BATCH_BYTES
  156. if {p.stem for p in pack_dir.glob("pack-*.pack")} - seen:
  157. kept = set() # a new pack may hold the other copy of what a kept pack holds
  158. return {"kept": kept, "merge_bytes": merge_bytes}
  159. def _save_state(pack_dir: Path, state: dict) -> None:
  160. live = sorted(p.stem for p in pack_dir.glob("pack-*.pack"))
  161. data = {"v": _STATE_VERSION, "kept": sorted(state["kept"] & set(live)), "seen": live,
  162. "merge_bytes": state["merge_bytes"]}
  163. try:
  164. (pack_dir.parent.parent / _STATE_FILE).write_text(json.dumps(data), encoding="utf-8")
  165. except OSError:
  166. logger.debug("could not record pack tidy state in %s", pack_dir, exc_info=True)
  167. def _is_redundant(oids: Iterable[bytes], others: list[_Index], deadline: float) -> Optional[bool]:
  168. """Whether every oid is in one of ``others``; None when the deadline passed first."""
  169. hit = 0
  170. for n, oid in enumerate(oids):
  171. if n % 1024 == 0 and time.monotonic() > deadline:
  172. return None
  173. if oid in others[hit]:
  174. continue
  175. for j, other in enumerate(others): # objects cluster: try the pack that had the last one first
  176. if j != hit and oid in other:
  177. hit = j
  178. break
  179. else:
  180. return False
  181. return True
  182. def _erase_redundant_packs(pack_dir: Path, deadline: float, result: TidyResult, state: dict) -> None:
  183. cutoff = time.time() - _MIN_PACK_AGE_SECONDS
  184. sizes: dict[Path, int] = {}
  185. candidates: list[Path] = []
  186. for pack in _promisor_packs(pack_dir):
  187. try:
  188. st = pack.stat()
  189. except OSError:
  190. continue
  191. sizes[pack] = st.st_size
  192. if pack.stem not in state["kept"] and st.st_mtime < cutoff and not _pinned(pack, cutoff):
  193. candidates.append(pack)
  194. if not candidates:
  195. return
  196. indexes: dict[str, _Index] = {}
  197. try:
  198. for pack in sizes:
  199. try:
  200. indexes[pack.stem] = _Index(pack.with_suffix(".idx"))
  201. except (OSError, ValueError): # unreadable now: neither a candidate nor a copy, retried next run
  202. logger.debug("pack index %s unreadable", pack.name, exc_info=True)
  203. every_index_read = len(indexes) == len(sizes)
  204. by_size = sorted(sizes, key=lambda p: sizes[p], reverse=True) # big packs hold most copies
  205. for pack in sorted(candidates, key=lambda p: sizes[p]): # small first: cheapest to prove, likeliest copies
  206. if pack.stem not in indexes:
  207. continue
  208. others = [indexes[p.stem] for p in by_size if p.stem != pack.stem and p.stem in indexes]
  209. verdict = _is_redundant(indexes[pack.stem].oids(), others, deadline) if others else False
  210. if verdict is None:
  211. result.out_of_time = True
  212. return
  213. if not verdict:
  214. if every_index_read: # unique among ALL packs; with one unread it is only unknown
  215. state["kept"].add(pack.stem)
  216. continue
  217. indexes.pop(pack.stem).close()
  218. try:
  219. result.freed_bytes += _remove_pack(pack)
  220. result.erased += 1
  221. except OSError as exc:
  222. logger.warning("Could not erase redundant pack %s (a later update retries): %s", pack.name, exc)
  223. finally:
  224. for index in indexes.values():
  225. index.close()
  226. def _merge_git(repo_root: Path, staging: Path, batch: list[Path], timeout: float):
  227. """``git pack-objects --stdin-packs`` over *batch* into *staging*, in the update's custody (a
  228. mutator: it holds the checkout lock fd, and inside an update it is job-bound on Windows).
  229. ``None`` when it ran out of time or could not start; a custody refusal propagates, so an update
  230. stops the way it does for any updater git Windows will not bind."""
  231. from hermes_cli.update_custody import CustodyRefused, run_git
  232. try:
  233. return run_git(
  234. ["git"], ["-c", "pack.packSizeLimit=0", "pack-objects", "-q", "--stdin-packs", str(staging / "pack")],
  235. cwd=str(repo_root), env=_git_env(), input="".join(p.name + "\n" for p in batch),
  236. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout,
  237. creationflags=windows_hide_flags())
  238. except CustodyRefused:
  239. raise
  240. except (OSError, subprocess.SubprocessError):
  241. return None
  242. def _merge_smallest_packs(repo_root: Path, pack_dir: Path, deadline: float, result: TidyResult,
  243. state: dict) -> None:
  244. """Merge the smallest promisor packs, a batch at a time, until the count is under target."""
  245. staging = pack_dir.parent.parent / "hermes-tidy-staging" # outside objects/: git counts strays there as garbage
  246. cutoff = time.time() - _MIN_PACK_AGE_SECONDS
  247. while True:
  248. packs = sorted((p.stat().st_size, p) for p in _promisor_packs(pack_dir) if not _pinned(p, cutoff))
  249. if len(packs) <= PACK_COUNT_TARGET:
  250. return
  251. batch, total = [], 0
  252. for size, pack in packs:
  253. if len(batch) >= 2 and total + size > state["merge_bytes"]:
  254. break
  255. batch.append(pack)
  256. total += size
  257. left = deadline - time.monotonic()
  258. if left <= 0:
  259. result.out_of_time = True
  260. return
  261. shutil.rmtree(staging, ignore_errors=True)
  262. staging.mkdir()
  263. try:
  264. # packSizeLimit=0 whatever the user's config says: inputs retire only after everything
  265. # they held is published, and that is one output pack.
  266. done = _merge_git(repo_root, staging, batch, left)
  267. if done is None:
  268. result.out_of_time = True
  269. state["merge_bytes"] = max(state["merge_bytes"] // 2, 1) # next update tries a smaller batch
  270. return
  271. outputs = list(staging.glob("pack-*.idx"))
  272. if done.returncode != 0 or len(outputs) != 1:
  273. logger.warning("Merging %d packs failed (%d outputs): %s", len(batch), len(outputs),
  274. done.stderr.strip()[-300:])
  275. return
  276. new = outputs[0].with_suffix("")
  277. # Payload last: git ignores an index whose .pack is absent, so a kill before the final move
  278. # leaves only metadata, which _sweep_remnants removes (a lone .pack would be kept forever).
  279. (pack_dir / (new.name + ".promisor")).write_bytes(
  280. b"".join(p.with_suffix(".promisor").read_bytes() for p in batch))
  281. if new.with_suffix(".rev").exists():
  282. os.replace(new.with_suffix(".rev"), pack_dir / (new.name + ".rev"))
  283. os.replace(new.with_suffix(".idx"), pack_dir / (new.name + ".idx"))
  284. os.replace(new.with_suffix(".pack"), pack_dir / (new.name + ".pack"))
  285. for pack in batch:
  286. if pack.stem != new.name:
  287. try:
  288. _remove_pack(pack)
  289. except OSError as exc: # its objects are in the merged pack; a later run retries
  290. logger.warning("Could not retire merged pack %s: %s", pack.name, exc)
  291. result.merged += len(batch)
  292. finally:
  293. shutil.rmtree(staging, ignore_errors=True)
  294. def _take_lock(git_dir: Path) -> Optional[int]:
  295. """A kernel lock on the tidy lock file, or None while another tidy holds it. The OS drops it
  296. when the holder exits, killed or not, so there is no stale lock to judge or take over; the file
  297. itself is never removed (a removed path lets a later run lock a fresh file beside a live one)."""
  298. from pm.filesystem import lock_fd
  299. fd = os.open(git_dir / _LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o600)
  300. try:
  301. if lock_fd(fd, wait=False):
  302. return fd
  303. except OSError: # a filesystem that cannot lock: skip the tidy rather than run unguarded
  304. logger.debug("pack tidy lock unavailable in %s", git_dir, exc_info=True)
  305. os.close(fd)
  306. return None
  307. def tidy_partial_clone_packs(repo_root: Path, *, budget_seconds: float = TIDY_BUDGET_SECONDS) -> TidyResult:
  308. """Spend at most ``budget_seconds`` erasing redundant and merging small promisor packs. Never raises."""
  309. from hermes_cli.gitlock import _partial_clone_filter
  310. result = TidyResult()
  311. pack_dir = Path(repo_root) / ".git" / "objects" / "pack"
  312. try:
  313. if not pack_dir.is_dir() or _partial_clone_filter(repo_root, creationflags=windows_hide_flags()) is None:
  314. return result
  315. deadline = time.monotonic() + budget_seconds
  316. lock = _take_lock(pack_dir.parent.parent)
  317. if lock is None:
  318. return result
  319. try:
  320. state = _load_state(pack_dir)
  321. try:
  322. _sweep_remnants(pack_dir)
  323. _erase_redundant_packs(pack_dir, deadline, result, state)
  324. if not result.out_of_time:
  325. _merge_smallest_packs(repo_root, pack_dir, deadline, result, state)
  326. finally:
  327. _save_state(pack_dir, state)
  328. finally:
  329. os.close(lock)
  330. result.packs_left = len(list(pack_dir.glob("pack-*.pack")))
  331. except Exception:
  332. logger.warning("partial-clone pack tidy failed in %s", repo_root, exc_info=True)
  333. return result