| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386 |
- """Progressive, time-boxed cleanup of a partial clone's promisor packfiles (#129712).
- A partial clone downloads objects when git first needs them, and git asks for them without telling
- the server what it already has, so the same trees (and, for history downloads, commits) land again
- and again, each download as its own packfile that nothing removes: ``.git`` reached 39 GiB on a
- ~1 GiB repository. A full ``git gc`` over that is all-or-nothing and on a big checkout runs for tens
- of minutes, so ``hermes update`` instead spends at most ``TIDY_BUDGET_SECONDS`` per run and every
- unit of work it finishes is kept:
- 1. Erase promisor packs whose every object is also in another promisor pack. Nothing is lost, and
- nothing depends on the remote still serving it: the rule is proven against the local packs alone.
- 2. While more than ``PACK_COUNT_TARGET`` promisor packs remain, merge the smallest few into one
- with ``git pack-objects --stdin-packs``; the merged pack holds each object once.
- Each erase and each merge is atomic for git (a pack exists for git only while both its ``.idx`` and
- ``.pack`` do), so an update that runs out of budget, fails or is killed leaves the rest for the next.
- """
- from __future__ import annotations
- import contextlib
- import json
- import logging
- import mmap
- import os
- import shutil
- import struct
- import subprocess
- import time
- from dataclasses import dataclass
- from pathlib import Path
- from typing import Dict, Iterable, Iterator, List, Optional
- from hermes_cli._subprocess_compat import (
- NO_LAZY_FETCH_ENV,
- noninteractive_git_env,
- windows_hide_flags,
- )
- logger = logging.getLogger(__name__)
- TIDY_BUDGET_SECONDS = 60
- PACK_COUNT_TARGET = 50
- # A pack this young may still be in use by the git that just fetched it; index-pack also writes a
- # new pack's .promisor/.keep before its .pack, so a young marker without a payload may be a live fetch.
- _MIN_PACK_AGE_SECONDS = 60 * 60
- _MERGE_BATCH_BYTES = 256 * 1024 * 1024
- # Kept in .git/: packs proven to hold an object no other pack has (valid until a new pack appears,
- # since removing packs never adds copies) and the merge batch size, halved after a merge that ran out
- # of time. Remembering both is what guarantees progress: no update repeats work an earlier one finished.
- _STATE_FILE = "hermes-pack-tidy.json"
- _STATE_VERSION = 2 # v1 could record a pack as unique after comparing it with only some packs
- # Two tidies at once could each erase a pack because the other still has its copies.
- _LOCK_FILE = "hermes-pack-tidy.lock"
- # Payload first: git loads a pack only when both .idx and .pack exist, so once .pack is gone the pack is
- # gone for git, and a failure on .pack itself (a reader still maps it on Windows) changes nothing.
- _PACK_SUFFIXES = (".pack", ".idx", ".rev", ".bitmap", ".mtimes", ".promisor", ".keep")
- _IDX_V2 = b"\xfftOc\x00\x00\x00\x02"
- _FANOUT = 8
- _NAMES = _FANOUT + 256 * 4
- @dataclass
- class TidyResult:
- erased: int = 0
- freed_bytes: int = 0
- merged: int = 0
- packs_left: int = 0
- out_of_time: bool = False
- class _Index:
- """A version-2 SHA-1 pack index (what a GitHub clone writes), searched in place."""
- def __init__(self, idx: Path) -> None:
- with open(idx, "rb") as fh:
- self._map = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
- if self._map[:8] != _IDX_V2:
- self.close()
- raise ValueError(f"{idx.name}: not a v2 SHA-1 pack index")
- self.count = struct.unpack_from(">I", self._map, _FANOUT + 255 * 4)[0]
- def oids(self) -> Iterator[bytes]:
- for i in range(self.count): # lazily: a big index must not run past the deadline before its first check
- yield self._map[_NAMES + i * 20: _NAMES + i * 20 + 20]
- def __contains__(self, oid: bytes) -> bool:
- lo = struct.unpack_from(">I", self._map, _FANOUT + (oid[0] - 1) * 4)[0] if oid[0] else 0
- hi = struct.unpack_from(">I", self._map, _FANOUT + oid[0] * 4)[0]
- while lo < hi:
- mid = (lo + hi) // 2
- name = self._map[_NAMES + mid * 20: _NAMES + mid * 20 + 20]
- if name == oid:
- return True
- if name < oid:
- lo = mid + 1
- else:
- hi = mid
- return False
- def close(self) -> None:
- self._map.close() # Windows refuses to unlink a mapped file
- def _git_env() -> dict[str, str]:
- return {**noninteractive_git_env(), **NO_LAZY_FETCH_ENV}
- def _promisor_packs(pack_dir: Path) -> list[Path]:
- return [p for p in pack_dir.glob("pack-*.pack")
- if p.with_suffix(".promisor").exists() and p.with_suffix(".idx").exists()]
- def _pinned(pack: Path, cutoff: float) -> bool:
- """Whether a ``.keep`` pins the pack. fetch-pack and index-pack write ``<cmd> <pid> on <host>`` and
- delete it once the fetch has updated its refs, so an old one of theirs belongs to a fetch that was
- killed; it pins nothing, and git's own repack never touches the pack while it exists."""
- keep = pack.with_suffix(".keep")
- try:
- return not (keep.read_text(encoding="utf-8-sig", errors="replace").startswith(("fetch-pack ", "index-pack "))
- and keep.stat().st_mtime < cutoff)
- except FileNotFoundError:
- return False
- except OSError:
- return True
- def _unlink(part: Path) -> None:
- if os.name == "nt":
- os.chmod(part, 0o666) # git writes packs read-only; Windows refuses to unlink those
- part.unlink()
- def _drop_midx(pack_dir: Path) -> None:
- """A multi-pack-index names the packs it covers; dropping it before any pack goes means no crash
- can leave one naming a deleted pack. git rebuilds it on its own maintenance.
- The incremental layout (``multi-pack-index.d/``) is read through its chain file, so once the chain
- is gone git ignores the layers and removing them is only tidying."""
- (pack_dir / "multi-pack-index").unlink(missing_ok=True)
- layers = pack_dir / "multi-pack-index.d"
- (layers / "multi-pack-index-chain").unlink(missing_ok=True)
- for layer in layers.glob("*"):
- with contextlib.suppress(OSError): # a layer a reader still maps goes on a later run
- _unlink(layer)
- with contextlib.suppress(OSError):
- layers.rmdir()
- def _remove_pack(pack: Path) -> int:
- """Delete one pack's files, payload first; returns the bytes freed.
- Raises on the first part that will not go. Leftovers after the payload are invisible to git and
- :func:`_sweep_remnants` retries them on a later run."""
- _drop_midx(pack.parent)
- freed = 0
- for suffix in _PACK_SUFFIXES:
- part = pack.with_suffix(suffix)
- try:
- size = part.stat().st_size
- _unlink(part)
- freed += size
- except FileNotFoundError:
- continue
- return freed
- def _sweep_remnants(pack_dir: Path) -> None:
- """Finish deletions an earlier run could not complete: pack files whose payload is already gone.
- Aborted-transfer ``tmp_*`` files are gitlock.clear_stale_tmp_packs's job, earlier in the update."""
- cutoff = time.time() - _MIN_PACK_AGE_SECONDS
- for part in pack_dir.glob("pack-*.*"):
- try:
- if part.suffix in _PACK_SUFFIXES[1:] and not part.with_suffix(".pack").exists() \
- and part.stat().st_mtime < cutoff:
- _remove_pack(part)
- except OSError:
- logger.debug("pack remnant %s still in use", part.name, exc_info=True)
- def _load_state(pack_dir: Path) -> dict:
- try:
- state = json.loads((pack_dir.parent.parent / _STATE_FILE).read_text(encoding="utf-8-sig"))
- if state.get("v") != _STATE_VERSION:
- raise ValueError("pack tidy state from another version")
- kept, seen = set(state.get("kept", [])), set(state.get("seen", []))
- merge_bytes = int(state.get("merge_bytes", _MERGE_BATCH_BYTES))
- except (OSError, ValueError, TypeError, AttributeError):
- kept, seen, merge_bytes = set(), set(), _MERGE_BATCH_BYTES
- if {p.stem for p in pack_dir.glob("pack-*.pack")} - seen:
- kept = set() # a new pack may hold the other copy of what a kept pack holds
- return {"kept": kept, "merge_bytes": merge_bytes}
- def _save_state(pack_dir: Path, state: dict) -> None:
- live = sorted(p.stem for p in pack_dir.glob("pack-*.pack"))
- data = {"v": _STATE_VERSION, "kept": sorted(state["kept"] & set(live)), "seen": live,
- "merge_bytes": state["merge_bytes"]}
- try:
- (pack_dir.parent.parent / _STATE_FILE).write_text(json.dumps(data), encoding="utf-8")
- except OSError:
- logger.debug("could not record pack tidy state in %s", pack_dir, exc_info=True)
- def _is_redundant(oids: Iterable[bytes], others: list[_Index], deadline: float) -> Optional[bool]:
- """Whether every oid is in one of ``others``; None when the deadline passed first."""
- hit = 0
- for n, oid in enumerate(oids):
- if n % 1024 == 0 and time.monotonic() > deadline:
- return None
- if oid in others[hit]:
- continue
- for j, other in enumerate(others): # objects cluster: try the pack that had the last one first
- if j != hit and oid in other:
- hit = j
- break
- else:
- return False
- return True
- def _erase_redundant_packs(pack_dir: Path, deadline: float, result: TidyResult, state: dict) -> None:
- cutoff = time.time() - _MIN_PACK_AGE_SECONDS
- sizes: dict[Path, int] = {}
- candidates: list[Path] = []
- for pack in _promisor_packs(pack_dir):
- try:
- st = pack.stat()
- except OSError:
- continue
- sizes[pack] = st.st_size
- if pack.stem not in state["kept"] and st.st_mtime < cutoff and not _pinned(pack, cutoff):
- candidates.append(pack)
- if not candidates:
- return
- indexes: dict[str, _Index] = {}
- try:
- for pack in sizes:
- try:
- indexes[pack.stem] = _Index(pack.with_suffix(".idx"))
- except (OSError, ValueError): # unreadable now: neither a candidate nor a copy, retried next run
- logger.debug("pack index %s unreadable", pack.name, exc_info=True)
- every_index_read = len(indexes) == len(sizes)
- by_size = sorted(sizes, key=lambda p: sizes[p], reverse=True) # big packs hold most copies
- for pack in sorted(candidates, key=lambda p: sizes[p]): # small first: cheapest to prove, likeliest copies
- if pack.stem not in indexes:
- continue
- others = [indexes[p.stem] for p in by_size if p.stem != pack.stem and p.stem in indexes]
- verdict = _is_redundant(indexes[pack.stem].oids(), others, deadline) if others else False
- if verdict is None:
- result.out_of_time = True
- return
- if not verdict:
- if every_index_read: # unique among ALL packs; with one unread it is only unknown
- state["kept"].add(pack.stem)
- continue
- indexes.pop(pack.stem).close()
- try:
- result.freed_bytes += _remove_pack(pack)
- result.erased += 1
- except OSError as exc:
- logger.warning("Could not erase redundant pack %s (a later update retries): %s", pack.name, exc)
- finally:
- for index in indexes.values():
- index.close()
- def _merge_git(repo_root: Path, staging: Path, batch: list[Path], timeout: float):
- """``git pack-objects --stdin-packs`` over *batch* into *staging*, in the update's custody (a
- mutator: it holds the checkout lock fd, and inside an update it is job-bound on Windows).
- ``None`` when it ran out of time or could not start; a custody refusal propagates, so an update
- stops the way it does for any updater git Windows will not bind."""
- from hermes_cli.update_custody import CustodyRefused, run_git
- try:
- return run_git(
- ["git"], ["-c", "pack.packSizeLimit=0", "pack-objects", "-q", "--stdin-packs", str(staging / "pack")],
- cwd=str(repo_root), env=_git_env(), input="".join(p.name + "\n" for p in batch),
- capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout,
- creationflags=windows_hide_flags())
- except CustodyRefused:
- raise
- except (OSError, subprocess.SubprocessError):
- return None
- def _merge_smallest_packs(repo_root: Path, pack_dir: Path, deadline: float, result: TidyResult,
- state: dict) -> None:
- """Merge the smallest promisor packs, a batch at a time, until the count is under target."""
- staging = pack_dir.parent.parent / "hermes-tidy-staging" # outside objects/: git counts strays there as garbage
- cutoff = time.time() - _MIN_PACK_AGE_SECONDS
- while True:
- packs = sorted((p.stat().st_size, p) for p in _promisor_packs(pack_dir) if not _pinned(p, cutoff))
- if len(packs) <= PACK_COUNT_TARGET:
- return
- batch, total = [], 0
- for size, pack in packs:
- if len(batch) >= 2 and total + size > state["merge_bytes"]:
- break
- batch.append(pack)
- total += size
- left = deadline - time.monotonic()
- if left <= 0:
- result.out_of_time = True
- return
- shutil.rmtree(staging, ignore_errors=True)
- staging.mkdir()
- try:
- # packSizeLimit=0 whatever the user's config says: inputs retire only after everything
- # they held is published, and that is one output pack.
- done = _merge_git(repo_root, staging, batch, left)
- if done is None:
- result.out_of_time = True
- state["merge_bytes"] = max(state["merge_bytes"] // 2, 1) # next update tries a smaller batch
- return
- outputs = list(staging.glob("pack-*.idx"))
- if done.returncode != 0 or len(outputs) != 1:
- logger.warning("Merging %d packs failed (%d outputs): %s", len(batch), len(outputs),
- done.stderr.strip()[-300:])
- return
- new = outputs[0].with_suffix("")
- # Payload last: git ignores an index whose .pack is absent, so a kill before the final move
- # leaves only metadata, which _sweep_remnants removes (a lone .pack would be kept forever).
- (pack_dir / (new.name + ".promisor")).write_bytes(
- b"".join(p.with_suffix(".promisor").read_bytes() for p in batch))
- if new.with_suffix(".rev").exists():
- os.replace(new.with_suffix(".rev"), pack_dir / (new.name + ".rev"))
- os.replace(new.with_suffix(".idx"), pack_dir / (new.name + ".idx"))
- os.replace(new.with_suffix(".pack"), pack_dir / (new.name + ".pack"))
- for pack in batch:
- if pack.stem != new.name:
- try:
- _remove_pack(pack)
- except OSError as exc: # its objects are in the merged pack; a later run retries
- logger.warning("Could not retire merged pack %s: %s", pack.name, exc)
- result.merged += len(batch)
- finally:
- shutil.rmtree(staging, ignore_errors=True)
- def _take_lock(git_dir: Path) -> Optional[int]:
- """A kernel lock on the tidy lock file, or None while another tidy holds it. The OS drops it
- when the holder exits, killed or not, so there is no stale lock to judge or take over; the file
- itself is never removed (a removed path lets a later run lock a fresh file beside a live one)."""
- from pm.filesystem import lock_fd
- fd = os.open(git_dir / _LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o600)
- try:
- if lock_fd(fd, wait=False):
- return fd
- except OSError: # a filesystem that cannot lock: skip the tidy rather than run unguarded
- logger.debug("pack tidy lock unavailable in %s", git_dir, exc_info=True)
- os.close(fd)
- return None
- def tidy_partial_clone_packs(repo_root: Path, *, budget_seconds: float = TIDY_BUDGET_SECONDS) -> TidyResult:
- """Spend at most ``budget_seconds`` erasing redundant and merging small promisor packs. Never raises."""
- from hermes_cli.gitlock import _partial_clone_filter
- result = TidyResult()
- pack_dir = Path(repo_root) / ".git" / "objects" / "pack"
- try:
- if not pack_dir.is_dir() or _partial_clone_filter(repo_root, creationflags=windows_hide_flags()) is None:
- return result
- deadline = time.monotonic() + budget_seconds
- lock = _take_lock(pack_dir.parent.parent)
- if lock is None:
- return result
- try:
- state = _load_state(pack_dir)
- try:
- _sweep_remnants(pack_dir)
- _erase_redundant_packs(pack_dir, deadline, result, state)
- if not result.out_of_time:
- _merge_smallest_packs(repo_root, pack_dir, deadline, result, state)
- finally:
- _save_state(pack_dir, state)
- finally:
- os.close(lock)
- result.packs_left = len(list(pack_dir.glob("pack-*.pack")))
- except Exception:
- logger.warning("partial-clone pack tidy failed in %s", repo_root, exc_info=True)
- return result
|