git_commit_graph.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. """Keep a checkout's commit-graph fresh: the ``hermes -w`` pass and every ``hermes update``.
  2. Generation numbers in the commit-graph keep ``git tag --merged HEAD`` (the version lookup) and
  3. ancestry checks off a full history walk: on hermes-agent a graph two months behind cost 0.33 s
  4. per lookup, a fresh one 0.04 s. Partial clones have git's own graph writers switched off
  5. (gitlock._TREE0_MAINTENANCE_OFF, #127711), so nothing else refreshes it there. Without Bloom data
  6. the write reads commits only, never trees, so it cannot start that lazy-fetch loop, and it costs
  7. ~3 s instead of ~27 s when changed paths would be recomputed.
  8. Also home of the per-clone maintenance slot and the bounded git runner the ``-w`` repack shares,
  9. so the update path imports this module and not all of ``worktree_ops``.
  10. """
  11. from __future__ import annotations
  12. import atexit
  13. import logging
  14. import os
  15. import subprocess
  16. import sys
  17. import tempfile
  18. import time
  19. from contextlib import contextmanager
  20. from pathlib import Path
  21. from typing import Iterator, Optional, Sequence
  22. from hermes_cli._subprocess_compat import (
  23. IS_WINDOWS,
  24. NO_LAZY_FETCH_ENV,
  25. bounded_probe_run,
  26. kill_process_tree,
  27. noninteractive_git_env,
  28. windows_detach_flags_without_breakaway,
  29. windows_detach_popen_kwargs,
  30. )
  31. logger = logging.getLogger(__name__)
  32. _WRITE = ("commit-graph", "write", "--reachable", "--split", "--no-changed-paths", "--no-progress")
  33. _MAX_AGE = 24 * 3600
  34. _TIMEOUT = 600
  35. _MIN_INTERVAL = 6 * 3600
  36. _LOCK = "hermes-commit-graph.lock"
  37. # A write with no new commits leaves git's files untouched; this stamp records that it succeeded,
  38. # so an idle repo is not rewritten on every slot.
  39. _FRESH_STAMP = "hermes-commit-graph.fresh"
  40. # git deletes its lock and tmp files on SIGTERM; a SIGKILLed writer strands commit-graph-chain.lock
  41. # and tmp_graph_*, and that lock fails every later write until the stale-lock sweep removes it.
  42. _KILL_GRACE = 5.0
  43. _CHILD = ("import sys; sys.path.insert(0, sys.argv[1]); "
  44. "from hermes_cli.git_commit_graph import refresh_in_child; refresh_in_child(*sys.argv[2:])")
  45. @contextmanager
  46. def _locked_slot(lock: Path, *, wait: bool) -> Iterator[Optional[int]]:
  47. """The slot file's fd under an exclusive cross-process lock, or None while another process holds
  48. it (``wait=False``). The file is never unlinked: a waiter on a deleted inode would claim a slot
  49. nobody else can see. The kernel drops the lock with the process."""
  50. fd = os.open(lock, os.O_RDWR | os.O_CREAT, 0o644)
  51. try:
  52. try:
  53. if IS_WINDOWS:
  54. import msvcrt
  55. msvcrt.locking(fd, msvcrt.LK_LOCK if wait else msvcrt.LK_NBLCK, 1)
  56. else:
  57. import fcntl
  58. fcntl.flock(fd, fcntl.LOCK_EX | (0 if wait else fcntl.LOCK_NB))
  59. except OSError:
  60. yield None
  61. return
  62. try:
  63. yield fd
  64. finally:
  65. if IS_WINDOWS:
  66. os.lseek(fd, 0, os.SEEK_SET)
  67. msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
  68. finally:
  69. os.close(fd)
  70. def claim_maintenance_slot(git_dir: Path, lock_name: str, interval: float) -> bool:
  71. """Exactly one process per clone gets to run a maintenance job per ``interval``.
  72. The slot file holds the claimant's pid and its mtime is the stamp: younger than the interval
  73. means another launch is running the job (or just tried and timed out) — skip. Check and claim
  74. happen under one file lock, so of N launches finding an expired or empty slot exactly one wins.
  75. """
  76. with _locked_slot(git_dir / lock_name, wait=False) as fd:
  77. if fd is None:
  78. return False
  79. st = os.fstat(fd)
  80. if st.st_size and time.time() - st.st_mtime < interval:
  81. return False
  82. os.ftruncate(fd, 0)
  83. os.lseek(fd, 0, os.SEEK_SET)
  84. os.write(fd, f"{os.getpid()}\n".encode())
  85. return True
  86. def release_maintenance_slot(git_dir: Path, lock_name: str) -> None:
  87. """Give back this process's claim (its job never started) so the next run retries; a claim
  88. another process made since is left alone."""
  89. with _locked_slot(git_dir / lock_name, wait=True) as fd:
  90. if fd is not None and os.read(fd, 32).strip() == str(os.getpid()).encode():
  91. os.ftruncate(fd, 0)
  92. def run_bounded_git(repo_root: str, args: Sequence[str], timeout: float, env: Optional[dict] = None) -> Optional[int]:
  93. """Run a niced ``git *args`` whose whole process tree dies with the timeout or with us.
  94. Returns the exit code, or None when the timeout killed it; a failure is logged with git's last
  95. stderr line. ``repack`` forks ``pack-objects``; ``subprocess.run(timeout=)`` killed only the
  96. parent and left the grandchild packing for days, and a daemon thread's child outlived the CLI the
  97. same way. An owned process group (``spawn_owned``) + ``atexit`` reaps both cases, SIGTERM first so
  98. git can delete its own lock and tmp files, and the owner-death supervisor kills the group when we
  99. are SIGKILLed or OOM-killed mid-repack.
  100. """
  101. from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
  102. cmd = ["git", *args]
  103. if os.name == "posix":
  104. cmd = ["nice", "-n", "19", *cmd]
  105. group_kw: dict = {}
  106. else:
  107. group_kw = {"creationflags": getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)}
  108. with tempfile.TemporaryFile() as err:
  109. proc = spawn_owned(cmd, purpose=f"git {args[0]}", cwd=repo_root, stdin=subprocess.DEVNULL,
  110. stdout=subprocess.DEVNULL, stderr=err, env=env, **group_kw)
  111. def _reap() -> None:
  112. if proc.poll() is None:
  113. kill_process_tree(proc, grace=_KILL_GRACE)
  114. atexit.register(_reap)
  115. try:
  116. rc = proc.wait(timeout=timeout)
  117. except subprocess.TimeoutExpired:
  118. _reap()
  119. logger.warning("git %s in %s exceeded %ss; killed", args[0], repo_root, timeout)
  120. return None
  121. finally:
  122. atexit.unregister(_reap)
  123. release_process_group_once_empty(proc.pid, proc=proc)
  124. if rc:
  125. err.seek(0)
  126. lines = err.read()[-4096:].decode("utf-8", "replace").strip().splitlines()
  127. # git's "File exists" ends on advice; the fatal/error line names the cause.
  128. cause = next((ln for ln in reversed(lines) if ln.startswith(("fatal:", "error:"))), lines[-1] if lines else "no output")
  129. logger.warning("git %s in %s failed (exit %d): %s", args[0], repo_root, rc, cause)
  130. return rc
  131. def commit_graph_stale(git_dir: Path) -> bool:
  132. """No commit-graph, or none written or confirmed current for a day."""
  133. info = git_dir / "objects" / "info"
  134. graphs = [p.stat().st_mtime for p in (info / "commit-graph", info / "commit-graphs" / "commit-graph-chain")
  135. if p.is_file()]
  136. if not graphs:
  137. return True
  138. stamp = git_dir / _FRESH_STAMP
  139. newest = max(graphs + ([stamp.stat().st_mtime] if stamp.is_file() else []))
  140. return time.time() - newest > _MAX_AGE
  141. def claim_commit_graph_refresh(git_dir: Path) -> bool:
  142. return commit_graph_stale(git_dir) and claim_maintenance_slot(git_dir, _LOCK, _MIN_INTERVAL)
  143. def _commit_graph_disabled(repo_root: str) -> bool:
  144. """``core.commitGraph=false`` in the repo, global or system config: never write a graph there.
  145. Read with the user's global/system config, which the writer's isolated env blanks."""
  146. env = noninteractive_git_env()
  147. for key in ("GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM"):
  148. env.pop(key)
  149. if key in os.environ:
  150. env[key] = os.environ[key]
  151. probe = bounded_probe_run(["git", "config", "--type=bool", "--get", "core.commitGraph"],
  152. timeout=10, cwd=repo_root, env=env)
  153. return probe is not None and probe.stdout.strip() == "false"
  154. def common_git_dir(repo_root: str | Path) -> Optional[Path]:
  155. """The git dir holding the object store: ``.git`` itself, or the dir a linked worktree's or
  156. submodule's ``.git`` file leads to. None when ``repo_root`` is no checkout."""
  157. from hermes_cli.update_lock import _git_common_dir
  158. return _git_common_dir(Path(repo_root))
  159. def refresh_commit_graph(repo_root: str, git_dir: Path) -> None:
  160. """Clear what a killed writer stranded, then write the graph (bounded, niced, never lazy-fetching).
  161. ``git_dir`` is the common git dir (:func:`common_git_dir`): objects, debris and stamp live there."""
  162. from hermes_cli.gitlock import clear_stale_object_store_locks, clear_stale_tmp_packs
  163. if _commit_graph_disabled(repo_root):
  164. logger.info("core.commitGraph=false in %s; not writing a commit-graph", repo_root)
  165. return
  166. # Object-store debris only: ``hermes -w`` runs this in whatever repo it was launched from, where
  167. # index.lock / HEAD.lock belong to the user's own git.
  168. clear_stale_object_store_locks(git_dir)
  169. clear_stale_tmp_packs(git_dir)
  170. logger.info("git commit-graph missing or stale — writing it in background")
  171. if run_bounded_git(repo_root, _WRITE, _TIMEOUT, env={**noninteractive_git_env(), **NO_LAZY_FETCH_ENV}) != 0:
  172. return
  173. # The stamp vouches for a graph on disk: a write another git undid (or whose files vanished)
  174. # exits 0 too, and a stamp then would hold the retry off for a day with no graph at all.
  175. if _graph_on_disk(git_dir):
  176. (git_dir / _FRESH_STAMP).touch()
  177. else:
  178. logger.info("git commit-graph write in %s exited 0 but left no graph (no commits yet?)", repo_root)
  179. def _graph_on_disk(git_dir: Path) -> bool:
  180. """A split chain whose every layer file exists, or a monolithic graph."""
  181. info = git_dir / "objects" / "info"
  182. try:
  183. layers = (info / "commit-graphs" / "commit-graph-chain").read_text(encoding="ascii").split()
  184. except (OSError, UnicodeDecodeError):
  185. return (info / "commit-graph").is_file()
  186. return bool(layers) and all((info / "commit-graphs" / f"graph-{layer}.graph").is_file() for layer in layers)
  187. class _LogDirOnFailureHandler(logging.FileHandler):
  188. """Opens (``delay``) only when there is a failure to record, creating ``logs/`` then: a home
  189. whose logs were never set up still gets the line; a deleted profile is not resurrected. A log
  190. that cannot be opened drops the line (``handleError``) instead of raising out of the logger."""
  191. def emit(self, record: logging.LogRecord) -> None:
  192. if self.stream is None:
  193. from hermes_constants import mkdir_under_hermes_home
  194. try:
  195. mkdir_under_hermes_home(Path(self.baseFilename).parent)
  196. self.stream = self._open()
  197. except OSError: # a missing/deleted named profile is FileNotFoundError
  198. self.handleError(record)
  199. return
  200. super().emit(record)
  201. def refresh_in_child(repo_root: str, git_dir: str, log_file: str) -> None:
  202. """Body of the detached child: a failure is appended to the home's agent.log; never raises.
  203. A plain handler, not ``setup_logging``: that reads config.yaml and seeds the home's layout,
  204. side effects a git maintenance child has no business having in a profile mid-update.
  205. """
  206. handler = _LogDirOnFailureHandler(log_file, encoding="utf-8", delay=True)
  207. handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
  208. logging.getLogger().addHandler(handler)
  209. try:
  210. refresh_commit_graph(repo_root, Path(git_dir))
  211. except Exception:
  212. logger.warning("Background commit-graph refresh of %s failed", repo_root, exc_info=True)
  213. def spawn_commit_graph_refresh(repo_root: Path) -> bool:
  214. """Start the bounded commit-graph refresh in a detached child when it is due; never waits.
  215. ``hermes update`` is the one regular git pass an install checkout gets (``hermes -w`` is
  216. rarely launched from it), so without this its version lookups never get a graph. Detached
  217. because the write takes seconds and must neither delay the update nor die with it. A spawn that
  218. fails gives the slot back, so the next update retries instead of waiting out the interval.
  219. """
  220. from hermes_constants import get_hermes_home
  221. git_dir = common_git_dir(repo_root)
  222. if git_dir is None or not (git_dir / "objects").is_dir() or not claim_commit_graph_refresh(git_dir):
  223. return False
  224. code_root = Path(__file__).resolve().parents[1]
  225. argv = [sys.executable, "-c", _CHILD, str(code_root), str(repo_root), str(git_dir),
  226. str(get_hermes_home() / "logs" / "agent.log")]
  227. kwargs: dict = dict(cwd=str(repo_root), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
  228. stderr=subprocess.DEVNULL, close_fds=True)
  229. try:
  230. try:
  231. # detached-spawn: ok — the refresh must neither delay the update nor die with it; slot-locked, bounded
  232. subprocess.Popen(argv, **kwargs, **windows_detach_popen_kwargs())
  233. except OSError:
  234. if not IS_WINDOWS:
  235. raise
  236. # A job object without BREAKAWAY_OK refuses CREATE_BREAKAWAY_FROM_JOB; retry without it.
  237. subprocess.Popen(argv, **kwargs, creationflags=windows_detach_flags_without_breakaway())
  238. except OSError:
  239. release_maintenance_slot(git_dir, _LOCK)
  240. raise
  241. return True