checkpoint_manager.py 77 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876
  1. """
  2. Checkpoint Manager — Transparent filesystem snapshots via a single shared
  3. shadow git store.
  4. Creates automatic snapshots of working directories before file-mutating
  5. operations (``write_file``, ``patch``, ``terminal`` with destructive flags),
  6. triggered once per conversation turn. Provides rollback to any previous
  7. checkpoint.
  8. This is NOT a tool — the LLM never sees it. It's transparent infrastructure
  9. controlled by the ``checkpoints`` config flag or ``--checkpoints`` CLI flag.
  10. Storage layout (single shared store, git objects deduplicated across projects)
  11. -----------------------------------------------------------------------------
  12. ~/.hermes/checkpoints/
  13. store/ — single bare-ish git repo
  14. HEAD, config, objects/ — standard git internals (shared)
  15. refs/hermes/<hash16> — per-project branch tip
  16. indexes/<hash16> — per-project git index
  17. projects/<hash16>.json — {workdir, created_at, last_touch}
  18. info/exclude — default excludes (shared)
  19. .last_prune — auto-prune idempotency marker
  20. legacy-<timestamp>/ — archived pre-v2 per-project shadow
  21. repos (auto-migrated on first init)
  22. Why a single store?
  23. -------------------
  24. The pre-v2 design kept a full shadow repo per working directory. Each one
  25. re-stored most of the project's files under its own ``objects/`` tree, with
  26. zero sharing across worktrees of the same project. A single user with a
  27. dozen worktrees of the same repo burned ~40 MB each (~500 MB total) storing
  28. the same blobs over and over. A single shared store lets git's content-
  29. addressable object DB deduplicate across projects and across turns, so adding
  30. a new worktree costs near-zero.
  31. The shadow store uses ``GIT_DIR`` + ``GIT_WORK_TREE`` + ``GIT_INDEX_FILE``
  32. so no git state leaks into the user's project directory.
  33. Auto-maintenance (``tools.checkpoint_maintenance``)
  34. ---------------------------------------------------
  35. Shadow state accumulates over time. ``prune_checkpoints`` deletes refs whose
  36. recorded working directory no longer exists (orphan) or whose last touch is
  37. older than ``retention_days`` (stale), then runs ``git gc --prune=now`` to
  38. reclaim object storage. A size-cap pass drops the oldest checkpoints per
  39. project until total store size is under ``max_total_size_mb``.
  40. """
  41. import fnmatch
  42. import hashlib
  43. import json
  44. import logging
  45. import os
  46. import re
  47. import shutil
  48. import subprocess
  49. import tempfile
  50. import time
  51. from pathlib import Path, PurePosixPath
  52. from hermes_constants import get_hermes_home
  53. from hermes_cli._subprocess_compat import selected_git_env, windows_hide_flags
  54. from hermes_cli.gitlock import clear_stale_tmp_packs
  55. from typing import Dict, List, Optional, Set, Tuple
  56. from utils import env_int, rmtree_readonly
  57. logger = logging.getLogger(__name__)
  58. # ---------------------------------------------------------------------------
  59. # Constants
  60. # ---------------------------------------------------------------------------
  61. CHECKPOINT_BASE = get_hermes_home() / "checkpoints"
  62. _CHECKPOINT_BASE_AT_IMPORT = CHECKPOINT_BASE
  63. def _resolve_checkpoint_base() -> Path:
  64. """Active profile's checkpoint root at call time: the patched ``CHECKPOINT_BASE`` when a test
  65. changed it, else live profile-scoped HERMES_HOME — under the multiplexed gateway one process
  66. serves every profile, so the import-time constant would write every profile's code-edit
  67. checkpoints into the launch profile's store."""
  68. return CHECKPOINT_BASE if CHECKPOINT_BASE != _CHECKPOINT_BASE_AT_IMPORT else get_hermes_home() / "checkpoints"
  69. # Single shared store directory under CHECKPOINT_BASE.
  70. _STORE_DIRNAME = "store"
  71. _REFS_PREFIX = "refs/hermes"
  72. _INDEXES_DIRNAME = "indexes"
  73. _PROJECTS_DIRNAME = "projects"
  74. _LEDGERS_DIRNAME = "ledgers"
  75. _LEGACY_PREFIX = "legacy-"
  76. _PRUNE_MARKER_NAME = ".last_prune"
  77. # Agent-write ledger cap: newest entries retained per project.
  78. _LEDGER_MAX_ENTRIES = 2000
  79. DEFAULT_EXCLUDES = [
  80. # Dependency / build output
  81. "node_modules/",
  82. "dist/",
  83. "build/",
  84. "target/",
  85. "out/",
  86. ".next/",
  87. ".nuxt/",
  88. # Caches
  89. "__pycache__/",
  90. "*.pyc",
  91. "*.pyo",
  92. ".cache/",
  93. ".pytest_cache/",
  94. ".mypy_cache/",
  95. ".ruff_cache/",
  96. "coverage/",
  97. ".coverage",
  98. # Virtualenvs
  99. ".venv/",
  100. "venv/",
  101. "env/",
  102. # VCS
  103. ".git/",
  104. ".hg/",
  105. ".svn/",
  106. # Worktrees (Hermes convention — don't recursively snapshot siblings)
  107. ".worktrees/",
  108. # Native / compiled binaries
  109. "*.so",
  110. "*.dylib",
  111. "*.dll",
  112. "*.o",
  113. "*.a",
  114. "*.jar",
  115. "*.class",
  116. "*.exe",
  117. "*.obj",
  118. # Media / large binaries
  119. "*.mp4",
  120. "*.mov",
  121. "*.mkv",
  122. "*.webm",
  123. "*.zip",
  124. "*.tar",
  125. "*.tar.gz",
  126. "*.tgz",
  127. "*.7z",
  128. "*.rar",
  129. "*.iso",
  130. # Secrets
  131. ".env",
  132. ".env.*",
  133. ".env.local",
  134. ".env.*.local",
  135. # OS junk
  136. ".DS_Store",
  137. "Thumbs.db",
  138. # Logs
  139. "*.log",
  140. # Hermes browser profiles (CDP / agent-browser user-data dirs): locked SQLite files and
  141. # hundreds of MB of cache that are never project state.
  142. "chrome_cdp_profile/",
  143. "agent-browser-chrome-*/",
  144. ]
  145. # Git subprocess timeout (seconds).
  146. _GIT_TIMEOUT: int = max(10, min(60, env_int("HERMES_CHECKPOINT_TIMEOUT", 30)))
  147. # Max files to snapshot — skip huge directories to avoid slowdowns.
  148. _MAX_FILES = 50_000
  149. # Valid git commit hash pattern: 4–40 hex chars (short or full SHA-1/SHA-256).
  150. _COMMIT_HASH_RE = re.compile(r'^[0-9a-fA-F]{4,64}$')
  151. # ---------------------------------------------------------------------------
  152. # Input validation helpers
  153. # ---------------------------------------------------------------------------
  154. def _validate_commit_hash(commit_hash: str) -> Optional[str]:
  155. """Validate a commit hash to prevent git argument injection.
  156. Returns an error string if invalid, None if valid.
  157. Values starting with '-' would be interpreted as git flags
  158. (e.g., '--patch', '-p') instead of revision specifiers.
  159. """
  160. if not commit_hash or not commit_hash.strip():
  161. return "Empty commit hash"
  162. if commit_hash.startswith("-"):
  163. return f"Invalid commit hash (must not start with '-'): {commit_hash!r}"
  164. if not _COMMIT_HASH_RE.match(commit_hash):
  165. return f"Invalid commit hash (expected 4-64 hex characters): {commit_hash!r}"
  166. return None
  167. def _validate_file_path(file_path: str, working_dir: str) -> Optional[str]:
  168. """Validate a file path to prevent path traversal outside the working directory.
  169. Returns an error string if invalid, None if valid.
  170. """
  171. if not file_path or not file_path.strip():
  172. return "Empty file path"
  173. if os.path.isabs(file_path):
  174. return f"File path must be relative, got absolute path: {file_path!r}"
  175. abs_workdir = _normalize_path(working_dir)
  176. resolved = (abs_workdir / file_path).resolve()
  177. try:
  178. resolved.relative_to(abs_workdir)
  179. except ValueError:
  180. return f"File path escapes the working directory via traversal: {file_path!r}"
  181. return None
  182. # ---------------------------------------------------------------------------
  183. # Path / hash helpers
  184. # ---------------------------------------------------------------------------
  185. def _normalize_path(path_value: str) -> Path:
  186. """Return a canonical absolute path for checkpoint operations."""
  187. return Path(path_value).expanduser().resolve()
  188. def _project_hash(working_dir: str) -> str:
  189. """Deterministic per-project hash: sha256(abs_path)[:16]."""
  190. abs_path = str(_normalize_path(working_dir))
  191. return hashlib.sha256(abs_path.encode()).hexdigest()[:16]
  192. def _store_path(base: Optional[Path] = None) -> Path:
  193. """Return the single shared shadow store path."""
  194. return (base or _resolve_checkpoint_base()) / _STORE_DIRNAME
  195. def _store_has_head(store: Path) -> bool:
  196. return (store / "HEAD").exists()
  197. def _index_path(store: Path, dir_hash: str) -> Path:
  198. return store / _INDEXES_DIRNAME / dir_hash
  199. def _store_pathspecs(base: Path, working_dir) -> list[str]:
  200. """Pathspecs naming the checkpoint base and its ``store_lock`` file inside ``working_dir``.
  201. Empty when the base is outside the working directory (or is the working directory
  202. itself, where excluding ``.`` would stage nothing).
  203. """
  204. try:
  205. rel = _normalize_path(str(base)).relative_to(_normalize_path(str(working_dir)))
  206. except (OSError, ValueError):
  207. return []
  208. if not rel.parts:
  209. return []
  210. lock = rel.parent / f".{rel.name}.lock"
  211. return [rel.as_posix(), lock.as_posix()]
  212. def _stage_all_args(base: Path, working_dir) -> list[str]:
  213. """``git add -A`` that never stages the checkpoint base itself.
  214. With HERMES_HOME (or a profile home) as the working directory the store sits inside
  215. the snapshotted tree: every snapshot then staged the previous one's packfiles and
  216. index, the store grew by about its own size per checkpoint, and the eventual
  217. ``add -A`` timeout left ``indexes/<hash>.lock`` behind. Exclude the base and its
  218. ``store_lock`` file by pathspec, which works at any nesting depth without rewriting
  219. the store's ``info/exclude``.
  220. """
  221. return ["add", "-A", *_exclude_store_args(base, working_dir)]
  222. def _exclude_store_args(base: Path, working_dir) -> list[str]:
  223. """``-- :(exclude)<base> :(exclude)<lock>`` for git commands, or ``[]`` when not needed."""
  224. paths = _store_pathspecs(base, working_dir)
  225. return ["--", *(f":(exclude){p}" for p in paths)] if paths else []
  226. def _unstage_store(store: Path, working_dir, index_file: Path) -> None:
  227. """Drop the base from an index re-seeded from a checkpoint taken before it was excluded.
  228. An exclude pathspec only stops ``add -A`` from staging new store paths; it leaves
  229. entries already in the index alone, so a project whose earlier snapshots captured
  230. the store would carry them into every later snapshot. ``--cached`` keeps the files on
  231. disk; ``-f`` is needed because a live store's index/ref files always differ from the
  232. copies the old snapshot recorded.
  233. """
  234. paths = _store_pathspecs(store.parent, working_dir)
  235. if paths:
  236. _run_git(
  237. ["rm", "--cached", "-r", "-f", "-q", "--ignore-unmatch", "--", *paths],
  238. store, working_dir, index_file=index_file,
  239. )
  240. def _unstage_excluded(store: Path, working_dir, index_file: Path) -> bool:
  241. """Drop paths the store's excludes now cover from an index re-seeded from an older checkpoint.
  242. ``add -A`` honours ``info/exclude`` only for paths it is not already tracking, so a
  243. directory excluded after it was first captured (browser profiles) would otherwise stay in
  244. every later snapshot.
  245. """
  246. ok, out, _ = _run_git(
  247. ["ls-files", "--cached", "--ignored", "-z", f"--exclude-from={store / 'info' / 'exclude'}"],
  248. store, working_dir, index_file=index_file,
  249. )
  250. if not ok:
  251. return False
  252. paths = [p for p in out.split("\x00") if p]
  253. for i in range(0, len(paths), 200):
  254. ok, _, _ = _run_git(
  255. ["rm", "--cached", "-f", "-q", "--ignore-unmatch", "--", *paths[i:i + 200]],
  256. store, working_dir, index_file=index_file,
  257. extra_env={"GIT_LITERAL_PATHSPECS": "1"},
  258. )
  259. if not ok:
  260. return False
  261. return True
  262. def _ledger_path(store: Path, dir_hash: str) -> Path:
  263. return store / _LEDGERS_DIRNAME / f"{dir_hash}.json"
  264. def _hash_file(path: Path) -> Optional[str]:
  265. """Streaming sha256 of a file's bytes. None if unreadable/missing."""
  266. try:
  267. h = hashlib.sha256()
  268. with open(path, "rb") as fh:
  269. for chunk in iter(lambda: fh.read(1 << 20), b""):
  270. h.update(chunk)
  271. return h.hexdigest()
  272. except OSError:
  273. return None
  274. def _load_ledger(store: Path, dir_hash: str) -> dict[str, dict]:
  275. """Load the agent-write ledger: {relpath: {"sha256": ..., "ts": ...}}.
  276. The ledger records the content hash of every file the last successful
  277. ``write_file`` / ``patch`` produced, so restores can tell "Hermes wrote
  278. this" apart from "the user hand-edited this afterwards".
  279. """
  280. try:
  281. raw = _ledger_path(store, dir_hash).read_text(encoding="utf-8-sig")
  282. data = json.loads(raw)
  283. return data if isinstance(data, dict) else {}
  284. except (OSError, ValueError):
  285. return {}
  286. def _save_ledger(store: Path, dir_hash: str, ledger: dict[str, dict]) -> None:
  287. """Persist the agent-write ledger, capped to the newest entries."""
  288. try:
  289. if len(ledger) > _LEDGER_MAX_ENTRIES:
  290. newest = sorted(
  291. ledger.items(),
  292. key=lambda kv: kv[1].get("ts", 0) if isinstance(kv[1], dict) else 0,
  293. reverse=True,
  294. )[:_LEDGER_MAX_ENTRIES]
  295. ledger = dict(newest)
  296. path = _ledger_path(store, dir_hash)
  297. path.parent.mkdir(parents=True, exist_ok=True)
  298. tmp = path.with_suffix(".json.tmp")
  299. tmp.write_text(json.dumps(ledger), encoding="utf-8")
  300. tmp.replace(path)
  301. except OSError:
  302. logger.debug("Failed to save agent-write ledger for %s", dir_hash, exc_info=True)
  303. def _ref_name(dir_hash: str) -> str:
  304. return f"{_REFS_PREFIX}/{dir_hash}"
  305. def _gitlink_paths(ls_tree_z: str) -> list[str]:
  306. """Paths of gitlinks (nested repositories stored as a commit reference, not
  307. their files) in ``git ls-tree -r -z`` output."""
  308. return [
  309. record.split("\t", 1)[1]
  310. for record in ls_tree_z.split("\x00")
  311. if record.startswith("160000 commit ") and "\t" in record
  312. ]
  313. def _uncaptured_note(nested_repos: list[str]) -> str:
  314. """Checkpoint-message suffix naming nested repositories whose files were
  315. not captured, so every /rollback listing discloses it."""
  316. shown = ", ".join(nested_repos[:3])
  317. more = f" (+{len(nested_repos) - 3})" if len(nested_repos) > 3 else ""
  318. return f" [nested git repos not captured: {shown}{more}]"
  319. def _project_meta_path(store: Path, dir_hash: str) -> Path:
  320. return store / _PROJECTS_DIRNAME / f"{dir_hash}.json"
  321. # ---------------------------------------------------------------------------
  322. # Git env
  323. # ---------------------------------------------------------------------------
  324. def _git_env(
  325. store: Path,
  326. working_dir: str,
  327. index_file: Optional[Path] = None,
  328. ) -> dict:
  329. """Build env dict that redirects git to the shared store.
  330. The shared store is internal Hermes infrastructure — it must NOT inherit
  331. the user's global or system git config. User-level settings like
  332. ``commit.gpgsign = true``, signing hooks, or credential helpers would
  333. either break background snapshots or, worse, spawn interactive prompts
  334. (pinentry GUI windows) mid-session every time a file is written.
  335. Isolation strategy:
  336. * ``GIT_CONFIG_GLOBAL=<os.devnull>`` — ignore ``~/.gitconfig`` (git 2.32+).
  337. * ``GIT_CONFIG_SYSTEM=<os.devnull>`` — ignore ``/etc/gitconfig`` (git 2.32+).
  338. * ``GIT_CONFIG_NOSYSTEM=1`` — legacy belt-and-suspenders for older git.
  339. ``index_file``, if given, forces git to use a per-project index under
  340. ``store/indexes/<hash>`` so projects don't race on a shared index.
  341. """
  342. normalized_working_dir = _normalize_path(working_dir)
  343. # git child with hand-isolated config env; exact preservation — a HOME
  344. # rewrite would change which ~/.gitconfig the isolation vars are hiding.
  345. from tools.environments.local import build_subprocess_env
  346. env = selected_git_env(build_subprocess_env(scrub_secrets=False, inherit_profile_home=False))
  347. env["GIT_DIR"] = str(store)
  348. env["GIT_WORK_TREE"] = str(normalized_working_dir)
  349. env.pop("GIT_NAMESPACE", None)
  350. env.pop("GIT_ALTERNATE_OBJECT_DIRECTORIES", None)
  351. if index_file is not None:
  352. env["GIT_INDEX_FILE"] = str(index_file)
  353. else:
  354. env.pop("GIT_INDEX_FILE", None)
  355. env["GIT_CONFIG_GLOBAL"] = os.devnull
  356. env["GIT_CONFIG_SYSTEM"] = os.devnull
  357. env["GIT_CONFIG_NOSYSTEM"] = "1"
  358. return env
  359. def _run_git(
  360. args: list[str],
  361. store: Path,
  362. working_dir: str,
  363. timeout: int = _GIT_TIMEOUT,
  364. allowed_returncodes: Optional[set[int]] = None,
  365. index_file: Optional[Path] = None,
  366. extra_env: Optional[dict[str, str]] = None,
  367. ) -> tuple[bool, str, str]:
  368. """Run a git command against the shared store. Returns (ok, stdout, stderr).
  369. ``allowed_returncodes`` suppresses error logging for known/expected non-zero
  370. exits while preserving the normal ``ok = (returncode == 0)`` contract.
  371. Example: ``git diff --cached --quiet`` returns 1 when changes exist.
  372. """
  373. normalized_working_dir = _normalize_path(working_dir)
  374. if not normalized_working_dir.exists():
  375. msg = f"working directory not found: {normalized_working_dir}"
  376. logger.error("Git command skipped: %s (%s)", " ".join(["git"] + list(args)), msg)
  377. return False, "", msg
  378. if not normalized_working_dir.is_dir():
  379. msg = f"working directory is not a directory: {normalized_working_dir}"
  380. logger.error("Git command skipped: %s (%s)", " ".join(["git"] + list(args)), msg)
  381. return False, "", msg
  382. env = _git_env(store, str(normalized_working_dir), index_file=index_file)
  383. if extra_env:
  384. env.update(extra_env)
  385. git = shutil.which("git", path=env.get("PATH", ""))
  386. if git is None:
  387. return False, "", "git is not installed or not on PATH"
  388. cmd = [git, *args]
  389. allowed_returncodes = allowed_returncodes or set()
  390. try:
  391. # NUL-delimited git output contains literal filenames, not text lines.
  392. text_options = {} if "-z" in args else {"text": True, "encoding": "utf-8", "errors": "replace"}
  393. result = subprocess.run(
  394. cmd,
  395. capture_output=True,
  396. **text_options,
  397. timeout=timeout,
  398. env=env,
  399. cwd=str(normalized_working_dir),
  400. stdin=subprocess.DEVNULL,
  401. # Checkpoints fire several bare git calls per turn from the
  402. # console-less desktop/gateway backend; suppress the per-call
  403. # conhost flash on Windows (no-op on POSIX).
  404. creationflags=windows_hide_flags(),
  405. check=False,
  406. )
  407. ok = result.returncode == 0
  408. stdout = os.fsdecode(result.stdout) if "-z" in args else result.stdout.strip()
  409. stderr = result.stderr.decode("utf-8", errors="replace").strip() if "-z" in args else result.stderr.strip()
  410. if not ok and result.returncode not in allowed_returncodes:
  411. logger.error(
  412. "Git command failed: %s (rc=%d) stderr=%s",
  413. " ".join(cmd), result.returncode, stderr,
  414. )
  415. return ok, stdout, stderr
  416. except subprocess.TimeoutExpired:
  417. msg = f"git timed out after {timeout}s: {' '.join(cmd)}"
  418. logger.error(msg, exc_info=True)
  419. return False, "", msg
  420. except FileNotFoundError as exc:
  421. missing_target = getattr(exc, "filename", None)
  422. if missing_target == "git":
  423. logger.error("Git executable not found: %s", " ".join(cmd), exc_info=True)
  424. return False, "", "git not found"
  425. msg = f"working directory not found: {normalized_working_dir}"
  426. logger.error("Git command failed before execution: %s (%s)", " ".join(cmd), msg, exc_info=True)
  427. return False, "", msg
  428. except Exception as exc:
  429. logger.error("Unexpected git error running %s: %s", " ".join(cmd), exc, exc_info=True)
  430. return False, "", str(exc)
  431. def _git_out(args: list[str], store: Path, working_dir: str, rc: Optional[set[int]] = None) -> str:
  432. """stdout of a successful git call, else ``""``."""
  433. ok, out, _ = _run_git(args, store, working_dir, allowed_returncodes=rc)
  434. return out if ok else ""
  435. def _ref_tip(store: Path, working_dir: str, ref: str) -> Optional[str]:
  436. """Commit sha at ``ref``, or None when the ref does not exist yet."""
  437. return _git_out(["rev-parse", "--verify", ref + "^{commit}"], store, working_dir, {128}) or None
  438. def _list_project_refs(store: Path, working_dir: str) -> list[str]:
  439. out = _git_out(["for-each-ref", "--format=%(refname)", _REFS_PREFIX], store, working_dir, {128})
  440. return [r for r in out.splitlines() if r.strip()]
  441. def _unlink_quiet(path: Path) -> None:
  442. try:
  443. path.unlink(missing_ok=True)
  444. except OSError:
  445. pass
  446. # ---------------------------------------------------------------------------
  447. # Store initialisation + legacy migration
  448. # ---------------------------------------------------------------------------
  449. def _migrate_legacy_store(base: Path) -> Optional[Path]:
  450. """Move pre-v2 per-project shadow repos into a ``legacy-<ts>/`` dir.
  451. The pre-v2 layout had one shadow git repo per working directory directly
  452. under ``CHECKPOINT_BASE``. The v2 layout wants a single ``store/`` dir.
  453. Rather than delete the old data (users might want to recover), rename
  454. everything except our own v2 entries into ``legacy-<timestamp>/``. The
  455. legacy dir is subject to the same retention sweep and can be manually
  456. cleared with ``hermes checkpoints clear-legacy``.
  457. Returns the legacy-archive path, or None if nothing to migrate.
  458. """
  459. if not base.exists():
  460. return None
  461. store = _store_path(base)
  462. legacy_root: Optional[Path] = None
  463. # Reserved top-level entries managed by v2.
  464. reserved = {_STORE_DIRNAME, _PRUNE_MARKER_NAME}
  465. for child in list(base.iterdir()):
  466. name = child.name
  467. if name in reserved or name.startswith(_LEGACY_PREFIX):
  468. continue
  469. # Candidate: pre-v2 shadow repo (has HEAD) OR stray dir. Either way
  470. # we archive it so v2 starts clean.
  471. if legacy_root is None:
  472. stamp = time.strftime("%Y%m%d-%H%M%S")
  473. legacy_root = base / f"{_LEGACY_PREFIX}{stamp}"
  474. try:
  475. legacy_root.mkdir(parents=True, exist_ok=True)
  476. except OSError as exc:
  477. logger.warning("Could not create legacy archive dir: %s", exc)
  478. return None
  479. dest = legacy_root / name
  480. try:
  481. shutil.move(str(child), str(dest))
  482. except OSError as exc:
  483. logger.warning("Could not archive legacy checkpoint %s: %s", child, exc)
  484. # If the store still hasn't been created, create it here.
  485. _ = store
  486. if legacy_root is not None:
  487. logger.info(
  488. "Migrated pre-v2 checkpoint repos to %s. "
  489. "Clear with `hermes checkpoints clear-legacy` when safe.",
  490. legacy_root,
  491. )
  492. return legacy_root
  493. def _init_store(store: Path, working_dir: str) -> Optional[str]:
  494. """Initialise the shared shadow store if needed. Returns error or None.
  495. Also performs one-time migration of pre-v2 per-directory shadow repos
  496. into ``legacy-<timestamp>/``.
  497. """
  498. base = store.parent
  499. # One-time legacy migration before we create the store.
  500. if not store.exists():
  501. try:
  502. base.mkdir(parents=True, exist_ok=True)
  503. except OSError as exc:
  504. return f"Could not create checkpoint base: {exc}"
  505. # Only migrate if the base dir has pre-existing content that isn't
  506. # our own v2 layout.
  507. _migrate_legacy_store(base)
  508. if (store / "HEAD").exists():
  509. _write_default_excludes(store) # a store made by an older build gets new excludes too
  510. return None
  511. store.mkdir(parents=True, exist_ok=True)
  512. (store / _INDEXES_DIRNAME).mkdir(exist_ok=True)
  513. (store / _PROJECTS_DIRNAME).mkdir(exist_ok=True)
  514. # ``git init --bare`` rejects GIT_WORK_TREE, so we can't use _run_git
  515. # here (which always sets GIT_DIR + GIT_WORK_TREE). Use a raw
  516. # subprocess with just the config-isolation env vars.
  517. from tools.environments.local import build_subprocess_env
  518. init_env = selected_git_env(build_subprocess_env(scrub_secrets=False, inherit_profile_home=False))
  519. init_env["GIT_CONFIG_GLOBAL"] = os.devnull
  520. init_env["GIT_CONFIG_SYSTEM"] = os.devnull
  521. init_env["GIT_CONFIG_NOSYSTEM"] = "1"
  522. # Drop any inherited GIT_* that would interfere.
  523. for k in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_NAMESPACE",
  524. "GIT_ALTERNATE_OBJECT_DIRECTORIES"):
  525. init_env.pop(k, None)
  526. try:
  527. git = shutil.which("git", path=init_env.get("PATH", ""))
  528. if git is None:
  529. return "Shadow store init failed: git is not installed or not on PATH"
  530. result = subprocess.run(
  531. [git, "init", "--bare", str(store)],
  532. capture_output=True, text=True, encoding='utf-8', errors='replace',
  533. env=init_env, timeout=_GIT_TIMEOUT,
  534. stdin=subprocess.DEVNULL,
  535. creationflags=windows_hide_flags(),
  536. check=False,
  537. )
  538. if result.returncode != 0:
  539. return f"Shadow store init failed: {result.stderr.strip()}"
  540. except (subprocess.TimeoutExpired, FileNotFoundError) as exc:
  541. return f"Shadow store init failed: {exc}"
  542. # Per-store config (isolated by env vars above, but belt-and-suspenders).
  543. # Use the base dir as the working_dir for config commands — it always
  544. # exists since we just created the store inside it.
  545. cfg_wd = str(base)
  546. _run_git(["config", "user.email", "hermes@local"], store, cfg_wd)
  547. _run_git(["config", "user.name", "Hermes Checkpoint"], store, cfg_wd)
  548. _run_git(["config", "commit.gpgsign", "false"], store, cfg_wd)
  549. _run_git(["config", "tag.gpgSign", "false"], store, cfg_wd)
  550. _run_git(["config", "gc.auto", "0"], store, cfg_wd)
  551. _write_default_excludes(store)
  552. logger.debug("Initialised checkpoint store at %s", store)
  553. return None
  554. def _write_default_excludes(store: Path) -> None:
  555. """(Re)write the store's ``info/exclude`` when it differs from :data:`DEFAULT_EXCLUDES`."""
  556. exclude = store / "info" / "exclude"
  557. text = "\n".join(DEFAULT_EXCLUDES) + "\n"
  558. try:
  559. if exclude.is_file() and exclude.read_text(encoding="utf-8-sig") == text:
  560. return
  561. exclude.parent.mkdir(exist_ok=True)
  562. exclude.write_text(text, encoding="utf-8")
  563. except OSError as exc:
  564. logger.debug("Could not write checkpoint excludes: %s", exc)
  565. def _volume_evidence(workdir: Path) -> dict:
  566. """Record the identity of ``workdir``'s parent while the project is live.
  567. ``(st_dev, st_ino)`` of the parent directory, captured at a moment when
  568. the workdir itself is reachable, identifies the *directory* — not just
  569. the path. A mount point resolves to the mounted filesystem's root while
  570. the volume is attached and to the underlying (underlay) directory after
  571. unmount: same path, different directory, different ``(st_dev, st_ino)``.
  572. Orphan pruning uses this to distinguish "the project was deleted out of
  573. the directory we knew" from "a different directory is now visible at
  574. that path because the volume is detached".
  575. Returns ``{}`` when the workdir is not currently reachable, when the
  576. filesystem does not provide a usable directory identity (a zero
  577. ``st_dev`` or ``st_ino`` — e.g. Windows filesystems without file IDs and
  578. some network shares), or when the probe fails — callers treat all of
  579. these as "no evidence recorded" and orphan pruning stays conservative
  580. for the project (never classified as orphan; retention still applies).
  581. """
  582. try:
  583. if not workdir.exists():
  584. return {}
  585. st = workdir.parent.stat()
  586. if not st.st_dev or not st.st_ino:
  587. return {}
  588. return {
  589. "workdir_parent_dev": st.st_dev,
  590. "workdir_parent_ino": st.st_ino,
  591. }
  592. except OSError:
  593. return {}
  594. def _register_project(store: Path, working_dir: str) -> None:
  595. """Create or update ``projects/<hash>.json`` with workdir + timestamps."""
  596. dir_hash = _project_hash(working_dir)
  597. meta_path = _project_meta_path(store, dir_hash)
  598. now = time.time()
  599. meta: dict = {"workdir": str(_normalize_path(working_dir)),
  600. "created_at": now, "last_touch": now}
  601. evidence = _volume_evidence(_normalize_path(working_dir))
  602. if evidence:
  603. meta.update(evidence)
  604. if meta_path.exists():
  605. try:
  606. existing = json.loads(meta_path.read_text(encoding="utf-8-sig"))
  607. if isinstance(existing, dict):
  608. meta["created_at"] = existing.get("created_at", now)
  609. if not evidence:
  610. # Fresh probe failed — keep the previously recorded
  611. # parent identity rather than dropping it. Stale evidence
  612. # only makes pruning MORE conservative (mismatch => not
  613. # an orphan).
  614. for key in ("workdir_parent_dev", "workdir_parent_ino"):
  615. if key in existing:
  616. meta[key] = existing[key]
  617. except (OSError, ValueError):
  618. pass
  619. try:
  620. meta_path.parent.mkdir(parents=True, exist_ok=True)
  621. meta_path.write_text(json.dumps(meta), encoding="utf-8")
  622. except OSError as exc:
  623. logger.debug("Could not write project metadata %s: %s", meta_path, exc)
  624. def _touch_project(store: Path, working_dir: str) -> None:
  625. """Update last_touch for a project, preserving created_at."""
  626. dir_hash = _project_hash(working_dir)
  627. meta_path = _project_meta_path(store, dir_hash)
  628. if not meta_path.exists():
  629. _register_project(store, working_dir)
  630. return
  631. try:
  632. meta = json.loads(meta_path.read_text(encoding="utf-8-sig"))
  633. except (OSError, ValueError):
  634. meta = {}
  635. if not isinstance(meta, dict):
  636. meta = {}
  637. meta["workdir"] = str(_normalize_path(working_dir))
  638. meta["last_touch"] = time.time()
  639. meta.setdefault("created_at", meta["last_touch"])
  640. # Refresh the parent-directory identity while the project is observably
  641. # live — a remount can legitimately change it (new device, new inode).
  642. # On probe failure the previous evidence is kept: stale evidence can only
  643. # make pruning MORE conservative (mismatch => not an orphan).
  644. evidence = _volume_evidence(_normalize_path(working_dir))
  645. if evidence:
  646. meta.update(evidence)
  647. try:
  648. meta_path.write_text(json.dumps(meta), encoding="utf-8")
  649. except OSError as exc:
  650. logger.debug("Could not update project metadata %s: %s", meta_path, exc)
  651. def _list_projects(store: Path) -> list[dict]:
  652. """Return all registered projects under the store."""
  653. projects_dir = store / _PROJECTS_DIRNAME
  654. if not projects_dir.exists():
  655. return []
  656. out: list[dict] = []
  657. for meta_path in projects_dir.glob("*.json"):
  658. dir_hash = meta_path.stem
  659. try:
  660. meta = json.loads(meta_path.read_text(encoding="utf-8-sig"))
  661. except (OSError, ValueError):
  662. continue
  663. if not isinstance(meta, dict):
  664. continue
  665. meta["_hash"] = dir_hash
  666. out.append(meta)
  667. return out
  668. def _pre_v2_shadow_repos(base: Path) -> list[dict]:
  669. """Return pre-v2 per-project shadow repos still directly under ``base``.
  670. Pre-v2 layout kept one shadow git repo per working directory directly
  671. under ``CHECKPOINT_BASE`` (identified by a ``HEAD`` file). This is the
  672. single source of truth for that scan so a preview built from it (e.g.
  673. ``store_status``) always matches what ``prune_checkpoints`` deletes.
  674. """
  675. out: list[dict] = []
  676. if not base.exists():
  677. return out
  678. for child in base.iterdir():
  679. if not child.is_dir():
  680. continue
  681. if child.name == _STORE_DIRNAME or child.name.startswith(_LEGACY_PREFIX):
  682. continue
  683. if not (child / "HEAD").exists():
  684. continue
  685. workdir: Optional[str] = None
  686. marker_unreadable = False
  687. wd_marker = child / "HERMES_WORKDIR"
  688. if wd_marker.exists():
  689. try:
  690. workdir = wd_marker.read_text(encoding="utf-8-sig").strip()
  691. except (OSError, UnicodeDecodeError):
  692. # The marker is there, we just could not read it. That is
  693. # not evidence the project is gone — never delete on it.
  694. workdir = None
  695. marker_unreadable = True
  696. out.append({
  697. "path": child,
  698. "workdir": workdir,
  699. "exists": bool(workdir) and Path(workdir).exists(),
  700. "marker_unreadable": marker_unreadable,
  701. })
  702. return out
  703. def _dir_file_count(path: str) -> int:
  704. """Count checkpoint candidates, pruning the default excludes."""
  705. count = 0
  706. try:
  707. root = _normalize_path(path)
  708. if not root.is_dir():
  709. return 0
  710. directory_patterns = tuple(
  711. pattern[:-1] for pattern in DEFAULT_EXCLUDES if pattern.endswith("/")
  712. )
  713. file_patterns = tuple(
  714. pattern for pattern in DEFAULT_EXCLUDES if not pattern.endswith("/")
  715. )
  716. for current_dir, directory_names, file_names in os.walk(
  717. root, followlinks=False
  718. ):
  719. retained_directories = []
  720. for name in directory_names:
  721. candidate = Path(current_dir) / name
  722. if candidate.is_symlink():
  723. if not any(fnmatch.fnmatchcase(name, p) for p in file_patterns):
  724. count += 1
  725. elif not any(
  726. # git applies slash-less patterns (``*.log``) to directories too
  727. fnmatch.fnmatchcase(name, p) for p in directory_patterns + file_patterns
  728. ):
  729. retained_directories.append(name)
  730. if count > _MAX_FILES:
  731. return count
  732. directory_names[:] = retained_directories
  733. for name in file_names:
  734. if any(fnmatch.fnmatchcase(name, p) for p in file_patterns):
  735. continue
  736. count += 1
  737. if count > _MAX_FILES:
  738. return count
  739. except OSError:
  740. pass
  741. return count
  742. def _too_broad(abs_dir: str) -> bool:
  743. """``/``, the user's home and every ancestor of it: never a project, always huge."""
  744. home = Path.home().resolve()
  745. return Path(abs_dir) in {Path("/"), home, *home.parents}
  746. def _dir_size_bytes(path: Path) -> int:
  747. """Best-effort recursive size in bytes. Returns 0 on error."""
  748. total = 0
  749. try:
  750. for p in path.rglob("*"):
  751. try:
  752. if p.is_file():
  753. total += p.stat().st_size
  754. except OSError:
  755. continue
  756. except OSError:
  757. pass
  758. return total
  759. # ---------------------------------------------------------------------------
  760. # CheckpointManager
  761. # ---------------------------------------------------------------------------
  762. class CheckpointManager:
  763. """Manages automatic filesystem checkpoints.
  764. Designed to be owned by AIAgent. Call ``new_turn()`` at the start of
  765. each conversation turn and ``ensure_checkpoint(dir, reason)`` before
  766. any file-mutating tool call. The manager deduplicates so at most one
  767. snapshot is taken per directory per turn.
  768. Parameters
  769. ----------
  770. enabled : bool
  771. Master switch (from config / CLI flag).
  772. max_snapshots : int
  773. Keep at most this many checkpoints per directory.
  774. max_total_size_mb : int
  775. Hard ceiling on total store size. Oldest checkpoints per project
  776. are dropped when the store exceeds this after a commit.
  777. max_file_size_mb : int
  778. Skip adding any single file larger than this to a checkpoint.
  779. (Implemented via pre-stage pathspec excludes + a post-stage size check.)
  780. """
  781. def __init__(
  782. self,
  783. enabled: bool = False,
  784. max_snapshots: int = 20,
  785. max_total_size_mb: int = 500,
  786. max_file_size_mb: int = 10,
  787. ):
  788. self.enabled = enabled
  789. self.max_snapshots = max(1, int(max_snapshots))
  790. self.max_total_size_mb = max(0, int(max_total_size_mb))
  791. self.max_file_size_mb = max(0, int(max_file_size_mb))
  792. self._checkpointed_dirs: set[str] = set()
  793. # ------------------------------------------------------------------
  794. # Turn lifecycle
  795. # ------------------------------------------------------------------
  796. def new_turn(self) -> None:
  797. """Reset per-turn dedup. Call at the start of each agent iteration."""
  798. self._checkpointed_dirs.clear()
  799. def unsupported_backend_reason(self, task_id: str = "default") -> Optional[str]:
  800. """Explain why host checkpoints are off limits for a container-backed session.
  801. Classifies the task's backend at call time (nothing is remembered), so /rollback is
  802. refused before the first mutation and follows a backend change within the session."""
  803. from tools.file_tools_paths import container_backend_for_task
  804. backend = container_backend_for_task(task_id)
  805. if backend is None:
  806. return None
  807. return (
  808. f"Checkpoints are not taken for terminal.backend={backend}: "
  809. "file paths belong to the container, not this host."
  810. )
  811. # ------------------------------------------------------------------
  812. # Public API
  813. # ------------------------------------------------------------------
  814. def record_agent_write(self, file_path: str) -> None:
  815. """Record the content hash of a file Hermes just successfully wrote.
  816. Feeds the agent-write ledger used by :meth:`restore` in safe mode:
  817. at restore time, a file whose current content no longer matches the
  818. recorded hash was hand-edited by the user after Hermes last touched
  819. it, and is skipped instead of clobbered.
  820. Never raises — the ledger is best-effort bookkeeping.
  821. """
  822. if not self.enabled:
  823. return
  824. try:
  825. from tools.checkpoint_pruning import store_lock
  826. path = _normalize_path(file_path)
  827. digest = _hash_file(path)
  828. if digest is None:
  829. return
  830. with store_lock(_resolve_checkpoint_base()):
  831. store = _store_path()
  832. dir_hash = self._ledger_key(str(path))
  833. ledger = _load_ledger(store, dir_hash)
  834. ledger[str(path)] = {"sha256": digest, "ts": time.time()}
  835. _save_ledger(store, dir_hash, ledger)
  836. except Exception as exc:
  837. logger.debug("record_agent_write failed for %s: %s", file_path, exc)
  838. def _safe_restore_plan(self, working_dir: str, commit_hash: str) -> dict:
  839. """Classify files changed since ``commit_hash`` for a safe restore.
  840. Returns ``{"success", "restore": [rel...], "skipped": [rel...],
  841. "error"?}`` where ``restore`` lists files whose current content
  842. still matches what Hermes last wrote (per the agent-write ledger)
  843. and ``skipped`` lists files the user hand-edited after Hermes'
  844. last write or that Hermes never wrote at all.
  845. """
  846. hash_err = _validate_commit_hash(commit_hash)
  847. if hash_err:
  848. return {"success": False, "error": hash_err}
  849. abs_dir = str(_normalize_path(working_dir))
  850. store = _store_path()
  851. if not (store / "HEAD").exists():
  852. return {"success": False, "error": "No checkpoints exist for this directory"}
  853. dir_hash = _project_hash(abs_dir)
  854. index_file = _index_path(store, dir_hash)
  855. # Stage the current tree so the name-only diff sees new files too.
  856. _run_git(_stage_all_args(store.parent, abs_dir), store, abs_dir,
  857. timeout=_GIT_TIMEOUT * 2, index_file=index_file)
  858. ok, names_out, err = _run_git(
  859. ["diff", "--name-only", "-z", commit_hash, "--cached",
  860. *_exclude_store_args(store.parent, abs_dir)],
  861. store, abs_dir, index_file=index_file,
  862. )
  863. # Reset the index back to the project ref so it doesn't drift.
  864. _run_git(["read-tree", _ref_name(dir_hash)], store, abs_dir,
  865. index_file=index_file, allowed_returncodes={128})
  866. if not ok:
  867. return {"success": False, "error": f"Could not compute changed files: {err}"}
  868. # Read the same marker-walked project key as record_agent_write.
  869. ledger = _load_ledger(store, self._ledger_key(abs_dir))
  870. if not ledger:
  871. # No agent-write ledger yet (pre-existing store, or Hermes has
  872. # not written any files here since the ledger was introduced).
  873. # Signal callers to fall back to a full restore rather than
  874. # skipping every file.
  875. return {"success": True, "restore": [], "skipped": [],
  876. "ledger_empty": True}
  877. restore: list[str] = []
  878. skipped: list[str] = []
  879. for rel in filter(None, names_out.split("\x00")):
  880. abs_path = Path(abs_dir) / rel
  881. entry = ledger.get(str(abs_path))
  882. recorded = entry.get("sha256") if isinstance(entry, dict) else None
  883. if recorded is None:
  884. # Hermes never wrote this file (or the ledger predates it) —
  885. # do not touch it in safe mode.
  886. skipped.append(rel)
  887. continue
  888. current = _hash_file(abs_path)
  889. if current is None:
  890. # File deleted since Hermes wrote it: restoring it back is
  891. # safe — its last content was Hermes-authored.
  892. restore.append(rel)
  893. elif current == recorded:
  894. restore.append(rel)
  895. else:
  896. skipped.append(rel)
  897. return {"success": True, "restore": restore, "skipped": skipped}
  898. def ensure_checkpoint(self, working_dir: str, reason: str = "auto") -> bool:
  899. """Take a checkpoint if enabled and not already done this turn.
  900. Returns True if a checkpoint was taken, False otherwise.
  901. Never raises — all errors are silently logged.
  902. """
  903. if not self.enabled:
  904. return False
  905. abs_dir = str(_normalize_path(working_dir))
  906. # Skip root, home, and other overly broad directories
  907. if _too_broad(abs_dir):
  908. logger.debug("Checkpoint skipped: directory too broad (%s)", abs_dir)
  909. return False
  910. if abs_dir in self._checkpointed_dirs:
  911. return False
  912. self._checkpointed_dirs.add(abs_dir)
  913. try:
  914. from tools.checkpoint_pruning import store_lock
  915. with store_lock(_resolve_checkpoint_base()):
  916. return self._take(abs_dir, reason)
  917. except Exception as e:
  918. logger.debug("Checkpoint failed (non-fatal): %s", e)
  919. return False
  920. def list_checkpoints(self, working_dir: str) -> list[dict]:
  921. """List available checkpoints for a directory (most recent first)."""
  922. abs_dir = str(_normalize_path(working_dir))
  923. store = _store_path()
  924. if not (store / "HEAD").exists():
  925. return []
  926. ref = _ref_name(_project_hash(abs_dir))
  927. ok, stdout, _ = _run_git(
  928. ["log", ref, "--format=%H|%h|%aI|%s", "-n", str(self.max_snapshots)],
  929. store, abs_dir,
  930. allowed_returncodes={128, 129},
  931. )
  932. if not ok or not stdout:
  933. return []
  934. results: list[dict] = []
  935. for line in stdout.splitlines():
  936. parts = line.split("|", 3)
  937. if len(parts) == 4:
  938. entry = {
  939. "hash": parts[0],
  940. "short_hash": parts[1],
  941. "timestamp": parts[2],
  942. "reason": parts[3],
  943. "files_changed": 0,
  944. "insertions": 0,
  945. "deletions": 0,
  946. }
  947. stat_ok, stat_out, _ = _run_git(
  948. ["diff", "--shortstat", f"{parts[0]}~1", parts[0]],
  949. store, abs_dir,
  950. allowed_returncodes={128, 129},
  951. )
  952. if stat_ok and stat_out:
  953. self._parse_shortstat(stat_out, entry)
  954. results.append(entry)
  955. return results
  956. def list_all_checkpoints(self) -> list[dict]:
  957. """List checkpoints across every registered project (most recent first).
  958. Surgical reapply of PR #10633 by @nightq (#10505) onto the v2
  959. single-store layout: iterate ``projects/<hash>.json`` metadata via
  960. ``_list_projects`` instead of the pre-v2 per-shadow-dir scan. Each
  961. entry carries the extra ``workdir`` key so callers can label which
  962. project a checkpoint belongs to.
  963. """
  964. store = _store_path()
  965. if not (store / "HEAD").exists():
  966. return []
  967. results: list[dict] = []
  968. for meta in _list_projects(store):
  969. workdir = meta.get("workdir") or ""
  970. if not workdir:
  971. continue
  972. for entry in self.list_checkpoints(workdir):
  973. entry["workdir"] = workdir
  974. results.append(entry)
  975. results.sort(key=lambda x: x.get("timestamp", ""), reverse=True)
  976. return results
  977. @staticmethod
  978. def _parse_shortstat(stat_line: str, entry: dict) -> None:
  979. """Parse git --shortstat output into entry dict."""
  980. m = re.search(r'(\d+) file', stat_line)
  981. if m:
  982. entry["files_changed"] = int(m.group(1))
  983. m = re.search(r'(\d+) insertion', stat_line)
  984. if m:
  985. entry["insertions"] = int(m.group(1))
  986. m = re.search(r'(\d+) deletion', stat_line)
  987. if m:
  988. entry["deletions"] = int(m.group(1))
  989. def diff(self, working_dir: str, commit_hash: str) -> dict:
  990. """Show diff between a checkpoint and the current working tree."""
  991. from tools.checkpoint_pruning import PruneError, store_lock
  992. try:
  993. with store_lock(_resolve_checkpoint_base()):
  994. return self._diff(working_dir, commit_hash)
  995. except (PruneError, OSError) as exc:
  996. return {"success": False, "error": str(exc)}
  997. def _diff(self, working_dir: str, commit_hash: str) -> dict:
  998. hash_err = _validate_commit_hash(commit_hash)
  999. if hash_err:
  1000. return {"success": False, "error": hash_err}
  1001. abs_dir = str(_normalize_path(working_dir))
  1002. store = _store_path()
  1003. if not (store / "HEAD").exists():
  1004. return {"success": False, "error": "No checkpoints exist for this directory"}
  1005. ok, _, _err = _run_git(
  1006. ["cat-file", "-t", commit_hash], store, abs_dir,
  1007. )
  1008. if not ok:
  1009. return {"success": False, "error": f"Checkpoint '{commit_hash}' not found"}
  1010. dir_hash = _project_hash(abs_dir)
  1011. index_file = _index_path(store, dir_hash)
  1012. # Stage current state into the per-project index to compare.
  1013. _run_git(_stage_all_args(store.parent, abs_dir), store, abs_dir,
  1014. timeout=_GIT_TIMEOUT * 2, index_file=index_file)
  1015. # A checkpoint taken before the store was excluded may still contain it; never
  1016. # report the store as a change against such a checkpoint.
  1017. store_excludes = _exclude_store_args(store.parent, abs_dir)
  1018. ok_stat, stat_out, _ = _run_git(
  1019. ["diff", "--stat", commit_hash, "--cached", *store_excludes],
  1020. store, abs_dir, index_file=index_file,
  1021. )
  1022. ok_diff, diff_out, _ = _run_git(
  1023. ["diff", commit_hash, "--cached", "--no-color", *store_excludes],
  1024. store, abs_dir, index_file=index_file,
  1025. )
  1026. # Reset staged tree back to the project's last checkpoint so the
  1027. # index doesn't drift out of sync with the ref.
  1028. ref = _ref_name(dir_hash)
  1029. _run_git(["read-tree", ref], store, abs_dir,
  1030. index_file=index_file,
  1031. allowed_returncodes={128})
  1032. if not ok_stat and not ok_diff:
  1033. return {"success": False, "error": "Could not generate diff"}
  1034. return {
  1035. "success": True,
  1036. "stat": stat_out if ok_stat else "",
  1037. "diff": diff_out if ok_diff else "",
  1038. }
  1039. def session_diff(self, working_dir: str) -> dict:
  1040. """Show the cumulative diff of everything changed in this directory.
  1041. This powers ``/diff session``. It answers "what has Hermes changed
  1042. here?" by diffing the *earliest retained checkpoint* — the snapshot
  1043. taken before the first recorded edit — against the current working
  1044. tree. Because checkpoints are captured just before each file-mutating
  1045. tool call, that baseline is the pre-edit state, so the diff covers the
  1046. first edit and everything after it.
  1047. Note: checkpoints are a persistent per-project ref, so the earliest
  1048. *retained* checkpoint may predate the current session (or, after
  1049. pruning, postdate its true start). It is an approximation of "what
  1050. Hermes changed", not an exact per-session ledger.
  1051. Returns the same shape as :meth:`diff` (``{"success", "stat",
  1052. "diff"}``). When no checkpoints exist yet — nothing has been edited —
  1053. the call still *succeeds* with empty output and ``"empty": True`` so
  1054. callers can show a friendly "no changes" message rather than an error.
  1055. """
  1056. checkpoints = self.list_checkpoints(working_dir)
  1057. if not checkpoints:
  1058. return {"success": True, "stat": "", "diff": "", "empty": True}
  1059. baseline = checkpoints[-1].get("hash") or ""
  1060. result = self.diff(working_dir, baseline)
  1061. if result.get("success"):
  1062. result.setdefault("baseline", baseline)
  1063. if not result.get("stat") and not result.get("diff"):
  1064. result["empty"] = True
  1065. return result
  1066. def restore(
  1067. self,
  1068. working_dir: str,
  1069. commit_hash: str,
  1070. file_path: str | None = None,
  1071. safe: bool = False,
  1072. ) -> dict:
  1073. """Restore files to a checkpoint state.
  1074. With ``safe=True`` (full-directory restores only), files the user
  1075. hand-edited after Hermes' last write — per the agent-write ledger —
  1076. are left untouched, and only Hermes-authored changes are reverted.
  1077. The result gains ``skipped_user_edits`` listing the preserved paths,
  1078. ``skipped_oversize`` listing paths kept because the size cap excluded
  1079. them from every checkpoint, and — only when a delete failed —
  1080. ``failed_deletes`` listing paths that could not be removed.
  1081. """
  1082. from tools.checkpoint_pruning import PruneError, store_lock
  1083. try:
  1084. with store_lock(_resolve_checkpoint_base()):
  1085. return self._restore(working_dir, commit_hash, file_path, safe)
  1086. except (PruneError, OSError) as exc:
  1087. return {"success": False, "error": str(exc)}
  1088. def _restore(self, working_dir: str, commit_hash: str, file_path: str | None, safe: bool) -> dict:
  1089. hash_err = _validate_commit_hash(commit_hash)
  1090. if hash_err:
  1091. return {"success": False, "error": hash_err}
  1092. abs_dir = str(_normalize_path(working_dir))
  1093. if file_path:
  1094. path_err = _validate_file_path(file_path, abs_dir)
  1095. if path_err:
  1096. return {"success": False, "error": path_err}
  1097. store = _store_path()
  1098. if not (store / "HEAD").exists():
  1099. return {"success": False, "error": "No checkpoints exist for this directory"}
  1100. ok, _, err = _run_git(
  1101. ["cat-file", "-t", commit_hash], store, abs_dir,
  1102. )
  1103. if not ok:
  1104. return {"success": False, "error": f"Checkpoint '{commit_hash}' not found",
  1105. "debug": err or None}
  1106. ok, tree_out, err = _run_git(
  1107. ["ls-tree", "-r", "-z", commit_hash], store, abs_dir,
  1108. )
  1109. if not ok:
  1110. return {"success": False, "error": f"Could not inspect checkpoint: {err}"}
  1111. nested_repos = _gitlink_paths(tree_out)
  1112. selected_paths: Optional[list[str]] = None
  1113. if nested_repos:
  1114. blocked_repos = nested_repos
  1115. if file_path:
  1116. # Select by the path as recorded in the checkpoint, not by
  1117. # where it points now: a captured file later replaced by a
  1118. # symlink into a nested repo is still recoverable. Resolved
  1119. # confinement is enforced separately by _validate_file_path.
  1120. root_path = Path(abs_dir)
  1121. requested_path = Path(os.path.normpath(root_path / file_path))
  1122. try:
  1123. requested_rel = requested_path.relative_to(root_path)
  1124. except ValueError:
  1125. requested_rel = Path(file_path)
  1126. requested_parts = tuple(requested_rel.parts)
  1127. def _scope_intersects_nested_repo(repo_path: str) -> bool:
  1128. repo_parts = tuple(PurePosixPath(repo_path).parts)
  1129. return (
  1130. requested_parts[:len(repo_parts)] == repo_parts
  1131. or repo_parts[:len(requested_parts)] == requested_parts
  1132. )
  1133. blocked_repos = [
  1134. repo for repo in nested_repos
  1135. if _scope_intersects_nested_repo(repo)
  1136. ]
  1137. # Literal paths below a gitlink have no tree entry to match.
  1138. # Keep the explicit refusal above, but let Git resolve all
  1139. # other selections exactly as checkout does (glob, icase,
  1140. # top, exclude, literal, etc.). ls-tree lacks that pathspec
  1141. # support. A disposable index avoids touching the project's
  1142. # index, refs or files before we know rollback is possible.
  1143. if not blocked_repos:
  1144. with tempfile.TemporaryDirectory(prefix="restore-inspect-", dir=store) as scratch:
  1145. inspect_index = Path(scratch) / "index"
  1146. ok, _, err = _run_git(
  1147. ["read-tree", commit_hash], store, abs_dir,
  1148. index_file=inspect_index,
  1149. )
  1150. if ok:
  1151. ok, selected, err = _run_git(
  1152. ["ls-files", "--stage", "--error-unmatch", "-z", "--", file_path],
  1153. store, abs_dir, index_file=inspect_index,
  1154. )
  1155. if not ok:
  1156. return {"success": False, "error": f"Could not inspect restore selection: {err}"}
  1157. blocked_repos = [
  1158. record.split("\t", 1)[1]
  1159. for record in selected.split("\x00")
  1160. if record.startswith("160000 ") and "\t" in record
  1161. ]
  1162. store_paths = _store_pathspecs(store.parent, abs_dir)
  1163. selected_paths = [
  1164. path for record in selected.split("\x00") if "\t" in record
  1165. for path in [record.split("\t", 1)[1]]
  1166. if not any(path == sp or path.startswith(sp + "/") for sp in store_paths)
  1167. ]
  1168. if not selected_paths:
  1169. # An exclusion-only spec can match nothing. Checkout
  1170. # with no paths would switch the store's HEAD instead
  1171. # of restoring files, so refuse before the snapshot.
  1172. return {
  1173. "success": False,
  1174. "error": f"Restore selection matched no files in checkpoint: {file_path}",
  1175. }
  1176. if blocked_repos:
  1177. paths = ", ".join(blocked_repos)
  1178. return {
  1179. "success": False,
  1180. "error": (
  1181. "Checkpoint contains nested git repositories that were not captured "
  1182. f"({paths}); rollback was not performed"
  1183. ),
  1184. "nested_repositories": blocked_repos,
  1185. }
  1186. skipped_user_edits: list[str] = []
  1187. kept_oversize: list[str] = []
  1188. failed_deletes: list[str] = []
  1189. restore_paths: Optional[list[str]] = None
  1190. if safe and not file_path:
  1191. plan = self._safe_restore_plan(abs_dir, commit_hash)
  1192. if not plan.get("success"):
  1193. return {"success": False, "error": plan.get("error", "Safe-restore plan failed")}
  1194. if plan.get("ledger_empty"):
  1195. # No agent-write history to compare against — fall back to
  1196. # the classic full restore rather than restoring nothing.
  1197. restore_paths = None
  1198. else:
  1199. restore_paths = plan["restore"]
  1200. skipped_user_edits = plan["skipped"]
  1201. if not restore_paths:
  1202. return {
  1203. "success": True,
  1204. "restored_to": commit_hash[:8],
  1205. "reason": "nothing to restore (all changed files were user-edited)",
  1206. "directory": abs_dir,
  1207. "restored_files": [],
  1208. "skipped_user_edits": skipped_user_edits,
  1209. "skipped_oversize": [],
  1210. }
  1211. # Take a pre-rollback snapshot so you can undo the undo.
  1212. self._take(abs_dir, f"pre-rollback snapshot (restoring to {commit_hash[:8]})", prune=False)
  1213. dir_hash = _project_hash(abs_dir)
  1214. index_file = _index_path(store, dir_hash)
  1215. if restore_paths is not None:
  1216. # Split into files present in the checkpoint (checkout) and
  1217. # Hermes-created files absent from it (delete to restore state).
  1218. checkout_targets: list[str] = []
  1219. delete_targets: list[str] = []
  1220. for rel in restore_paths:
  1221. ok_in_commit, _, _ = _run_git(
  1222. ["cat-file", "-e", f"{commit_hash}:{rel}"],
  1223. store, abs_dir, allowed_returncodes={1, 128},
  1224. )
  1225. if ok_in_commit:
  1226. checkout_targets.append(rel)
  1227. elif self._exceeds_size_cap(Path(abs_dir) / rel):
  1228. # Absent from the checkpoint because ``max_file_size_mb``
  1229. # kept it out (_drop_oversize_from_index), not because
  1230. # Hermes created it. Deleting it would not restore a prior
  1231. # state — no checkpoint holds one — it would destroy the
  1232. # only copy. The ledger records a content hash, not whether
  1233. # a write created or modified the file, so an oversize path
  1234. # cannot be proven agent-created; leaving it costs a stale
  1235. # file, deleting it costs the file.
  1236. kept_oversize.append(rel)
  1237. else:
  1238. delete_targets.append(rel)
  1239. for rel in delete_targets:
  1240. try:
  1241. target = Path(abs_dir) / rel
  1242. if target.is_file() or target.is_symlink():
  1243. target.unlink()
  1244. except OSError as exc:
  1245. logger.warning(
  1246. "Safe restore: could not remove %s: %s", rel, exc,
  1247. )
  1248. failed_deletes.append(rel)
  1249. if not checkout_targets:
  1250. ok, _stdout, err = True, "", ""
  1251. else:
  1252. ok, _stdout, err = _run_git(
  1253. ["checkout", commit_hash, "--", *checkout_targets],
  1254. store, abs_dir, timeout=_GIT_TIMEOUT * 2,
  1255. index_file=index_file,
  1256. )
  1257. elif selected_paths is not None:
  1258. # Check out exactly the entries inspected above. Re-evaluating
  1259. # file_path here could select something else: _take() has just
  1260. # restaged the project index, and attribute pathspecs read
  1261. # .gitattributes from it. A NUL-separated literal pathspec file
  1262. # keeps long selections off the command line.
  1263. with tempfile.TemporaryDirectory(prefix="restore-select-", dir=store) as scratch:
  1264. spec_file = Path(scratch) / "pathspec"
  1265. spec_file.write_bytes(b"".join(os.fsencode(p) + b"\0" for p in selected_paths))
  1266. ok, _stdout, err = _run_git(
  1267. ["checkout", commit_hash, f"--pathspec-from-file={spec_file}",
  1268. "--pathspec-file-nul"],
  1269. store, abs_dir, timeout=_GIT_TIMEOUT * 2,
  1270. index_file=index_file,
  1271. extra_env={"GIT_LITERAL_PATHSPECS": "1"},
  1272. )
  1273. else:
  1274. # A checkpoint taken before the store was excluded holds a copy of it: never check
  1275. # that out over the live store (it would rewind refs to the captured ones).
  1276. store_excludes = _exclude_store_args(store.parent, abs_dir)[1:]
  1277. ok, _stdout, err = _run_git(
  1278. ["checkout", commit_hash, "--", file_path if file_path else ".", *store_excludes],
  1279. store, abs_dir, timeout=_GIT_TIMEOUT * 2,
  1280. index_file=index_file,
  1281. )
  1282. if not ok:
  1283. return {"success": False, "error": f"Restore failed: {err}",
  1284. "debug": err or None}
  1285. ok2, reason_out, _ = _run_git(
  1286. ["log", "--format=%s", "-1", commit_hash], store, abs_dir,
  1287. )
  1288. reason = reason_out if ok2 else "unknown"
  1289. result = {
  1290. "success": True,
  1291. "restored_to": commit_hash[:8],
  1292. "reason": reason,
  1293. "directory": abs_dir,
  1294. }
  1295. if file_path:
  1296. result["file"] = file_path
  1297. if restore_paths is not None:
  1298. # Only what was actually acted on. A kept oversize path was not
  1299. # restored (and a failed unlink left the file in place), and
  1300. # reporting either as restored is how the data loss above stayed
  1301. # silent: the user was told "Restored" for a file that had just
  1302. # been unlinked.
  1303. not_restored = set(kept_oversize) | set(failed_deletes)
  1304. result["restored_files"] = [
  1305. rel for rel in restore_paths if rel not in not_restored
  1306. ]
  1307. result["skipped_user_edits"] = skipped_user_edits
  1308. result["skipped_oversize"] = kept_oversize
  1309. if failed_deletes:
  1310. result["failed_deletes"] = failed_deletes
  1311. # The selected tree was needed until checkout completed. Only now may
  1312. # the safety snapshot's count/size budget make that tree unreachable.
  1313. self._prune(store, abs_dir, _ref_name(dir_hash))
  1314. return result
  1315. def _ledger_key(self, path: str) -> str:
  1316. """Agent-write ledger key: hash of the marker-walked project dir, for writer and reader alike."""
  1317. return _project_hash(self.get_working_dir_for_path(path))
  1318. def get_working_dir_for_path(self, file_path: str) -> str:
  1319. """Resolve a file path to its working directory for checkpointing."""
  1320. path = _normalize_path(file_path)
  1321. if path.is_dir():
  1322. candidate = path
  1323. else:
  1324. candidate = path.parent
  1325. # An explicitly checkpointed root owns its ledger even when an
  1326. # ancestor carries a project marker. Prefer the nearest owner.
  1327. roots = [Path(meta["workdir"]) for meta in _list_projects(_store_path()) if meta.get("workdir")]
  1328. owners = [root for root in roots if candidate.is_relative_to(root)]
  1329. if owners:
  1330. return str(max(owners, key=lambda root: len(root.parts)))
  1331. markers = {".git", "pyproject.toml", "package.json", "Cargo.toml",
  1332. "go.mod", "Makefile", "pom.xml", ".hg", "Gemfile"}
  1333. home = Path.home().resolve()
  1334. broad = {home, *home.parents}
  1335. check = candidate
  1336. while check != check.parent and check not in broad:
  1337. if any((check / m).exists() for m in markers):
  1338. return str(check)
  1339. check = check.parent
  1340. return str(candidate)
  1341. # ------------------------------------------------------------------
  1342. # Internal
  1343. # ------------------------------------------------------------------
  1344. def _take(self, working_dir: str, reason: str, *, prune: bool = True) -> bool:
  1345. """Take a snapshot. Returns True on success."""
  1346. store = _store_path()
  1347. err = _init_store(store, working_dir)
  1348. if err:
  1349. logger.debug("Checkpoint store init failed: %s", err)
  1350. return False
  1351. _touch_project(store, working_dir)
  1352. # Quick size guard — don't try to snapshot enormous directories
  1353. if _dir_file_count(working_dir) > _MAX_FILES:
  1354. logger.debug("Checkpoint skipped: >%d files in %s", _MAX_FILES, working_dir)
  1355. return False
  1356. dir_hash = _project_hash(working_dir)
  1357. index_file = _index_path(store, dir_hash)
  1358. ref = _ref_name(dir_hash)
  1359. # Seed the per-project index from the last checkpoint, if any, so the
  1360. # diff/commit machinery sees only changes since then. On first call,
  1361. # clear the index so ``git add -A`` produces a clean tree.
  1362. if index_file.exists():
  1363. # Reset index to current ref tip to avoid accumulating stale paths.
  1364. ok_ref, ref_commit, _ = _run_git(
  1365. ["rev-parse", "--verify", ref + "^{commit}"],
  1366. store, working_dir,
  1367. allowed_returncodes={128},
  1368. )
  1369. if ok_ref and ref_commit:
  1370. _run_git(
  1371. ["read-tree", ref_commit],
  1372. store, working_dir,
  1373. index_file=index_file,
  1374. allowed_returncodes={128},
  1375. )
  1376. else:
  1377. try:
  1378. index_file.unlink()
  1379. except OSError:
  1380. pass
  1381. else:
  1382. # First snapshot for this project.
  1383. index_file.parent.mkdir(parents=True, exist_ok=True)
  1384. # Drop entries the old tree carries but this build excludes BEFORE staging: ``add -A``
  1385. # re-reads every tracked path, and a locked browser-profile file would fail it each time.
  1386. _unstage_store(store, working_dir, index_file)
  1387. if not _unstage_excluded(store, working_dir, index_file):
  1388. logger.debug("Checkpoint could not drop newly excluded paths from the index")
  1389. return False
  1390. ok, err = self._stage_with_oversize_preflight(store, working_dir, index_file)
  1391. if not ok:
  1392. logger.debug("Checkpoint git-add failed: %s", err)
  1393. return False
  1394. if self.max_file_size_mb > 0:
  1395. if not self._drop_oversize_from_index(store, working_dir, index_file):
  1396. logger.debug("Checkpoint oversize index cleanup failed")
  1397. return False
  1398. # Compare against the current ref tip (not HEAD — HEAD points to a
  1399. # branch that doesn't exist on a bare store, so ``diff --cached``
  1400. # against HEAD would always show "new file" for every staged path).
  1401. ok_ref, ref_commit, _ = _run_git(
  1402. ["rev-parse", "--verify", ref + "^{commit}"],
  1403. store, working_dir,
  1404. allowed_returncodes={128},
  1405. )
  1406. has_ref = ok_ref and bool(ref_commit)
  1407. if has_ref:
  1408. ok_diff, _, _ = _run_git(
  1409. ["diff-index", "--cached", "--quiet", ref_commit],
  1410. store, working_dir,
  1411. allowed_returncodes={1},
  1412. index_file=index_file,
  1413. )
  1414. if ok_diff:
  1415. logger.debug("Checkpoint skipped: no changes in %s", working_dir)
  1416. return False
  1417. else:
  1418. # No ref yet — skip only if the index is empty.
  1419. ok_ls, ls_out, _ = _run_git(
  1420. ["ls-files", "--cached"],
  1421. store, working_dir,
  1422. index_file=index_file,
  1423. )
  1424. if ok_ls and not ls_out.strip():
  1425. logger.debug("Checkpoint skipped: empty tree in %s", working_dir)
  1426. return False
  1427. # Write tree from per-project index.
  1428. ok_tree, tree_sha, err = _run_git(
  1429. ["write-tree"], store, working_dir,
  1430. index_file=index_file,
  1431. )
  1432. if not ok_tree or not tree_sha:
  1433. logger.debug("Checkpoint write-tree failed: %s", err)
  1434. return False
  1435. # A nested repository is stored as a gitlink, not its files: say so in
  1436. # the checkpoint itself rather than only when a rollback is refused.
  1437. ok_ls, tree_out, _ = _run_git(["ls-tree", "-r", "-z", tree_sha], store, working_dir)
  1438. nested_repos = _gitlink_paths(tree_out) if ok_ls else []
  1439. if nested_repos:
  1440. logger.info("Checkpoint of %s does not capture nested git repositories: %s",
  1441. working_dir, ", ".join(nested_repos))
  1442. # Listings read the subject (%s), which ends at the first blank
  1443. # line, so keep a multiline reason (a terminal command) on one
  1444. # line or the note falls outside every listing.
  1445. reason = " ".join(
  1446. line.strip() for line in reason.splitlines() if line.strip()
  1447. ) + _uncaptured_note(nested_repos)
  1448. # Build commit (parent = current ref tip, if any).
  1449. commit_args = ["commit-tree", tree_sha, "-m", reason, "--no-gpg-sign"]
  1450. if has_ref:
  1451. commit_args = ["commit-tree", tree_sha, "-p", ref_commit, "-m", reason, "--no-gpg-sign"]
  1452. ok_commit, new_sha, err = _run_git(
  1453. commit_args, store, working_dir,
  1454. index_file=index_file,
  1455. )
  1456. if not ok_commit or not new_sha:
  1457. logger.debug("Checkpoint commit-tree failed: %s", err)
  1458. return False
  1459. # Update the per-project ref.
  1460. update_args = ["update-ref", ref, new_sha]
  1461. if has_ref:
  1462. update_args = ["update-ref", ref, new_sha, ref_commit]
  1463. ok_update, _, err = _run_git(
  1464. update_args, store, working_dir,
  1465. )
  1466. if not ok_update:
  1467. logger.debug("Checkpoint update-ref failed: %s", err)
  1468. return False
  1469. logger.debug("Checkpoint taken in %s: %s (%s)", working_dir, reason, new_sha[:8])
  1470. # Count and size budgets share one failure boundary.
  1471. if prune:
  1472. self._prune(store, working_dir, ref)
  1473. return True
  1474. def _exceeds_size_cap(self, path: Path) -> bool:
  1475. """Whether *path* is larger than ``max_file_size_mb``.
  1476. The same test :meth:`_drop_oversize_from_index` applies when building a
  1477. checkpoint, so "excluded from the checkpoint" and "refused deletion at
  1478. restore" agree on one definition. A cap of 0 disables it, and an
  1479. unstattable path is not claimed to be oversize.
  1480. """
  1481. cap = self.max_file_size_mb * 1024 * 1024
  1482. if cap <= 0:
  1483. return False
  1484. try:
  1485. return path.stat().st_size > cap
  1486. except OSError:
  1487. return False
  1488. def _stage_with_oversize_preflight(
  1489. self, store: Path, working_dir: str, index_file: Path,
  1490. ) -> tuple[bool, str]:
  1491. """Stage the worktree without opening files already over the size cap."""
  1492. add_args = _stage_all_args(store.parent, working_dir)
  1493. oversize_pathspec: Optional[Path] = None
  1494. if self.max_file_size_mb > 0:
  1495. oversize = self._oversize_paths_for_staging(store, working_dir, index_file)
  1496. if oversize is None:
  1497. return False, "oversize preflight failed"
  1498. if oversize:
  1499. # git refuses command-line pathspecs beside --pathspec-from-file, so the
  1500. # store self-exclusion rides in the same file.
  1501. oversize_pathspec = self._write_oversize_pathspec(
  1502. oversize + _store_pathspecs(store.parent, working_dir))
  1503. add_args = ["add", "-A"]
  1504. add_args.extend([
  1505. f"--pathspec-from-file={oversize_pathspec}",
  1506. "--pathspec-file-nul",
  1507. ])
  1508. try:
  1509. ok, _, err = _run_git(
  1510. add_args, store, working_dir,
  1511. timeout=_GIT_TIMEOUT * 2, index_file=index_file,
  1512. )
  1513. return ok, err
  1514. finally:
  1515. if oversize_pathspec is not None:
  1516. _unlink_quiet(oversize_pathspec)
  1517. def _oversize_paths_for_staging(
  1518. self, store: Path, working_dir: str, index_file: Path,
  1519. ) -> Optional[list[str]]:
  1520. """Return tracked or untracked paths that Git must not open for staging."""
  1521. ok, stdout, _ = _run_git(
  1522. ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
  1523. store, working_dir, index_file=index_file,
  1524. )
  1525. if not ok:
  1526. return None
  1527. abs_workdir = _normalize_path(working_dir)
  1528. return [
  1529. rel for rel in stdout.split("\x00")
  1530. if rel and self._exceeds_size_cap(abs_workdir / rel)
  1531. ]
  1532. @staticmethod
  1533. def _write_oversize_pathspec(paths: list[str]) -> Path:
  1534. """Write NUL-delimited literal excludes suitable for ``git add``."""
  1535. fd, raw_path = tempfile.mkstemp(prefix="hermes-checkpoint-", suffix=".pathspec")
  1536. path = Path(raw_path)
  1537. try:
  1538. with os.fdopen(fd, "wb") as handle:
  1539. for rel in paths:
  1540. handle.write(os.fsencode(f":(exclude,top,literal){rel}"))
  1541. handle.write(b"\0")
  1542. except Exception:
  1543. _unlink_quiet(path)
  1544. raise
  1545. return path
  1546. def _drop_oversize_from_index(
  1547. self, store: Path, working_dir: str, index_file: Path,
  1548. ) -> bool:
  1549. """Remove any staged file larger than ``max_file_size_mb`` from the index.
  1550. Lets the agent keep snapshotting source code while refusing to
  1551. swallow generated assets (datasets, model weights, logs, videos).
  1552. """
  1553. if self.max_file_size_mb <= 0:
  1554. return True
  1555. ok, stdout, _ = _run_git(
  1556. ["ls-files", "--cached", "-z"],
  1557. store, working_dir, index_file=index_file,
  1558. )
  1559. if not ok:
  1560. return False
  1561. if not stdout:
  1562. return True
  1563. # NUL separators preserve whitespace within each literal filename.
  1564. paths = [p for p in stdout.split("\x00") if p]
  1565. abs_workdir = _normalize_path(working_dir)
  1566. # Same predicate safe restore consults, called rather than restated:
  1567. # a threshold that drifted between the two would make a file both
  1568. # absent from the checkpoint and not recognised as capped at restore,
  1569. # which is precisely the deletion this change exists to prevent.
  1570. oversize = [
  1571. rel for rel in paths if self._exceeds_size_cap(abs_workdir / rel)
  1572. ]
  1573. if not oversize:
  1574. return True
  1575. logger.debug(
  1576. "Checkpoint: dropping %d oversize file(s) (>%d MB) from index",
  1577. len(oversize), self.max_file_size_mb,
  1578. )
  1579. # Use --pathspec-from-file for safety with many paths.
  1580. # Chunk into manageable batches.
  1581. BATCH = 200
  1582. for i in range(0, len(oversize), BATCH):
  1583. chunk = oversize[i:i + BATCH]
  1584. ok, _, _ = _run_git(
  1585. ["update-index", "--force-remove", "--"] + chunk,
  1586. store, working_dir, index_file=index_file,
  1587. )
  1588. if not ok:
  1589. return False
  1590. return True
  1591. def _prune(self, store: Path, working_dir: str, ref: str) -> None:
  1592. """Checkpoint-take path: snapshot-count budget plus one size round, gc deferred to the
  1593. periodic prune — a repack here held the tool call for the whole gc on a large store."""
  1594. from tools.checkpoint_pruning import Pruner, PruneError
  1595. pruner = Pruner(_run_git, store, working_dir, _GIT_TIMEOUT, _dir_size_bytes, _REFS_PREFIX)
  1596. try:
  1597. pruner.trim(ref, self.max_snapshots)
  1598. if pruner.drop_one_round(self.max_total_size_mb * 1024 * 1024):
  1599. logger.info("Checkpoint store exceeded %d MB — dropped the oldest snapshot per project; "
  1600. "space is reclaimed by the next prune", self.max_total_size_mb)
  1601. except (PruneError, OSError) as exc:
  1602. logger.warning("Checkpoint pruning stopped: %s", exc)
  1603. def format_checkpoint_list(checkpoints: list[dict], directory: str) -> str:
  1604. """Format checkpoint list for display to user."""
  1605. if not checkpoints:
  1606. return f"No checkpoints found for {directory}"
  1607. lines = [f"📸 Checkpoints for {directory}:\n"]
  1608. for i, cp in enumerate(checkpoints, 1):
  1609. ts = cp["timestamp"]
  1610. if "T" in ts:
  1611. ts = ts.split("T")[1].split("+")[0].split("-")[0][:5]
  1612. date = cp["timestamp"].split("T")[0]
  1613. ts = f"{date} {ts}"
  1614. files = cp.get("files_changed", 0)
  1615. ins = cp.get("insertions", 0)
  1616. dele = cp.get("deletions", 0)
  1617. if files:
  1618. stat = f" ({files} file{'s' if files != 1 else ''}, +{ins}/-{dele})"
  1619. else:
  1620. stat = ""
  1621. # Label per-project entries when showing the cross-project view
  1622. # (workdir key only present on list_all_checkpoints results).
  1623. workdir = cp.get("workdir", "")
  1624. if workdir and directory == "all directories":
  1625. workdir_short = Path(workdir).name or workdir
  1626. lines.append(
  1627. f" {i}. {cp['short_hash']} {ts} [{workdir_short}] {cp['reason']}{stat}"
  1628. )
  1629. else:
  1630. lines.append(f" {i}. {cp['short_hash']} {ts} {cp['reason']}{stat}")
  1631. lines.append("\n /rollback <N> restore to checkpoint N")
  1632. lines.append(" /rollback diff <N> preview changes since checkpoint N")
  1633. lines.append(" /rollback <N> <file> restore a single file from checkpoint N")
  1634. return "\n".join(lines)