disk_cleanup.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. """disk_cleanup — ephemeral file cleanup library behind the disk-cleanup plugin.
  2. Rules: test files delete at task end (age >= 0); temp after 7 days; cron-output
  3. after 14 days; empty dirs under HERMES_HOME always. Prompt-only: research
  4. (keep 10 newest, > 30 days), chrome-profile > 14 days, any file > 500 MB.
  5. Scope: strictly HERMES_HOME and /tmp/hermes-*; never ~/.hermes/logs/ or system dirs.
  6. """
  7. from __future__ import annotations
  8. import contextlib
  9. import functools
  10. import json
  11. import logging
  12. import shutil
  13. from datetime import datetime, timezone, UTC
  14. from pathlib import Path
  15. from typing import Any, Dict, Iterator, List, Optional, Tuple
  16. from hermes_constants import get_hermes_home
  17. logger = logging.getLogger(__name__)
  18. _LARGE_FILE_BYTES = 500 * 1024 * 1024
  19. def _state_file(name: str) -> Path:
  20. """``$HERMES_HOME/disk-cleanup/<name>`` — deliberately outside ``$HERMES_HOME/logs/``."""
  21. return get_hermes_home() / "disk-cleanup" / name
  22. def is_safe_path(path: Path) -> bool:
  23. """Accept only paths under HERMES_HOME or ``/tmp/hermes-*``.
  24. Rejects Windows mounts (``/mnt/c`` etc.) and any system directory.
  25. """
  26. hermes_home = get_hermes_home()
  27. try:
  28. resolved = path.resolve()
  29. except (OSError, RuntimeError):
  30. return False
  31. try:
  32. resolved.relative_to(hermes_home)
  33. return True
  34. except ValueError:
  35. pass
  36. # Allow /tmp/hermes-* explicitly. Compare resolved roots rather than raw
  37. # path parts because macOS resolves /tmp to /private/tmp.
  38. try:
  39. # Deliberate /tmp alias detection (#98854): we resolve the POSIX root
  40. # itself so the same directory matches under /private/tmp.
  41. relative_tmp = resolved.relative_to(Path("/tmp").resolve()) # no-tmp: ok — alias detection, not a scratch path
  42. except (ValueError, OSError):
  43. return False
  44. return bool(relative_tmp.parts and relative_tmp.parts[0].startswith("hermes-"))
  45. def _log(message: str) -> None:
  46. """Append to the audit log; never let it break the agent loop."""
  47. with contextlib.suppress(OSError):
  48. log_file = _state_file("cleanup.log")
  49. log_file.parent.mkdir(parents=True, exist_ok=True)
  50. ts = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S")
  51. with open(log_file, "a", encoding="utf-8") as f:
  52. f.write(f"[{ts}] {message}\n")
  53. def load_tracked() -> list[dict[str, Any]]:
  54. """Load tracked.json. Restores from ``.bak`` on corruption."""
  55. tf = _state_file("tracked.json")
  56. tf.parent.mkdir(parents=True, exist_ok=True)
  57. if not tf.exists():
  58. return []
  59. try:
  60. return json.loads(tf.read_text(encoding="utf-8-sig"))
  61. except (json.JSONDecodeError, ValueError):
  62. bak = tf.with_suffix(".json.bak")
  63. if bak.exists():
  64. try:
  65. data = json.loads(bak.read_text(encoding="utf-8-sig"))
  66. _log("WARN: tracked.json corrupted — restored from .bak")
  67. return data
  68. except Exception:
  69. pass
  70. _log("WARN: tracked.json corrupted, no backup — starting fresh")
  71. return []
  72. def save_tracked(tracked: list[dict[str, Any]]) -> None:
  73. """Atomic write: ``.tmp`` → backup old → rename."""
  74. tf = _state_file("tracked.json")
  75. tf.parent.mkdir(parents=True, exist_ok=True)
  76. tmp = tf.with_suffix(".json.tmp")
  77. tmp.write_text(json.dumps(tracked, indent=2), encoding="utf-8")
  78. if tf.exists():
  79. shutil.copy2(tf, tf.with_suffix(".json.bak"))
  80. tmp.replace(tf)
  81. ALLOWED_CATEGORIES = {
  82. "temp", "test", "research", "download", "chrome-profile", "cron-output", "other"}
  83. # Top-level HERMES_HOME dirs whose empty subdirs are never swept (last row: user project trees).
  84. _EMPTY_DIR_PROTECTED_TOP_LEVEL = frozenset({
  85. "logs", "memories", "sessions", "cron", "cronjobs",
  86. "cache", "skills", "plugins", "disk-cleanup", "optional-skills",
  87. "hermes-agent", "backups", "profiles", ".worktrees",
  88. "patches", "projects", "skins", "themes", "contributors",
  89. # Per-profile user trees bootstrapped by ``profiles.py::_PROFILE_DIRS`` (#112859).
  90. "workspace", "plans", "home",
  91. # Kanban owns its own lifecycle (workspaces GC'd at terminal state, attachments live with the task).
  92. "kanban"})
  93. _EMPTY_DIR_SWEEP_PRUNE_DIRS = frozenset({
  94. ".git", "node_modules", "venv", ".venv", "site-packages", "__pycache__"})
  95. # Top-level HERMES_HOME entries guess_category() never auto-tracks: state, logs, memory,
  96. # sessions, config/secrets, and user project trees (test_* inside projects/ is not disposable).
  97. _NEVER_TRACK_TOP_LEVEL = frozenset({
  98. "disk-cleanup", "logs", "memories", "sessions", "config.yaml",
  99. "skills", "plugins", ".env", "USER.md", "MEMORY.md", "SOUL.md",
  100. "auth.json", "hermes-agent",
  101. # User-authored project trees — never sweep empty directories inside these (#75403).
  102. # User-authored and project trees — never auto-delete files inside these just because they happen to be
  103. # named test_* or tmp_* (#75403, also #32164, #37721). ``workspace``, ``plans`` and ``home`` are the
  104. # per-profile user trees bootstrapped by ``profiles.py::_PROFILE_DIRS`` (#112859).
  105. "patches", "projects", "skins", "themes", "contributors",
  106. "profiles", "backups", "optional-skills", "workspace", "plans", "home",
  107. # Kanban task attachments/workspaces have their own lifecycle; test_* staging files there are
  108. # not disposable (#114552).
  109. "kanban"})
  110. def _is_protected_dir(p: Path) -> bool:
  111. """A tracked DIRECTORY that is HERMES_HOME itself or lives under a protected top-level tree
  112. (``cache/terminal`` holds terminal snapshots) is never rmtree'd; only its files age out."""
  113. if not p.is_dir():
  114. return False
  115. with contextlib.suppress(ValueError, OSError):
  116. rel = p.resolve().relative_to(get_hermes_home())
  117. return not rel.parts or rel.parts[0] in _EMPTY_DIR_PROTECTED_TOP_LEVEL
  118. return False
  119. @functools.lru_cache(maxsize=8) # keyed by home: a multiplexed process serves several profiles
  120. def _protected_cron_paths(home: Path) -> frozenset:
  121. """Defense-in-depth for quick(): EXACT cron control-plane paths (``cron/``, ``output/`` root,
  122. ``jobs.json``, ``.tick.lock``) never deleted regardless of stored category (stale tracked.json).
  123. Never widen to everything under ``cron/output/``: run artifacts there are disposable; only
  124. wholesale deletion of ``output/`` is fatal."""
  125. return frozenset(str(x) for parent in ("cron", "cronjobs") for base in (home / parent,)
  126. for x in (base, base / "output", base / "jobs.json", base / ".tick.lock"))
  127. # Paths under $HERMES_HOME that must NEVER be deleted by quick(), regardless of what the stored category
  128. # says. This is a defense-in-depth guard against stale tracked.json entries from before #34840.
  129. def _is_protected_cron_path(p: Path) -> bool:
  130. return str(p.resolve()) in _protected_cron_paths(get_hermes_home())
  131. def fmt_size(n: float) -> str:
  132. for unit in ("B", "KB", "MB", "GB", "TB"):
  133. if n < 1024:
  134. return f"{n:.1f} {unit}"
  135. n /= 1024
  136. return f"{n:.1f} PB"
  137. def track(path_str: str, category: str, silent: bool = False) -> bool:
  138. """Register a file for tracking. Returns True if newly tracked."""
  139. if category not in ALLOWED_CATEGORIES:
  140. _log(f"WARN: unknown category '{category}', using 'other'")
  141. category = "other"
  142. try:
  143. path = Path(path_str).resolve()
  144. except (OSError, RuntimeError):
  145. _log(f"REJECT: {path_str} (could not resolve path)")
  146. return False
  147. if not path.exists():
  148. _log(f"SKIP: {path} (does not exist)")
  149. return False
  150. if not is_safe_path(path):
  151. _log(f"REJECT: {path} (outside HERMES_HOME)")
  152. return False
  153. size = path.stat().st_size if path.is_file() else 0
  154. tracked = load_tracked()
  155. if any(item["path"] == str(path) for item in tracked):
  156. return False
  157. tracked.append({"path": str(path), "timestamp": datetime.now(UTC).isoformat(),
  158. "category": category, "size": size})
  159. save_tracked(tracked)
  160. _log(f"TRACKED: {path} ({category}, {fmt_size(size)})")
  161. if not silent:
  162. print(f"Tracked: {path} ({category}, {fmt_size(size)})")
  163. return True
  164. def forget(path_str: str) -> int:
  165. """Remove a path from tracking without deleting the file."""
  166. p = Path(path_str).resolve()
  167. tracked = load_tracked()
  168. kept = [i for i in tracked if Path(i["path"]).resolve() != p]
  169. removed = len(tracked) - len(kept)
  170. if removed:
  171. save_tracked(kept)
  172. _log(f"FORGOT: {p} ({removed} entries)")
  173. return removed
  174. def _live_items(tracked: list[dict], now: datetime, *, log_stale: bool = False) -> Iterator[tuple[dict, Path, int]]:
  175. """Yield ``(item, path, age_days)`` for entries whose path still exists."""
  176. for item in tracked:
  177. p = Path(item["path"])
  178. if p.exists():
  179. yield item, p, (now - datetime.fromisoformat(item["timestamp"])).days
  180. elif log_stale:
  181. _log(f"STALE: {p} (removed from tracking)")
  182. def _is_auto_delete(cat: str, age: int) -> bool:
  183. return cat == "test" or (cat == "temp" and age > 7) or (cat == "cron-output" and age > 14)
  184. def _prompt_group(item: dict, age: int) -> Optional[str]:
  185. """Prompt-only bucket: ``research`` / ``chrome`` / ``large`` or None."""
  186. cat = item["category"]
  187. if cat == "research" and age > 30:
  188. return "research"
  189. if cat == "chrome-profile" and age > 14:
  190. return "chrome"
  191. return "large" if item["size"] > _LARGE_FILE_BYTES else None
  192. def _delete_item(item: dict) -> Optional[str]:
  193. """Delete a tracked file/dir and audit-log it. Returns an error string on OSError, else None."""
  194. p = Path(item["path"])
  195. try:
  196. if p.is_file():
  197. p.unlink()
  198. elif p.is_dir():
  199. shutil.rmtree(p)
  200. except OSError as e:
  201. _log(f"ERROR deleting {p}: {e}")
  202. return f"{p}: {e}"
  203. _log(f"DELETED: {p} ({item['category']}, {fmt_size(item['size'])})")
  204. return None
  205. # Stored categories re-validated against guess_category() before use: old tracked.json entries
  206. # may carry "cron-output" for control-plane files or "test" for files under protected trees.
  207. _STALE_SKIP_NOTE = {"cron-output": "", "test": " — under protected tree"}
  208. def dry_run() -> tuple[list[dict], list[dict]]:
  209. """Return (auto_delete_list, needs_prompt_list) without touching files."""
  210. auto, prompt = [], []
  211. for item, p, age in _live_items(load_tracked(), datetime.now(UTC)):
  212. cat = item["category"]
  213. # Stale cron-output entries and protected dirs are skipped by quick(); omit them here too.
  214. if (cat == "cron-output" and guess_category(p) != "cron-output") or _is_protected_dir(p):
  215. continue
  216. if _is_auto_delete(cat, age):
  217. auto.append(item)
  218. elif _prompt_group(item, age):
  219. prompt.append(item)
  220. return auto, prompt
  221. def quick() -> dict[str, Any]:
  222. """Safe deterministic cleanup — no prompts. Returns ``{deleted, empty_dirs, freed, errors}``."""
  223. deleted = freed = 0
  224. new_tracked: list[dict] = []
  225. errors: list[str] = []
  226. for item, p, age in _live_items(load_tracked(), datetime.now(UTC), log_stale=True):
  227. cat = item["category"]
  228. if cat in _STALE_SKIP_NOTE and (re_cat := guess_category(p)) != cat:
  229. # Misclassified stale entry — drop it rather than delete the file.
  230. _log(f"SKIP stale {cat} entry: {p} (re-classified as {re_cat!r}{_STALE_SKIP_NOTE[cat]})")
  231. continue
  232. # Hard safety net even if re-validation above somehow let it through.
  233. if _is_protected_cron_path(p):
  234. _log(f"SKIP protected cron path: {p}")
  235. continue
  236. if _is_protected_dir(p):
  237. _log(f"SKIPPED: {p} (protected top-level dir)")
  238. continue
  239. if not _is_auto_delete(cat, age):
  240. new_tracked.append(item)
  241. continue
  242. err = _delete_item(item)
  243. if err is None:
  244. freed += item["size"]
  245. deleted += 1
  246. else:
  247. errors.append(err)
  248. new_tracked.append(item)
  249. empty_removed = _sweep_empty_dirs(get_hermes_home())
  250. save_tracked(new_tracked)
  251. _log(f"QUICK_SUMMARY: {deleted} files, {empty_removed} dirs, {fmt_size(freed)}")
  252. return {"deleted": deleted, "empty_dirs": empty_removed, "freed": freed, "errors": errors}
  253. def _subdirs(dirpath: Path, exclude: frozenset) -> list[Path]:
  254. try:
  255. return [c for c in dirpath.iterdir() if c.is_dir() and not c.is_symlink() and c.name not in exclude]
  256. except OSError:
  257. return []
  258. def _sweep_empty_dirs(hermes_home: Path) -> int:
  259. """Remove empty dirs under HERMES_HOME without recursing into durable/heavy trees (a full
  260. rglob over a checkout+venv under HERMES_HOME can stall the gateway loop for minutes).
  261. Iterative post-order so parents emptied by child removal are caught."""
  262. removed = 0
  263. stack: list[tuple[Path, bool]] = [
  264. (top, False) for top in _subdirs(hermes_home, _EMPTY_DIR_PROTECTED_TOP_LEVEL | _EMPTY_DIR_SWEEP_PRUNE_DIRS)]
  265. while stack:
  266. dirpath, visited = stack.pop()
  267. if visited:
  268. with contextlib.suppress(OSError):
  269. if not any(dirpath.iterdir()):
  270. dirpath.rmdir()
  271. removed += 1
  272. _log(f"DELETED: {dirpath} (empty dir)")
  273. continue
  274. stack.append((dirpath, True))
  275. stack.extend((child, False) for child in _subdirs(dirpath, _EMPTY_DIR_SWEEP_PRUNE_DIRS))
  276. return removed
  277. def status() -> dict[str, Any]:
  278. """Return per-category breakdown and top 10 largest tracked files."""
  279. tracked = load_tracked()
  280. cats: dict[str, dict] = {}
  281. for item in tracked:
  282. c = cats.setdefault(item["category"], {"count": 0, "size": 0})
  283. c["count"] += 1
  284. c["size"] += item["size"]
  285. existing = sorted(((i["path"], i["size"], i["category"]) for i in tracked
  286. if Path(i["path"]).exists()), key=lambda x: x[1], reverse=True)
  287. return {"categories": cats, "top10": existing[:10], "total_tracked": len(tracked)}
  288. def format_status(s: dict[str, Any]) -> str:
  289. """Human-readable status string (for slash command output)."""
  290. lines = [f"{'Category':<20} {'Files':>6} {'Size':>10}", "-" * 40]
  291. cats = s["categories"]
  292. for cat, d in sorted(cats.items(), key=lambda x: x[1]["size"], reverse=True):
  293. lines.append(f"{cat:<20} {d['count']:>6} {fmt_size(d['size']):>10}")
  294. if not cats:
  295. lines.append("(nothing tracked yet)")
  296. lines += ["", "Top 10 largest tracked files:"]
  297. if not s["top10"]:
  298. lines.append(" (none)")
  299. for rank, (path, size, cat) in enumerate(s["top10"], 1):
  300. lines.append(f" {rank:>2}. {fmt_size(size):>8} [{cat}] {path}")
  301. return "\n".join(lines)
  302. _TEST_PATTERNS = ("test_", "tmp_")
  303. _TEST_SUFFIXES = (".test.py", ".test.js", ".test.ts", ".test.md")
  304. def _git_tracks(path: Path) -> bool:
  305. """True when the git repo enclosing *path* (at any depth) tracks it.
  306. Asked per candidate: ``guess_category`` only reaches this for ``test_*``/``tmp_*``
  307. names, so one ``ls-files --error-unmatch`` is cheap, needs no cache that could outlive
  308. the index (a file committed after first classification is seen immediately), and covers
  309. both a HERMES_HOME that IS a checkout and one nested in an enclosing repo (a ``~/.git``
  310. dotfiles repo tracking ``~/.hermes/scripts/test_x.py``). Unlike a bare ``.git`` probe
  311. above HERMES_HOME, an exact tracked check cannot make untracked scratch look Git-owned.
  312. ``:(literal)`` stops git globbing the name (``test_[1].py`` must not match ``test_1.py``).
  313. Git missing / not a repo / file untracked all mean "not tracked".
  314. """
  315. from hermes_cli.source_check import _git_ok
  316. return _git_ok(["-C", str(path.parent), "ls-files", "--error-unmatch", "--",
  317. ":(literal)" + path.name], timeout=5)
  318. def _inside_git_worktree(path: Path) -> bool:
  319. """True if *path* is Git-owned: a ``.git`` entry (a directory in a normal checkout, a
  320. pointer FILE in a linked worktree) exists on the directory chain below HERMES_HOME, or
  321. an enclosing repo (HERMES_HOME itself, or one above it) actually TRACKS the file.
  322. Files whose repo tracks them are Git-owned — a ``test_*`` file in a worktree is typically
  323. a committed regression test, not session scratch (#115295).
  324. Only ``.git`` entries strictly BELOW ``HERMES_HOME`` count for the parent-chain probe: a
  325. home kept in a dotfiles repo (``~/.git``) would otherwise make every scratch file look
  326. Git-owned. Git is only asked when a ``.git`` exists at or above HERMES_HOME; otherwise no
  327. repo can track the file and the spawn is skipped.
  328. """
  329. resolved = path.resolve()
  330. parents = list(resolved.parents)
  331. above: list[Path] = []
  332. with contextlib.suppress(ValueError):
  333. i = parents.index(get_hermes_home())
  334. parents, above = parents[:i], parents[i:]
  335. if any((parent / ".git").exists() for parent in parents):
  336. return True
  337. return any((parent / ".git").exists() for parent in above) and _git_tracks(resolved)
  338. def guess_category(path: Path) -> Optional[str]:
  339. """Category label for *path*, or None if we shouldn't track it (``post_tool_call`` hook)."""
  340. if not is_safe_path(path):
  341. return None
  342. with contextlib.suppress(ValueError): # not under HERMES_HOME (/tmp/hermes-*) — name rules only
  343. rel = path.resolve().relative_to(get_hermes_home())
  344. top = rel.parts[0] if rel.parts else ""
  345. if top in _NEVER_TRACK_TOP_LEVEL or _is_protected_dir(path):
  346. return None
  347. if top in ("cron", "cronjobs"):
  348. # Only the disposable ``output/`` subtree; control-plane state (jobs.json,
  349. # .tick.lock) must never be tracked — deleting it wipes the scheduler registry.
  350. return "cron-output" if len(rel.parts) >= 3 and rel.parts[1] == "output" else None
  351. if top == "cache":
  352. return "temp"
  353. name = path.name
  354. if name.startswith(_TEST_PATTERNS) or name.endswith(_TEST_SUFFIXES):
  355. # Git-owned trees manage their own files: never classify a test_* there as disposable,
  356. # so neither tracking nor quick() (which re-validates stored "test" entries through
  357. # this function) touches it (#115295).
  358. return None if _inside_git_worktree(path) else "test"
  359. return None