hermes_constants_scratch.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. """Scratch-dir retention: idle detection plus the reaping an idle tree needs before it goes.
  2. Deleting an idle ``cache/scratch/<entry>`` is not enough on its own: a lane's e2e run leaves
  3. headless browsers whose cwd was inside the tree (they survived for days with a ``(deleted)``
  4. cwd), and a repo whose linked worktree lived in the tree keeps a dangling registration until
  5. someone runs ``git worktree prune``. Both are reaped here, right before ``rmtree``.
  6. A process Hermes itself started is never an orphan, though: an idle entry a registered live
  7. process still works inside is kept, not reaped (#132401). Ownership comes from the two
  8. registries Hermes writes, not from a marker file a lane must remember to drop.
  9. """
  10. from __future__ import annotations
  11. import contextlib
  12. import json
  13. import logging
  14. import os
  15. import shutil
  16. import subprocess
  17. import time
  18. from logging.handlers import RotatingFileHandler
  19. from pathlib import Path
  20. import hermes_constants_scratch_stuck as stuck
  21. logger = logging.getLogger(__name__)
  22. # How long a TERMed process gets before KILL; browsers exit well within this.
  23. _REAP_GRACE_SECONDS = 3.0
  24. # ``.git`` files (linked worktrees) are looked for this deep; lanes nest repo/tree/subtree.
  25. _GIT_FILE_MAX_DEPTH = 4
  26. # logs/scratch-prune.log: one line per entry deleted, per process signalled, and per
  27. # entry rescued or vanished mid-prune (#132401). Same fixed rotation as tool_calls.log.
  28. _PRUNE_LOG_MAX_BYTES = 5 * 1024 * 1024
  29. _PRUNE_LOG_BACKUPS = 3
  30. # Paths and process names go in with %r: a control character (a newline in a legal POSIX name)
  31. # is written escaped, so it cannot split a record or forge one.
  32. class _PruneLogHandler(RotatingFileHandler):
  33. """Rollover that tolerates a concurrent prune of the same home rotating first: its rename
  34. of ``scratch-prune.log`` can land between this handler's existence check and its own
  35. rename. The rotation is then already done, so the record goes to the new file instead of
  36. being dropped with a FileNotFoundError."""
  37. def doRollover(self) -> None:
  38. with contextlib.suppress(FileNotFoundError):
  39. super().doRollover()
  40. def _open_prune_log(log_file: Path | None) -> logging.Logger:
  41. """Logger for one prune's records, with its own RotatingFileHandler on *log_file*, so a
  42. prune that runs at boot (before ``setup_logging()``) still leaves them on disk. Records also
  43. propagate through this module's logger to whatever logging is set up.
  44. One unregistered logger per prune, closed when it ends: no process holds the file open
  45. between prunes (an open handle blocks rollover on Windows), and prunes of two homes never
  46. share a handler. Without a *log_file* (test-runner temp roots) this module's logger is used.
  47. """
  48. if log_file is None:
  49. return logger
  50. try:
  51. log_file.parent.mkdir(parents=True, exist_ok=True)
  52. except OSError as exc:
  53. logger.warning("scratch prune: cannot create %s: %s", log_file.parent, exc)
  54. return logger
  55. audit = logging.Logger(logger.name, logging.INFO)
  56. audit.parent = logger
  57. # backslashreplace: a legal POSIX name that is not UTF-8 is written escaped instead of
  58. # dropping the whole record.
  59. handler = _PruneLogHandler(
  60. log_file, maxBytes=_PRUNE_LOG_MAX_BYTES, backupCount=_PRUNE_LOG_BACKUPS, encoding="utf-8",
  61. errors="backslashreplace", delay=True,
  62. )
  63. handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
  64. audit.addHandler(handler)
  65. return audit
  66. def _tree_bytes(path: Path) -> int:
  67. """Apparent size of *path* and everything under it, symlinks not followed."""
  68. try:
  69. total = os.lstat(path).st_size
  70. except OSError:
  71. return 0
  72. stack = [str(path)] if path.is_dir() and not path.is_symlink() else []
  73. while stack:
  74. try:
  75. with os.scandir(stack.pop()) as it:
  76. for child in it:
  77. try:
  78. total += child.stat(follow_symlinks=False).st_size
  79. except OSError:
  80. continue
  81. if child.is_dir(follow_symlinks=False):
  82. stack.append(child.path)
  83. except OSError:
  84. continue
  85. return total
  86. _TOUCHED = "touched"
  87. _SCAN_FAILED = "scan-failed"
  88. _IDLE = "idle"
  89. def subtree_touch_state(path: Path, cutoff: float) -> str:
  90. """Tri-state form of :func:`subtree_touched_since`: ``touched`` (recent write seen),
  91. ``scan-failed`` (unreadable — idle cannot be established, so the entry is kept, but
  92. a failed scan is not activity and is never reported as such), or ``idle``.
  93. Same walk, same stop-at-first-recent shortcut, same symlink rules. A hardlinked file
  94. (``st_nlink > 1``) does not count either: its mtime is the inode's, and
  95. ``git clone --local`` hardlinks packs that git freshens from the source repo."""
  96. return _scan_subtree(path, cutoff)[0]
  97. def _scan_subtree(path: Path, cutoff: float) -> tuple[str, str | None]:
  98. """``(state, denied_dir)``: *denied_dir* is the directory below *path* that refused
  99. listing with EACCES/EPERM when that is why the scan failed (else None)."""
  100. try:
  101. if os.lstat(path).st_mtime >= cutoff:
  102. return _TOUCHED, None
  103. if not path.is_dir() or path.is_symlink():
  104. return _IDLE, None
  105. except OSError:
  106. return _SCAN_FAILED, None
  107. stack = [str(path)]
  108. while stack:
  109. current = stack.pop()
  110. try:
  111. with os.scandir(current) as it:
  112. for child in it:
  113. try:
  114. st = child.stat(follow_symlinks=False)
  115. except OSError as exc:
  116. return _SCAN_FAILED, current if stuck.is_permission_error(exc) else None
  117. if st.st_mtime >= cutoff and not (st.st_nlink > 1 and child.is_file(follow_symlinks=False)):
  118. return _TOUCHED, None
  119. if child.is_dir(follow_symlinks=False):
  120. stack.append(child.path)
  121. except OSError as exc:
  122. return _SCAN_FAILED, current if stuck.is_permission_error(exc) else None
  123. return _IDLE, None
  124. def subtree_touched_since(path: Path, cutoff: float) -> bool:
  125. """True when *path* or anything beneath it has an mtime at or after *cutoff*.
  126. Stops at the first recent entry, so a live tree costs one hit and only a truly idle
  127. tree pays for the full walk (once, right before it is deleted). Symlinks are never
  128. followed: a link into the repo would make the target's activity keep the entry alive.
  129. An unreadable entry is kept: an incomplete scan cannot establish that it is idle —
  130. so True here means "touched **or** uninspectable", and callers that need to tell the
  131. two apart (the mid-prune audit) use :func:`subtree_touch_state` instead.
  132. """
  133. return subtree_touch_state(path, cutoff) != _IDLE
  134. def _under(path: str, root: str) -> bool:
  135. return path == root or path.startswith(root.rstrip(os.sep) + os.sep)
  136. # The ancestry of the Hermes command that launched a detached pass (hermes_constants_scratch_detach),
  137. # as ``(pid, create_time)`` pairs the launcher computed before it spawned the pass: the pass is
  138. # reparented once that command exits, so its own ancestry no longer reaches the user's shell.
  139. # ``None``: the launcher could not compute it, so the pass signals nothing and keeps every entry
  140. # a live process works in.
  141. _launcher_lineage: frozenset[tuple[int, float]] | None = frozenset()
  142. _HAS_PROC = os.path.isdir("/proc/self")
  143. def _birth_stamp(proc) -> float:
  144. """A start stamp for *proc* that a clock step cannot move. Linux: ``starttime`` from
  145. ``/proc/<pid>/stat`` (clock ticks since boot; psutil's ``create_time()`` adds a wall-clock
  146. boot time, so an NTP/VM/resume step shifts it by whole seconds). Elsewhere:
  147. :func:`hermes_cli.process_identity.process_birth` (the unadjusted kernel start on macOS).
  148. On a ``/proc`` host a failed read raises: falling back to a wall-clock value would compare
  149. an epoch against recorded ticks and disown a live process."""
  150. if _HAS_PROC:
  151. # Bytes, not text: comm is cut at 15 bytes and can end mid-UTF-8 sequence. After the
  152. # last ")" (comm may hold spaces or ")") fields start at #3, so #22 is index 19.
  153. stat = Path(f"/proc/{proc.pid}/stat").read_bytes()
  154. return float(int(stat.rsplit(b")", 1)[1].split()[19]))
  155. from hermes_cli.process_identity import process_birth
  156. return process_birth(proc)
  157. def _same_birth(proc, started: float) -> bool:
  158. """Exact on Linux (integer ticks); a 2 s tolerance where the stamp is a float epoch."""
  159. now = _birth_stamp(proc)
  160. return now == started if _HAS_PROC else abs(now - started) < 2.0
  161. def lineage_with_start_times() -> list[tuple[int, float]] | None:
  162. """This process and its ancestors with their create times, the set :func:`_own_lineage`
  163. computes in-process, in a form that stays checkable after they exit. ``None`` when it
  164. cannot be computed completely (no psutil, a process that refuses inspection)."""
  165. try:
  166. import psutil
  167. except ImportError:
  168. return None
  169. chain: dict[int, float] = {}
  170. try:
  171. proc = psutil.Process()
  172. while proc is not None and proc.pid not in chain:
  173. chain[proc.pid] = _birth_stamp(proc)
  174. proc = proc.parent()
  175. except (psutil.Error, OSError, IndexError, ValueError):
  176. return None
  177. return list(chain.items())
  178. def _own_lineage() -> set[int] | None:
  179. """This process and its ancestors: never reap the shell that is running the prune. A
  180. detached pass adds its launcher's ancestors that still run under the same create time (a
  181. reused pid is not spared), and returns ``None`` (spare every process) without them."""
  182. import psutil
  183. if _launcher_lineage is None:
  184. return None
  185. pids: set[int] = set()
  186. try:
  187. proc = psutil.Process()
  188. while proc is not None and proc.pid not in pids:
  189. pids.add(proc.pid)
  190. proc = proc.parent()
  191. except (psutil.Error, OSError):
  192. pass
  193. for pid, started in _launcher_lineage:
  194. try:
  195. if _same_birth(psutil.Process(pid), started):
  196. pids.add(pid)
  197. except psutil.NoSuchProcess:
  198. continue
  199. except (psutil.Error, OSError, IndexError, ValueError):
  200. pids.add(pid) # unreadable is not proof of a reused pid: spare it
  201. return pids
  202. def _registered_live_pids(home: Path) -> set[int]:
  203. """Live pids Hermes registered for *home*: the machine spawn ledger (every long-lived entry
  204. point and MCP helper, ``hermes_cli.process_identity``) and the background-process checkpoint
  205. (``tools.process_registry``, ``processes.json``). A recorded start time that no longer
  206. matches is a reused pid, not ownership."""
  207. import psutil
  208. from hermes_cli.process_identity import entry_still_running, spawner_is_dead
  209. from hermes_constants import get_default_hermes_root
  210. root = get_default_hermes_root(home=home)
  211. sources = [(root / "spawn-ledger.json", "create_time")]
  212. sources += [(base / "processes.json", "host_start_time") for base in {home, root}]
  213. pids: set[int] = set()
  214. for path, stamp_key in sources:
  215. try:
  216. entries = json.loads(path.read_text(encoding="utf-8-sig"))
  217. except (OSError, ValueError):
  218. continue
  219. for entry in entries if isinstance(entries, list) else []:
  220. pid, stamp = (entry.get("pid"), entry.get(stamp_key)) if isinstance(entry, dict) else (None, None)
  221. if not isinstance(pid, int) or pid <= 0 or entry.get("pid_scope", "host") != "host":
  222. continue
  223. # An MCP helper whose recorded spawner is provably gone is an orphan, not owned: the
  224. # rule reap_orphaned_mcp_helpers applies. Long-lived entry points (gateway, serve) that
  225. # outlive their launcher stay owned; an unprovable spawner keeps ownership.
  226. if entry.get("purpose") == "mcp-helper" and spawner_is_dead(entry) is True:
  227. continue
  228. try:
  229. if stamp_key == "create_time":
  230. alive = entry_still_running(entry) # boot ticks survive a clock step
  231. elif stamp is None:
  232. alive = psutil.pid_exists(pid)
  233. else:
  234. from gateway.status import get_process_start_time
  235. alive = get_process_start_time(pid) == stamp
  236. except (psutil.Error, OSError, TypeError, ValueError):
  237. continue
  238. if alive:
  239. pids.add(pid)
  240. return pids
  241. def _owned_by_hermes(proc, registered: set[int]) -> bool:
  242. """Registered itself, or a descendant of a live registered process (a background shell's
  243. child). An orphan whose spawning shell is gone was reparented away from that lineage, so
  244. it is not owned and is still reaped."""
  245. import psutil
  246. try:
  247. # pid 1 is excluded as an ancestor: a Hermes running as a container init adopts every orphan.
  248. return proc.pid in registered or any(p.pid in registered and p.pid != 1 for p in proc.parents())
  249. except (psutil.Error, OSError):
  250. return False
  251. def reap_processes_rooted_in(scratch_root: Path, doomed: list[Path], audit: logging.Logger = logger) -> int:
  252. """TERM (then KILL) same-user processes whose cwd is inside an entry about to be
  253. pruned, or is a path under *scratch_root* that no longer exists. Returns the count.
  254. Only the cwd is consulted: a process merely holding a file open inside scratch (an
  255. editor, a log tail) is not ours to kill, but one *living* in a directory we are
  256. about to delete, or in one already gone, has nothing left to run for.
  257. A process Hermes owns (:func:`_owned_by_hermes`) is never signalled; each doomed entry it
  258. works inside is dropped from *doomed* in place and recorded as kept.
  259. """
  260. import psutil
  261. root = os.path.realpath(str(scratch_root))
  262. targets = [os.path.realpath(str(p)) for p in doomed]
  263. skip = _own_lineage()
  264. uid = os.getuid() if hasattr(os, "getuid") else None
  265. victims: list[tuple[psutil.Process, str]] = []
  266. # The pruner's own lineage is never signalled, but its cwd still keeps an entry: a live
  267. # session that launched this Hermes command may be working inside it.
  268. lineage: list[tuple[psutil.Process, str]] = []
  269. for proc in psutil.process_iter(["pid"]):
  270. try:
  271. if uid is not None and proc.uids().real != uid:
  272. continue
  273. cwd = proc.cwd()
  274. except (psutil.Error, OSError):
  275. continue
  276. if not cwd:
  277. continue
  278. deleted = cwd.endswith(" (deleted)")
  279. cwd_path = cwd[: -len(" (deleted)")] if deleted else cwd
  280. if not _under(cwd_path, root):
  281. continue
  282. if deleted or not os.path.exists(cwd_path) or any(_under(cwd_path, t) for t in targets):
  283. (lineage if skip is None or proc.pid in skip else victims).append((proc, cwd_path))
  284. # The owning home comes from the scratch path as configured, not its resolved target:
  285. # a cache/scratch symlinked onto another disk still belongs to <home>.
  286. registered = _registered_live_pids(Path(scratch_root).absolute().parent.parent) if victims else set()
  287. owned = lineage + [(proc, cwd_path) for proc, cwd_path in victims if _owned_by_hermes(proc, registered)]
  288. for proc, cwd_path in owned:
  289. for entry in [e for e in doomed if _under(cwd_path, os.path.realpath(str(e)))]:
  290. doomed.remove(entry)
  291. try:
  292. name = proc.name()
  293. except (psutil.Error, OSError):
  294. name = "?"
  295. audit.info("scratch prune: kept %r — live Hermes process pid=%d (%r) works inside",
  296. os.fspath(entry), proc.pid, name)
  297. # Re-filter against the shrunk *doomed*: an orphan in a kept entry is spared with it (the
  298. # reap only exists to clear the way for rmtree); one whose cwd is already gone still goes.
  299. victims = [(proc, cwd_path) for proc, cwd_path in victims if (proc, cwd_path) not in owned
  300. and (not os.path.exists(cwd_path) or any(_under(cwd_path, os.path.realpath(str(e))) for e in doomed))]
  301. if not victims:
  302. return 0
  303. for proc, cwd_path in victims:
  304. try:
  305. name = proc.name()
  306. proc.terminate()
  307. except (psutil.Error, OSError):
  308. continue
  309. audit.info("scratch prune: sent TERM pid=%d name=%r cwd=%r", proc.pid, name, cwd_path)
  310. _, alive = psutil.wait_procs([proc for proc, _ in victims], timeout=_REAP_GRACE_SECONDS)
  311. for proc in alive:
  312. try:
  313. # A zombie already exited; only its parent has not collected it yet.
  314. if proc.status() == psutil.STATUS_ZOMBIE:
  315. continue
  316. proc.kill()
  317. except (psutil.Error, OSError):
  318. continue
  319. audit.info("scratch prune: sent KILL pid=%d (still running after TERM)", proc.pid)
  320. logger.info("scratch prune: reaped %d process(es) rooted in pruned entries", len(victims))
  321. return len(victims)
  322. def _linked_worktree_repos(entry: Path) -> set[str]:
  323. """Repos whose linked worktrees live inside *entry* (``.git`` FILES, ``gitdir: <repo>/.git/worktrees/<n>``)."""
  324. repos: set[str] = set()
  325. stack = [(str(entry), 0)]
  326. while stack:
  327. current, depth = stack.pop()
  328. try:
  329. with os.scandir(current) as it:
  330. for child in it:
  331. if child.name == ".git" and child.is_file(follow_symlinks=False):
  332. try:
  333. line = Path(child.path).read_text(encoding="utf-8-sig", errors="replace").strip()
  334. except OSError:
  335. continue
  336. if line.startswith("gitdir:"):
  337. gitdir = Path(line[len("gitdir:"):].strip())
  338. # <repo>/.git/worktrees/<name> -> <repo>
  339. if gitdir.parent.name == "worktrees" and gitdir.parent.parent.name == ".git":
  340. repos.add(str(gitdir.parent.parent.parent))
  341. elif child.is_dir(follow_symlinks=False) and depth < _GIT_FILE_MAX_DEPTH \
  342. and child.name not in ("node_modules", ".venv", "venv"):
  343. stack.append((child.path, depth + 1))
  344. except OSError:
  345. continue
  346. return repos
  347. def release_git_worktrees(repos: set[str]) -> None:
  348. """``git worktree prune`` in each repo: drops registrations whose tree we just deleted."""
  349. for repo in sorted(repos):
  350. if not os.path.isdir(repo):
  351. continue
  352. try:
  353. subprocess.run(
  354. ["git", "-C", repo, "worktree", "prune"],
  355. stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace",
  356. timeout=15, check=False,
  357. )
  358. except (OSError, subprocess.SubprocessError) as exc:
  359. logger.debug("git worktree prune in %s failed: %s", repo, exc)
  360. def prune_idle_entries(
  361. root: Path, max_idle_hours: float, skip_names: frozenset[str], log_file: Path | None = None,
  362. ) -> int:
  363. """Delete top-level entries of *root* with no write anywhere in their subtree for
  364. *max_idle_hours*, reaping processes and worktree registrations rooted in them first.
  365. Each removal, each signalled process and each entry rescued mid-prune (touched
  366. again after selection) is recorded in *log_file*. Returns the count actually
  367. removed."""
  368. audit = _open_prune_log(log_file)
  369. try:
  370. return _prune_idle_entries(root, max_idle_hours, skip_names, audit)
  371. finally:
  372. if audit is not logger:
  373. for handler in audit.handlers:
  374. handler.close()
  375. def _prune_idle_entries(
  376. root: Path, max_idle_hours: float, skip_names: frozenset[str], audit: logging.Logger,
  377. ) -> int:
  378. cutoff = time.time() - max_idle_hours * 3600
  379. # Entries already known to hold files this user cannot read or delete are skipped before
  380. # any walk, until that is fixed (hermes_constants_scratch_stuck).
  381. skip_names = skip_names | stuck.load_skips(root) | {stuck.STUCK_STATE_NAME, stuck.STUCK_LOCK_NAME}
  382. try:
  383. entries = [e for e in root.iterdir() if e.name not in skip_names]
  384. except OSError:
  385. return 0
  386. doomed = []
  387. for e in entries:
  388. state, denied_dir = _scan_subtree(e, cutoff)
  389. if state == _IDLE:
  390. doomed.append(e)
  391. elif denied_dir is not None and stuck.tracking_supported():
  392. # A dir this user may not list (root-owned 0700) fails every idle scan: kept
  393. # (fail-closed) as before, but reported once and no longer re-scanned hourly.
  394. stuck.record(root, e, [denied_dir], audit)
  395. # Runs even with nothing to delete: orphans whose cwd was removed by an earlier pass
  396. # (or by hand) are found by the deleted-cwd rule, not by membership in ``doomed``.
  397. try:
  398. reap_processes_rooted_in(root, doomed, audit)
  399. # health: allow BLE001 -- fail-closed boundary: any failure leaves who works inside unknown; audited below
  400. except Exception as exc: # psutil missing or restricted host
  401. audit.info("scratch prune: process check failed (%s: %r), nothing deleted this pass",
  402. type(exc).__name__, str(exc))
  403. return 0
  404. if not doomed:
  405. return 0
  406. repos: set[str] = set()
  407. removed = 0
  408. for entry in doomed:
  409. # Worktree scan first: a registration whose tree an earlier pass (or hand)
  410. # removed is stale even in a rescued entry, and the re-check below sits as
  411. # close to the delete as the loop allows.
  412. if entry.is_dir() and not entry.is_symlink():
  413. repos |= _linked_worktree_repos(entry)
  414. # Last-moment re-validation (#132401 C1/F1): ``doomed`` is a snapshot from
  415. # before the reap ran, and a writer whose cwd is outside scratch — invisible
  416. # to the reap — can land fresh work in that window. The snapshot is a
  417. # candidate list, never a verdict: anything touched since selection is
  418. # rescued here, not deleted. A re-scan that cannot read the entry is also
  419. # fail-closed (the entry is kept), but a failed scan is not activity — the
  420. # audit names the failed re-scan instead of inventing a touch that did not
  421. # happen, so operators can tell an uninspectable candidate from verified
  422. # recent activity (P3 on #134173).
  423. # Size is collected before the re-check, not after it: a full-tree walk between
  424. # the re-check and rmtree reopens the window for a write landing during that
  425. # walk to be destroyed (review F6 on #139850).
  426. size = _tree_bytes(entry)
  427. state = subtree_touch_state(entry, cutoff)
  428. if state != _IDLE:
  429. # Only a missing path is a vanish: a stat that raises PermissionError means
  430. # the entry is there but unreadable, and reporting it as vanished would be
  431. # the same invented reason the audit exists to prevent.
  432. try:
  433. os.lstat(entry)
  434. entry_gone = False
  435. except FileNotFoundError:
  436. entry_gone = True
  437. except OSError:
  438. entry_gone = False
  439. if entry_gone:
  440. audit.info("scratch prune: entry %r vanished since selection", os.fspath(entry))
  441. elif state == _TOUCHED:
  442. audit.info(
  443. "scratch prune: rescued %r — touched since selection, kept",
  444. os.fspath(entry),
  445. )
  446. else:
  447. audit.info(
  448. "scratch prune: kept %r — activity re-scan failed, left untouched",
  449. os.fspath(entry),
  450. )
  451. continue
  452. denied: list[str] = []
  453. try:
  454. if entry.is_dir() and not entry.is_symlink():
  455. denied = stuck.rmtree_collecting_permission_errors(entry)
  456. else:
  457. entry.unlink()
  458. except OSError as exc:
  459. audit.info("scratch prune: could not remove %r: %s", os.fspath(entry), exc)
  460. continue
  461. # rmtree ignores errors, so only a path that is really gone counts as removed.
  462. # Only an explicit lstat proves absence: lexists() swallows OSError from lstat,
  463. # so an entry that stayed (rmtree left residue) but cannot be inspected would be
  464. # counted and recorded as removed — the same invented reason the vanish rule
  465. # above refuses to invent (P2 on #134173).
  466. try:
  467. os.lstat(entry)
  468. if denied and stuck.tracking_supported():
  469. stuck.record(root, entry, denied, audit)
  470. else:
  471. audit.info("scratch prune: could not fully remove %r", os.fspath(entry))
  472. continue
  473. except FileNotFoundError:
  474. pass
  475. except OSError:
  476. audit.info("scratch prune: could not confirm removal of %r", os.fspath(entry))
  477. continue
  478. audit.info("scratch prune: removed %r (%d bytes)", os.fspath(entry), size)
  479. removed += 1
  480. release_git_worktrees(repos)
  481. return removed