hermes_constants_scratch_stuck.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192
  1. """Scratch entries the prune cannot delete as this user: remembered once, skipped until fixed.
  2. A container that ran as root and wrote into a bind mount under scratch (the Docker terminal
  3. backend without ``docker_run_as_host_user``, a test or review lane's ``docker run -v``) leaves
  4. files this user cannot unlink, or directories it cannot even read. ``rmtree(ignore_errors=True)``
  5. removes what it can, the entry stays idle, and every hourly pass re-walked the whole subtree and
  6. logged the same "could not fully remove" line again (218 times per entry on one host). Here such
  7. an entry is recorded with the directories that refused access, the user gets ONE line naming the
  8. command that clears it, and later passes skip it before walking it. A record is dropped (and
  9. the entry judged normally again) when the entry is gone or its top directory changed, when
  10. every recorded blocking directory is accessible now (``chown``/``chmod`` of a nested dir), when
  11. it was made by another uid, or after :data:`RECHECK_AFTER_SECONDS` in any case. Nothing is
  12. escalated or shelled out to: the hint is for the user to run.
  13. POSIX only: without ``os.getuid`` (Windows) a PermissionError is usually a transient open
  14. handle, so the prune keeps retrying there exactly as before.
  15. """
  16. from __future__ import annotations
  17. import contextlib
  18. import errno
  19. import json
  20. import os
  21. import shlex
  22. import shutil
  23. import sys
  24. import tempfile
  25. import time
  26. from pathlib import Path
  27. STUCK_STATE_NAME = ".prune_stuck.json"
  28. STUCK_LOCK_NAME = STUCK_STATE_NAME + ".lock"
  29. # Bounded retry: even a record whose blockers still look inaccessible is re-judged weekly, so a
  30. # fix this module cannot see (ACLs, a sticky bit cleared, a changed group) is never skipped forever.
  31. RECHECK_AFTER_SECONDS = 7 * 24 * 3600
  32. # A record keeps this many blocking dirs; checking a subset can only re-judge early, never skip longer.
  33. _MAX_BLOCKERS = 16
  34. _PERMISSION_ERRNOS = frozenset({errno.EACCES, errno.EPERM})
  35. def tracking_supported() -> bool:
  36. return os.name == "posix" and hasattr(os, "geteuid")
  37. def is_permission_error(exc: BaseException) -> bool:
  38. return isinstance(exc, OSError) and exc.errno in _PERMISSION_ERRNOS
  39. def _fingerprint(entry: Path) -> list[int] | None:
  40. try:
  41. st = os.lstat(entry)
  42. except OSError:
  43. return None
  44. # ctime too: ``sudo chown -R``/``chmod -R`` fixes the entry without touching its mtime.
  45. return [st.st_ino, st.st_mtime_ns, st.st_ctime_ns]
  46. def _accessible(path: str) -> bool:
  47. """True when this user can now list and delete inside *path* (or it is gone)."""
  48. if not os.path.lexists(path):
  49. return True
  50. kw = {"effective_ids": True} if os.access in os.supports_effective_ids else {}
  51. return os.access(path, os.R_OK | os.W_OK | os.X_OK, **kw)
  52. def _still_stuck(root: Path, name: str, rec: dict, now: float) -> bool:
  53. blockers = rec.get("blockers")
  54. return (
  55. rec.get("uid") == os.geteuid() # windows-footgun: ok — only reached when tracking_supported()
  56. and now - float(rec.get("since") or 0) < RECHECK_AFTER_SECONDS
  57. and rec.get("fp") == _fingerprint(root / name)
  58. and isinstance(blockers, list) and bool(blockers)
  59. and not all(_accessible(b) for b in blockers if isinstance(b, str))
  60. )
  61. def _read(root: Path) -> dict[str, dict]:
  62. # A torn or hand-edited file reads as empty: the worst case is one re-walk and one line.
  63. try:
  64. data = json.loads((root / STUCK_STATE_NAME).read_text(encoding="utf-8-sig"))
  65. except (OSError, ValueError):
  66. return {}
  67. return {k: v for k, v in data.items() if isinstance(v, dict)} if isinstance(data, dict) else {}
  68. def _write(root: Path, state: dict[str, dict]) -> None:
  69. """Publish *state* via a unique temp file + ``os.replace``: a reader sees old or new, never torn."""
  70. path = root / STUCK_STATE_NAME
  71. if not state:
  72. with contextlib.suppress(OSError):
  73. path.unlink(missing_ok=True)
  74. return
  75. try:
  76. fd, tmp = tempfile.mkstemp(dir=root, prefix=STUCK_STATE_NAME + ".", suffix=".tmp")
  77. except OSError:
  78. return
  79. try:
  80. with os.fdopen(fd, "w", encoding="utf-8") as f:
  81. json.dump(state, f, indent=1, sort_keys=True)
  82. os.replace(tmp, path)
  83. except OSError:
  84. with contextlib.suppress(OSError):
  85. os.unlink(tmp)
  86. @contextlib.contextmanager
  87. def _locked(root: Path):
  88. """Exclusive cross-process lock around a read-merge-write of the state file. Two pruners of
  89. one home (separate processes past the ``.last_prune`` throttle together) otherwise overwrite
  90. each other's records from stale snapshots. POSIX only, like the tracking itself."""
  91. import fcntl
  92. fd = os.open(root / STUCK_LOCK_NAME, os.O_RDWR | os.O_CREAT, 0o600)
  93. try:
  94. fcntl.flock(fd, fcntl.LOCK_EX)
  95. yield
  96. finally:
  97. os.close(fd) # closing the descriptor releases the flock
  98. def load_skips(root: Path, *, persist: bool = True) -> frozenset[str]:
  99. """Names to skip this pass. Records that no longer hold (see module doc) are dropped, so
  100. that entry is judged normally again."""
  101. if not tracking_supported() or not (root / STUCK_STATE_NAME).exists():
  102. return frozenset()
  103. now = time.time()
  104. if not persist:
  105. state = _read(root)
  106. return frozenset(n for n, rec in state.items() if _still_stuck(root, n, rec, now))
  107. try:
  108. with _locked(root):
  109. state = _read(root)
  110. live = {n: rec for n, rec in state.items() if _still_stuck(root, n, rec, now)}
  111. if live != state:
  112. _write(root, live)
  113. except OSError:
  114. return frozenset()
  115. return frozenset(live)
  116. def rmtree_collecting_permission_errors(entry: Path) -> list[str]:
  117. """``rmtree`` that keeps going past errors (like ``ignore_errors=True``) and returns the
  118. directories that refused access with EACCES/EPERM (empty: nothing was denied)."""
  119. blockers: list[str] = []
  120. def _onexc(func, path, exc):
  121. if is_permission_error(exc):
  122. # scandir/open of a dir fails on the dir itself; unlink/rmdir on its parent's perms.
  123. blocker = path if func in (os.scandir, os.open, os.lstat) else os.path.dirname(path)
  124. if blocker not in blockers:
  125. blockers.append(blocker)
  126. if sys.version_info >= (3, 12):
  127. shutil.rmtree(entry, onexc=_onexc)
  128. else: # pragma: no cover - 3.11
  129. shutil.rmtree(entry, onerror=lambda f, p, info: _onexc(f, p, info[1]))
  130. return blockers
  131. def removal_hint(entry: Path) -> str:
  132. from hermes_platform.resolver import locate_command
  133. hint = f"sudo rm -rf {shlex.quote(os.fspath(entry))}"
  134. if locate_command("docker").command:
  135. hint += (
  136. f" (or without sudo: docker run --rm -v {shlex.quote(os.fspath(entry.parent) + ':/s')}"
  137. f" alpine rm -rf {shlex.quote('/s/' + entry.name)})"
  138. )
  139. return hint
  140. def record(root: Path, entry: Path, blockers: list[str], audit) -> None:
  141. """Remember *entry* as not removable by this user and log the one actionable line."""
  142. fp = _fingerprint(entry)
  143. if fp is None or not blockers:
  144. return
  145. rec = {"fp": fp, "since": time.time(), "uid": os.geteuid(), "blockers": blockers[:_MAX_BLOCKERS]} # windows-footgun: ok — only reached when tracking_supported()
  146. with contextlib.suppress(OSError), _locked(root):
  147. state = _read(root) # re-read under the lock: merge, never publish a stale snapshot
  148. state[entry.name] = rec
  149. _write(root, state)
  150. audit.info(
  151. "scratch prune: cannot remove %r: it holds files this user (uid %d) may not read or "
  152. "delete, usually root-owned output of a Docker container; skipping it until that is "
  153. "fixed (re-checked weekly). Remove it with: %s",
  154. os.fspath(entry), rec["uid"], removal_hint(entry),
  155. )
  156. def stuck_entries(root: Path) -> list[Path]:
  157. """Entries the prune is currently skipping as not removable (for ``hermes doctor``)."""
  158. return [root / name for name in sorted(load_skips(root, persist=False))]