download_state.py 1.8 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253
  1. """Cross-process ownership of resumable partials and their GC boundary."""
  2. from __future__ import annotations
  3. from contextlib import contextmanager
  4. import os
  5. from pathlib import Path
  6. import time
  7. from pm.filesystem import lock_fd
  8. @contextmanager
  9. def partial_lock(root: Path, key: str, *, cancelled=None, wait: bool = True):
  10. # Lock inodes must survive release: removing one lets a new owner lock a
  11. # different inode while an older waiter still holds the original handle.
  12. locks = root / ".locks"
  13. locks.mkdir(parents=True, exist_ok=True)
  14. fd = os.open(locks / key, os.O_CREAT | os.O_RDWR, 0o600)
  15. try:
  16. while not lock_fd(fd, wait=False):
  17. if not wait:
  18. yield False
  19. return
  20. if cancelled is not None and cancelled():
  21. yield False
  22. return
  23. time.sleep(0.05)
  24. yield True
  25. finally:
  26. os.close(fd)
  27. GC_GRACE_SECONDS = 6 * 60 * 60
  28. def collect_partials(root: Path, *, grace_seconds: float = GC_GRACE_SECONDS) -> None:
  29. """Do not let a GC snapshot race a downloader acquiring ownership."""
  30. if not root.is_dir():
  31. return
  32. keys = {path.stem for path in root.iterdir() if path.suffix in {".part", ".ranges"}}
  33. now = time.time()
  34. for key in sorted(keys):
  35. with partial_lock(root, key, wait=False) as acquired:
  36. if not acquired:
  37. continue
  38. pair = [root / f"{key}{suffix}" for suffix in (".part", ".ranges")]
  39. try:
  40. if any(path.exists() and grace_seconds > 0 and now - path.stat().st_mtime < grace_seconds for path in pair):
  41. continue
  42. for path in pair:
  43. path.unlink(missing_ok=True)
  44. except OSError:
  45. continue # An inaccessible partial is not evidence it is unused.