filesystem.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225
  1. """Stdlib-only filesystem primitives PM needs before it can replace the caller's interpreter.
  2. Boot-time dependency selection (``hermes_cli.runtime_state``) imports these, so nothing here
  3. may import a dependency or another PM module.
  4. """
  5. from __future__ import annotations
  6. from contextlib import suppress
  7. import errno
  8. import hashlib
  9. import ntpath
  10. import os
  11. from pathlib import Path
  12. import shutil
  13. import stat
  14. import sys
  15. import tempfile
  16. import time
  17. _LOCK_POLL_SECONDS = 0.05
  18. def is_junction(path: Path) -> bool:
  19. """Keep junctions opaque even before Python 3.12's Path.is_junction exists.
  20. A path that cannot be stat'ed is not a junction, matching ``os.path.isjunction``: callers
  21. probe slots that do not exist yet (a first plugin install) and must get False, not a crash.
  22. POSIX stat results carry no reparse tag, which lands in the same False.
  23. """
  24. if os.name != "nt":
  25. return False
  26. try:
  27. return path.lstat().st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT
  28. except (OSError, AttributeError):
  29. return False
  30. _VERBATIM = "\\\\?\\"
  31. _VERBATIM_UNC = "\\\\?\\UNC\\"
  32. def _long_form(text: str) -> str:
  33. if text.startswith(_VERBATIM):
  34. return text
  35. # Windows stops normalising a verbatim path, so it must already be absolute,
  36. # backslashed and free of `..` before the prefix goes on.
  37. text = ntpath.abspath(text)
  38. if text.startswith("\\\\"):
  39. return _VERBATIM_UNC + text[2:]
  40. return _VERBATIM + text
  41. def long_root(path: Path) -> Path:
  42. """Spell a PM-owned root so Windows file calls beneath it are not cut at
  43. MAX_PATH when LongPathsEnabled is 0 (the default). Every path joined onto the
  44. result inherits the spelling; relative paths and digests are unchanged."""
  45. return Path(_long_form(os.fspath(path))) if os.name == "nt" else path
  46. def native(path: str | os.PathLike[str]) -> str:
  47. """The ordinary spelling, for text that leaves PM's own file calls: argv, cwd,
  48. environment values, hashed identities, records and messages."""
  49. text = os.fspath(path)
  50. if text.startswith(_VERBATIM_UNC):
  51. return "\\\\" + text[len(_VERBATIM_UNC):]
  52. return text.removeprefix(_VERBATIM)
  53. def lock_fd(fd: int, *, wait: bool, timeout: float | None = None) -> bool:
  54. """Take the byte lock; ``timeout`` bounds the retry loop (None waits forever, 0 tries once)."""
  55. deadline = None if timeout is None else time.monotonic() + timeout
  56. if os.name == "nt":
  57. import msvcrt
  58. while True:
  59. try:
  60. os.lseek(fd, 0, os.SEEK_SET)
  61. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
  62. return True
  63. except OSError as exc:
  64. if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
  65. raise
  66. if not wait or (deadline is not None and time.monotonic() >= deadline):
  67. return False
  68. time.sleep(_LOCK_POLL_SECONDS)
  69. else:
  70. import fcntl
  71. while True:
  72. try:
  73. fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  74. return True
  75. except BlockingIOError:
  76. pass
  77. if not wait or (deadline is not None and time.monotonic() >= deadline):
  78. return False
  79. time.sleep(_LOCK_POLL_SECONDS)
  80. def unlock_fd(fd: int) -> None:
  81. """Release :func:`lock_fd`'s byte lock; best effort, since closing the descriptor releases it too."""
  82. with suppress(OSError):
  83. if os.name == "nt":
  84. import msvcrt
  85. os.lseek(fd, 0, os.SEEK_SET)
  86. msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
  87. else:
  88. import fcntl
  89. fcntl.flock(fd, fcntl.LOCK_UN)
  90. def lock_path(path: Path, *, timeout: float | None, create_dir: bool = False) -> int | None:
  91. """Open *path* (created if missing), wait up to ``timeout`` for its byte lock and return the
  92. held descriptor; None when the wait expired. The caller closes the descriptor to release.
  93. A holder that removes the lock file unlinks it while still holding it (the install-state
  94. collector), so a waiter that opened the old file gets a lock on an inode no one else can
  95. reach while the next opener creates a fresh file: both would hold "the" lock. Once held,
  96. the descriptor must still be the file at *path*, else retry on the current one. Windows
  97. cannot unlink an open file, so the check is POSIX-only. ``create_dir`` re-creates the
  98. parent the remover may have taken with it.
  99. """
  100. deadline = None if timeout is None else time.monotonic() + timeout
  101. while True:
  102. if create_dir:
  103. path.parent.mkdir(parents=True, exist_ok=True)
  104. try:
  105. fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
  106. except FileNotFoundError:
  107. # Only a parent removed between mkdir and open is worth another try; anything else
  108. # (a dangling symlink at *path*) fails the same way every time.
  109. if not create_dir or path.parent.is_dir():
  110. raise
  111. if deadline is not None and time.monotonic() >= deadline:
  112. return None
  113. continue
  114. try:
  115. remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
  116. if not lock_fd(fd, wait=True, timeout=remaining):
  117. os.close(fd)
  118. return None
  119. if os.name == "nt" or _same_file(fd, path):
  120. return fd
  121. except BaseException:
  122. os.close(fd)
  123. raise
  124. os.close(fd)
  125. if deadline is not None and time.monotonic() >= deadline:
  126. return None
  127. def _same_file(fd: int, path: Path) -> bool:
  128. try:
  129. return os.path.samestat(os.fstat(fd), os.stat(path))
  130. except FileNotFoundError:
  131. return False
  132. def read_bytes_or_none(path: Path) -> bytes | None:
  133. try:
  134. return path.read_bytes()
  135. except FileNotFoundError:
  136. return None
  137. def file_digest(path: Path) -> str | None:
  138. data = read_bytes_or_none(path)
  139. return hashlib.sha256(data).hexdigest() if data is not None else None
  140. def durable_write_bytes(path: Path, data: bytes) -> None:
  141. """Replace ``path`` atomically and fsync file and directory so a crash keeps old or new bytes."""
  142. path.parent.mkdir(parents=True, exist_ok=True)
  143. fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".publish-")
  144. try:
  145. with os.fdopen(fd, "wb") as stream:
  146. stream.write(data)
  147. stream.flush()
  148. os.fsync(stream.fileno())
  149. os.replace(temporary, path)
  150. if os.name != "nt":
  151. directory = os.open(path.parent, os.O_RDONLY)
  152. try:
  153. os.fsync(directory)
  154. finally:
  155. os.close(directory)
  156. finally:
  157. Path(temporary).unlink(missing_ok=True)
  158. #: A handle still open inside the tree. Windows reports antivirus and indexer scans of freshly
  159. #: written files this way (WinError 5 and 32 both surface as EACCES) and they clear in moments;
  160. #: a missing source or an occupied destination never does, so those raise at once.
  161. _HELD_ERRNOS = frozenset({errno.EACCES, errno.EPERM, errno.EBUSY})
  162. _HELD_RETRY_DELAYS = (0.1, 0.2, 0.4, 0.8, 1.6)
  163. def retry_held(operation):
  164. """Run a rename-like mutation, riding out a transient Windows hold (~3s) before raising."""
  165. for delay in (*_HELD_RETRY_DELAYS, None):
  166. try:
  167. return operation()
  168. except OSError as error:
  169. if delay is None or error.errno not in _HELD_ERRNOS:
  170. raise
  171. time.sleep(delay)
  172. def remove_tree(path: Path) -> None:
  173. """``shutil.rmtree`` that clears a Windows read-only attribute and retries that one unlink.
  174. Windows refuses to unlink read-only files (PortableGit ships ``etc/hosts`` read-only), and no
  175. retry clears that bit. Every other failure, and every POSIX failure, raises as plain rmtree
  176. would: a POSIX permission error is about the parent directory, and chmodding the path there
  177. would follow a symlink out of the tree. Lives here, not in ``hermes_cli``: PM installs run
  178. before the rest of the tree exists (the Docker toolchain stage copies only ``pm/``).
  179. """
  180. def _retry_writable(func, failed, error):
  181. error = error[1] if isinstance(error, tuple) else error
  182. if os.name != "nt" or func not in (os.unlink, os.rmdir) or not isinstance(error, PermissionError):
  183. raise error
  184. # Windows >= 3.13 chmod follows links; the attribute to clear is the link's own.
  185. os.chmod(failed, stat.S_IWRITE, **({"follow_symlinks": False} if os.chmod in os.supports_follow_symlinks else {}))
  186. func(failed)
  187. handler = {"onexc": _retry_writable} if sys.version_info >= (3, 12) else {"onerror": _retry_writable}
  188. shutil.rmtree(path, **handler)