store.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486
  1. """Machine-wide byte store: download, verify, extract, publish atomically."""
  2. from __future__ import annotations
  3. import os
  4. import platform
  5. import shutil
  6. import stat
  7. import sys
  8. import threading
  9. import tempfile
  10. import time
  11. from contextlib import contextmanager
  12. from pathlib import Path
  13. from typing import IO
  14. from pm.filesystem import is_junction, long_root, retry_held
  15. # Inert desktop metadata that must never decide a package's layout or digest.
  16. JUNK_NAMES = frozenset({".DS_Store", "Thumbs.db", "desktop.ini", ".localized"})
  17. ALL_TARGETS = (
  18. "win32-x64",
  19. "win32-arm64",
  20. "linux-x64",
  21. "linux-arm64",
  22. "linux-x64-musl",
  23. "linux-arm64-musl",
  24. "linux-arm64-bionic",
  25. "darwin-x64",
  26. "darwin-arm64",
  27. )
  28. MUSL_TARGETS = frozenset({"linux-x64-musl", "linux-arm64-musl"})
  29. def _native_machine() -> str:
  30. """Native host arch; PM also runs alone before the app is importable."""
  31. try:
  32. from hermes_platform.host.facts import native_arch
  33. except ModuleNotFoundError as exc:
  34. if exc.name != "hermes_platform":
  35. raise
  36. else:
  37. return native_arch()
  38. # Bootstrap's independent PM runtime contains only pm/, not the app.
  39. # Consult the OS for translated processes rather than trusting Python's
  40. # process architecture (Rosetta and WOW64 both report the wrong target).
  41. if sys.platform == "darwin" and platform.machine().lower() in ("x86_64", "amd64"):
  42. try:
  43. import ctypes
  44. value = ctypes.c_int()
  45. size = ctypes.c_size_t(ctypes.sizeof(value))
  46. if ctypes.CDLL(None).sysctlbyname(b"sysctl.proc_translated", ctypes.byref(value),
  47. ctypes.byref(size), None, 0) == 0 and value.value == 1:
  48. return "arm64"
  49. except (AttributeError, OSError):
  50. pass
  51. if sys.platform == "win32":
  52. try:
  53. import ctypes
  54. from ctypes import wintypes
  55. kernel = ctypes.WinDLL("kernel32", use_last_error=True)
  56. kernel.GetCurrentProcess.restype = wintypes.HANDLE
  57. kernel.IsWow64Process2.argtypes = [wintypes.HANDLE, ctypes.POINTER(ctypes.c_ushort),
  58. ctypes.POINTER(ctypes.c_ushort)]
  59. kernel.IsWow64Process2.restype = wintypes.BOOL
  60. process, native = ctypes.c_ushort(), ctypes.c_ushort()
  61. if kernel.IsWow64Process2(kernel.GetCurrentProcess(), ctypes.byref(process), ctypes.byref(native)):
  62. if native.value == 0xAA64:
  63. return "arm64"
  64. if native.value == 0x8664:
  65. return "x86_64"
  66. except (AttributeError, OSError):
  67. pass
  68. wow = os.environ.get("PROCESSOR_ARCHITEW6432", "").lower()
  69. if wow in ("arm64", "amd64"):
  70. return wow
  71. return platform.machine().lower()
  72. def _is_bionic_libc() -> bool:
  73. """True on Android/bionic userlands (Termux and friends).
  74. The libc flavor is part of the target triple, not a platform runtime
  75. branch: a linux-arm64 glibc artifact cannot exec under bionic and vice
  76. versa, so the resolver must pick the right row of the lock table. This
  77. is the ONE place hermes probes for bionic; nothing downstream of
  78. current_target() needs to know how it was decided.
  79. """
  80. if sys.platform == "android":
  81. return True
  82. # Termux's pre-3.13 interpreter reports Linux but records its Android
  83. # build target in sysconfig. This also works in a container without
  84. # the phone's /system mount or a recognizable string in libc's header.
  85. import sysconfig
  86. return bool(sysconfig.get_config_var("ANDROID_API_LEVEL"))
  87. def _elf_loader_is_musl(binary: Path) -> bool | None:
  88. """Read an ELF's interpreter string without executing foreign bytes."""
  89. try:
  90. with binary.resolve().open("rb") as handle:
  91. head = handle.read(8192)
  92. except OSError:
  93. return None
  94. if not head.startswith(b"\x7fELF"):
  95. return None
  96. if b"ld-musl-" in head:
  97. return True
  98. if b"ld-linux" in head:
  99. return False
  100. return None
  101. def _native_linux_uses_musl() -> bool | None:
  102. """libc of the native userland, independent of the Python bootstrap."""
  103. for candidate in (Path("/bin/sh"), Path("/bin/ls")):
  104. verdict = _elf_loader_is_musl(candidate)
  105. if verdict is not None:
  106. return verdict
  107. return None
  108. def _is_musl_libc() -> bool:
  109. """True on native Linux musl userlands (Alpine, Void-musl, etc.).
  110. libc is part of PM's artifact identity. The native userland takes
  111. precedence over a bootstrap Python built for a different libc.
  112. Python build metadata breaks ties when native binaries cannot be inspected;
  113. a musl loader on disk alone is the last resort because glibc hosts may
  114. install musl as a secondary toolchain.
  115. """
  116. native = _native_linux_uses_musl()
  117. if native is not None:
  118. return native
  119. import sysconfig
  120. for key in ("HOST_GNU_TYPE", "MULTIARCH"):
  121. value = str(sysconfig.get_config_var(key) or "").lower()
  122. if "musl" in value:
  123. return True
  124. return any(Path("/lib").glob("ld-musl-*.so.1"))
  125. def current_target() -> str:
  126. machine = _native_machine()
  127. if machine in ("arm64", "aarch64"):
  128. arch = "arm64"
  129. elif machine in ("x86_64", "amd64", "x64"):
  130. arch = "x64"
  131. else:
  132. raise RuntimeError(f"unsupported architecture: {platform.machine()}")
  133. if sys.platform.startswith("win"):
  134. return f"win32-{arch}"
  135. if sys.platform == "darwin":
  136. return f"darwin-{arch}"
  137. if _is_bionic_libc():
  138. return f"linux-{arch}-bionic"
  139. if _is_musl_libc():
  140. return f"linux-{arch}-musl"
  141. return f"linux-{arch}"
  142. def hash_url(url: str) -> str:
  143. """sha256 of a url's content, streamed. `pm lock` uses this to pin."""
  144. import hashlib
  145. import http.client
  146. import urllib.request
  147. from pm.downloader import _OPENER, _UA
  148. from pm.network import retry_network
  149. def request():
  150. digest = hashlib.sha256()
  151. size = 0
  152. with _OPENER.open(
  153. urllib.request.Request(url, headers=_UA), timeout=600
  154. ) as resp:
  155. declared = int(resp.headers.get("Content-Length") or 0)
  156. for block in iter(lambda: resp.read(1024 * 1024), b""):
  157. digest.update(block)
  158. size += len(block)
  159. if size < declared:
  160. raise http.client.IncompleteRead(b"", declared - size)
  161. return digest.hexdigest()
  162. return retry_network(request)
  163. def _tar_filter(member, dest: str):
  164. """The stdlib 'data' filter, with symlink targets resolved from the link's own
  165. directory. Bootstrap interpreters (Ubuntu 22.04 ships 3.10) resolve them from
  166. the archive root and reject python-build-standalone's terminfo links."""
  167. import tarfile
  168. if member.issym():
  169. if os.path.isabs(member.linkname):
  170. raise tarfile.AbsoluteLinkError(member)
  171. name = member.name.rstrip("/")
  172. placed = os.path.realpath(os.path.join(dest, name))
  173. link_dir = os.path.dirname(name)
  174. target = os.path.realpath(os.path.join(dest, link_dir, member.linkname))
  175. for path in (placed, target):
  176. if os.path.commonpath([path, dest]) != dest:
  177. raise tarfile.LinkOutsideDestinationError(member, path)
  178. return member.replace(deep=False, uid=None, gid=None, uname=None, gname=None, mode=None)
  179. return tarfile.data_filter(member, dest)
  180. def extract_tar(archive: Path | IO[bytes], dest: Path) -> None:
  181. """Extract a tarball (a path, or an open stream such as a .deb's data.tar)
  182. with the one containment policy every PM tar consumer shares. Unsafe
  183. members raise tarfile.FilterError.
  184. """
  185. import tarfile
  186. dest.mkdir(parents=True, exist_ok=True)
  187. real_dest = os.path.realpath(dest)
  188. opened = tarfile.open(archive) if isinstance(archive, (str, os.PathLike)) else tarfile.open(fileobj=archive)
  189. with opened as tf:
  190. tf.extractall(dest, filter=lambda member, path: _tar_filter(member, real_dest))
  191. def extract(archive: Path, dest: Path) -> None:
  192. shutil.rmtree(dest, ignore_errors=True)
  193. dest.mkdir(parents=True, exist_ok=True)
  194. name = archive.name.lower()
  195. if name.endswith((".tar.gz", ".tgz", ".tar.xz", ".txz", ".tar.bz2")):
  196. extract_tar(archive, dest)
  197. elif name.endswith(".zip"):
  198. _extract_zip(archive, dest)
  199. else:
  200. raise ValueError(f"unsupported archive: {archive.name}")
  201. def _extract_zip(archive: Path, dest: Path) -> None:
  202. import zipfile
  203. with zipfile.ZipFile(archive) as zf:
  204. symlinks: list[tuple] = []
  205. for info in zf.infolist():
  206. mode = info.external_attr >> 16
  207. if stat.S_ISLNK(mode):
  208. symlinks.append((info, zf.read(info).decode("utf-8")))
  209. continue
  210. written = Path(zf.extract(info, dest))
  211. if mode & 0o111 and written.is_file():
  212. written.chmod(mode & 0o777)
  213. for info, target in symlinks:
  214. _zip_symlink(info.filename, target, dest)
  215. def _zip_symlink(member: str, target: str, dest: Path) -> None:
  216. root = dest.resolve()
  217. link = (root / member).resolve()
  218. if not link.is_relative_to(root):
  219. return
  220. if Path(target).is_absolute():
  221. return
  222. resolved = (link.parent / target).resolve()
  223. if not resolved.is_relative_to(root):
  224. return
  225. link.parent.mkdir(parents=True, exist_ok=True)
  226. try:
  227. link.symlink_to(target)
  228. except OSError:
  229. link.write_text(target, encoding="utf-8")
  230. def _is_os_metadata(path: Path) -> bool:
  231. """Only inert regular metadata files are excluded, never links or directories.
  232. A directory or executable that happens to carry one of these names is
  233. payload, not metadata: skipping it would hide real package bytes and let a
  234. tree digest change under a caller that only added a .DS_Store.
  235. """
  236. if path.name not in JUNK_NAMES and not path.name.startswith("._"):
  237. return False
  238. mode = path.lstat().st_mode
  239. return stat.S_ISREG(mode) and not mode & 0o111
  240. def flatten_single_dir(dest: Path) -> None:
  241. """Hoist a lone top-level dir's contents unless it IS the layout
  242. (bin/, cmd/, lib/...). Refuses on name collisions. The tree was extracted
  243. seconds ago, when a Windows scanner still holds it (#131884)."""
  244. keep = {"bin", "cmd", "lib", "libexec", "share", "etc", "usr"}
  245. entries = [entry for entry in dest.iterdir() if not _is_os_metadata(entry)]
  246. if len(entries) != 1 or not entries[0].is_dir() or entries[0].name in keep:
  247. return
  248. inner = entries[0]
  249. items = list(inner.iterdir())
  250. # Check the whole move before modifying the layout. Duplicate desktop
  251. # metadata is harmless; every other collision must preserve both trees.
  252. for item in items:
  253. target = dest / item.name
  254. if os.path.lexists(target) and not (_is_os_metadata(item) and _is_os_metadata(target)):
  255. return
  256. for item in items:
  257. target = dest / item.name
  258. if os.path.lexists(target):
  259. retry_held(target.unlink) # both entries were verified as regular metadata
  260. retry_held(lambda: item.rename(target))
  261. retry_held(inner.rmdir)
  262. def merge_tree(src: Path, dst: Path) -> None:
  263. """Move src's tree into dst, keeping both layouts. A file present in
  264. both is unresolvable — two archives disagreeing about one file cannot
  265. be settled by extraction order, so it fails loudly instead."""
  266. for item in sorted(src.rglob("*")):
  267. if item.is_dir():
  268. continue
  269. rel = item.relative_to(src)
  270. target = dst / rel
  271. if target.exists():
  272. raise FileExistsError(f"archives disagree about {rel}")
  273. target.parent.mkdir(parents=True, exist_ok=True)
  274. retry_held(lambda: item.replace(target))
  275. def tree_digest(root: Path) -> str:
  276. return _tree_digest(root, skip_metadata=True)
  277. def digest_matches(root: Path, recorded: str | None) -> bool:
  278. """Facts recorded before desktop metadata was skipped hashed it as payload."""
  279. return recorded is not None and (
  280. tree_digest(root) == recorded or _tree_digest(root, skip_metadata=False) == recorded)
  281. def _tree_digest(root: Path, *, skip_metadata: bool) -> str:
  282. """Deterministic sha256 over a directory tree: walk every file, sort
  283. by posix relpath, hash `relpath\\0<content>` per entry. No mtimes, no
  284. mode bits. Symlinks contribute their LINK TARGET TEXT (os.readlink),
  285. not the target's bytes — the link is the data. Directory symlinks and
  286. junctions are not followed.
  287. ``__pycache__`` directories are skipped: CPython writes .pyc caches
  288. into them the first time the staged interpreter runs (uv venv/uv sync
  289. in a bundle build; first boot of a shipped app), so they are runtime
  290. state, not package bytes — the digest is over what pm published."""
  291. import hashlib
  292. files: list[tuple[str, Path]] = []
  293. for dirpath, dirnames, filenames in os.walk(root):
  294. descend = []
  295. for name in sorted(dirnames):
  296. path = Path(dirpath) / name
  297. if path.is_symlink() or is_junction(path):
  298. files.append((path.relative_to(root).as_posix(), path))
  299. elif name != "__pycache__":
  300. descend.append(name)
  301. dirnames[:] = descend
  302. for fname in filenames:
  303. path = Path(dirpath) / fname
  304. if skip_metadata and _is_os_metadata(path):
  305. continue
  306. files.append((path.relative_to(root).as_posix(), path))
  307. files.sort(key=lambda item: item[0])
  308. digest = hashlib.sha256()
  309. for rel, path in files:
  310. digest.update(rel.encode("utf-8"))
  311. digest.update(b"\0")
  312. if path.is_symlink() or is_junction(path):
  313. digest.update(os.readlink(path).encode("utf-8"))
  314. else:
  315. with open(path, "rb") as f:
  316. for block in iter(lambda: f.read(1024 * 1024), b""):
  317. digest.update(block)
  318. return digest.hexdigest()
  319. class Store:
  320. """One directory of immutable published entries plus a scratch area.
  321. Hash-keyed archives survive failed installs. Publication releases them."""
  322. def __init__(self, root: Path):
  323. self.root = long_root(root)
  324. def entry(self, name: str) -> Path:
  325. return self.root / name
  326. def published(self, name: str) -> bool:
  327. return self.entry(name).is_dir()
  328. def fetch(self, url: str, sha256: str, scratch: Path, progress=None, *, pause_event: threading.Event | None = None) -> Path:
  329. tick = (lambda done, total, ranges: progress(done, total)) if progress is not None else None
  330. return self.fetch_many([{"url": url, "sha256": sha256}], scratch,
  331. progress=tick, pause_event=pause_event)[0]
  332. def fetch_many(self, artifacts: list[dict], scratch: Path, *, progress=None,
  333. pause_event: threading.Event | None = None) -> list[Path]:
  334. """Fetch a pinned plan with one aggregate progress stream.
  335. Completed archives enter the cache even if a later source pauses.
  336. The downloader owns partial bytes outside this disposable scratch.
  337. """
  338. from pm.downloader import Download, DownloadPaused
  339. from pm.artifact_mirror import pinned_source
  340. sources = []
  341. for artifact in artifacts:
  342. if pause_event is not None and pause_event.is_set():
  343. raise DownloadPaused("download paused")
  344. url, digest = artifact["url"], artifact["sha256"]
  345. entry_name = f"fetch-{digest}"
  346. entry = self.entry(entry_name)
  347. files = list(entry.iterdir()) if entry.is_dir() else []
  348. if len(files) == 1 and files[0].is_file():
  349. destination = files[0]
  350. else:
  351. if entry.exists():
  352. shutil.rmtree(entry)
  353. destination = scratch / entry_name / url.rsplit("/", 1)[-1]
  354. sources.append(pinned_source(url, destination, digest))
  355. # Progress keeps the lockfile URL even when the transport is a mirror (#123132).
  356. urls = {str(source.dest): artifact["url"] for source, artifact in zip(sources, artifacts)}
  357. def tick(done, total, ranges):
  358. if progress is not None:
  359. progress(done, total, {urls[key]: rows for key, rows in ranges.items()})
  360. try:
  361. Download(sources, pause_event=pause_event).run(progress=tick)
  362. finally:
  363. # Only finalized, hash-verified files can exist at these paths.
  364. for source in sources:
  365. if source.dest.is_relative_to(scratch) and source.dest.is_file():
  366. self.publish(source.dest.parent, f"fetch-{source.sha256}")
  367. return [self.entry(f"fetch-{source.sha256}") / source.dest.name for source in sources]
  368. @contextmanager
  369. def scratch(self):
  370. self.root.mkdir(parents=True, exist_ok=True)
  371. path = Path(tempfile.mkdtemp(prefix=".staging-", dir=self.root))
  372. try:
  373. yield path
  374. finally:
  375. shutil.rmtree(path, ignore_errors=True)
  376. def publish(self, staged: Path, name: str) -> Path:
  377. """Atomic rename into place, retried for Windows file-lock holds
  378. (Defender, indexers). A concurrent winner's entry is kept."""
  379. target = self.entry(name)
  380. delay = 0.5
  381. for _ in range(5):
  382. try:
  383. os.replace(staged, target)
  384. return target
  385. except OSError:
  386. if target.is_dir():
  387. return target
  388. time.sleep(delay)
  389. delay *= 2
  390. try:
  391. os.replace(staged, target)
  392. except OSError:
  393. if not target.is_dir():
  394. raise
  395. return target
  396. @contextmanager
  397. def install_lock(self):
  398. """Serialize writers using the same advisory lock as runtime publication."""
  399. from pm.filesystem import lock_fd
  400. self.root.mkdir(parents=True, exist_ok=True)
  401. lock = self.root / ".install.lock"
  402. fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600)
  403. try:
  404. # A second `hermes pm install` behind an sdist build otherwise sits
  405. # silent for minutes; say what it is waiting on.
  406. if not lock_fd(fd, wait=True, timeout=2):
  407. print(f"waiting for {lock} (another PM operation holds it)", file=sys.stderr, flush=True)
  408. lock_fd(fd, wait=True)
  409. yield
  410. finally:
  411. os.close(fd)