| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486 |
- """Machine-wide byte store: download, verify, extract, publish atomically."""
- from __future__ import annotations
- import os
- import platform
- import shutil
- import stat
- import sys
- import threading
- import tempfile
- import time
- from contextlib import contextmanager
- from pathlib import Path
- from typing import IO
- from pm.filesystem import is_junction, long_root, retry_held
- # Inert desktop metadata that must never decide a package's layout or digest.
- JUNK_NAMES = frozenset({".DS_Store", "Thumbs.db", "desktop.ini", ".localized"})
- ALL_TARGETS = (
- "win32-x64",
- "win32-arm64",
- "linux-x64",
- "linux-arm64",
- "linux-x64-musl",
- "linux-arm64-musl",
- "linux-arm64-bionic",
- "darwin-x64",
- "darwin-arm64",
- )
- MUSL_TARGETS = frozenset({"linux-x64-musl", "linux-arm64-musl"})
- def _native_machine() -> str:
- """Native host arch; PM also runs alone before the app is importable."""
- try:
- from hermes_platform.host.facts import native_arch
- except ModuleNotFoundError as exc:
- if exc.name != "hermes_platform":
- raise
- else:
- return native_arch()
- # Bootstrap's independent PM runtime contains only pm/, not the app.
- # Consult the OS for translated processes rather than trusting Python's
- # process architecture (Rosetta and WOW64 both report the wrong target).
- if sys.platform == "darwin" and platform.machine().lower() in ("x86_64", "amd64"):
- try:
- import ctypes
- value = ctypes.c_int()
- size = ctypes.c_size_t(ctypes.sizeof(value))
- if ctypes.CDLL(None).sysctlbyname(b"sysctl.proc_translated", ctypes.byref(value),
- ctypes.byref(size), None, 0) == 0 and value.value == 1:
- return "arm64"
- except (AttributeError, OSError):
- pass
- if sys.platform == "win32":
- try:
- import ctypes
- from ctypes import wintypes
- kernel = ctypes.WinDLL("kernel32", use_last_error=True)
- kernel.GetCurrentProcess.restype = wintypes.HANDLE
- kernel.IsWow64Process2.argtypes = [wintypes.HANDLE, ctypes.POINTER(ctypes.c_ushort),
- ctypes.POINTER(ctypes.c_ushort)]
- kernel.IsWow64Process2.restype = wintypes.BOOL
- process, native = ctypes.c_ushort(), ctypes.c_ushort()
- if kernel.IsWow64Process2(kernel.GetCurrentProcess(), ctypes.byref(process), ctypes.byref(native)):
- if native.value == 0xAA64:
- return "arm64"
- if native.value == 0x8664:
- return "x86_64"
- except (AttributeError, OSError):
- pass
- wow = os.environ.get("PROCESSOR_ARCHITEW6432", "").lower()
- if wow in ("arm64", "amd64"):
- return wow
- return platform.machine().lower()
- def _is_bionic_libc() -> bool:
- """True on Android/bionic userlands (Termux and friends).
- The libc flavor is part of the target triple, not a platform runtime
- branch: a linux-arm64 glibc artifact cannot exec under bionic and vice
- versa, so the resolver must pick the right row of the lock table. This
- is the ONE place hermes probes for bionic; nothing downstream of
- current_target() needs to know how it was decided.
- """
- if sys.platform == "android":
- return True
- # Termux's pre-3.13 interpreter reports Linux but records its Android
- # build target in sysconfig. This also works in a container without
- # the phone's /system mount or a recognizable string in libc's header.
- import sysconfig
- return bool(sysconfig.get_config_var("ANDROID_API_LEVEL"))
- def _elf_loader_is_musl(binary: Path) -> bool | None:
- """Read an ELF's interpreter string without executing foreign bytes."""
- try:
- with binary.resolve().open("rb") as handle:
- head = handle.read(8192)
- except OSError:
- return None
- if not head.startswith(b"\x7fELF"):
- return None
- if b"ld-musl-" in head:
- return True
- if b"ld-linux" in head:
- return False
- return None
- def _native_linux_uses_musl() -> bool | None:
- """libc of the native userland, independent of the Python bootstrap."""
- for candidate in (Path("/bin/sh"), Path("/bin/ls")):
- verdict = _elf_loader_is_musl(candidate)
- if verdict is not None:
- return verdict
- return None
- def _is_musl_libc() -> bool:
- """True on native Linux musl userlands (Alpine, Void-musl, etc.).
- libc is part of PM's artifact identity. The native userland takes
- precedence over a bootstrap Python built for a different libc.
- Python build metadata breaks ties when native binaries cannot be inspected;
- a musl loader on disk alone is the last resort because glibc hosts may
- install musl as a secondary toolchain.
- """
- native = _native_linux_uses_musl()
- if native is not None:
- return native
- import sysconfig
- for key in ("HOST_GNU_TYPE", "MULTIARCH"):
- value = str(sysconfig.get_config_var(key) or "").lower()
- if "musl" in value:
- return True
- return any(Path("/lib").glob("ld-musl-*.so.1"))
- def current_target() -> str:
- machine = _native_machine()
- if machine in ("arm64", "aarch64"):
- arch = "arm64"
- elif machine in ("x86_64", "amd64", "x64"):
- arch = "x64"
- else:
- raise RuntimeError(f"unsupported architecture: {platform.machine()}")
- if sys.platform.startswith("win"):
- return f"win32-{arch}"
- if sys.platform == "darwin":
- return f"darwin-{arch}"
- if _is_bionic_libc():
- return f"linux-{arch}-bionic"
- if _is_musl_libc():
- return f"linux-{arch}-musl"
- return f"linux-{arch}"
- def hash_url(url: str) -> str:
- """sha256 of a url's content, streamed. `pm lock` uses this to pin."""
- import hashlib
- import http.client
- import urllib.request
- from pm.downloader import _OPENER, _UA
- from pm.network import retry_network
- def request():
- digest = hashlib.sha256()
- size = 0
- with _OPENER.open(
- urllib.request.Request(url, headers=_UA), timeout=600
- ) as resp:
- declared = int(resp.headers.get("Content-Length") or 0)
- for block in iter(lambda: resp.read(1024 * 1024), b""):
- digest.update(block)
- size += len(block)
- if size < declared:
- raise http.client.IncompleteRead(b"", declared - size)
- return digest.hexdigest()
- return retry_network(request)
- def _tar_filter(member, dest: str):
- """The stdlib 'data' filter, with symlink targets resolved from the link's own
- directory. Bootstrap interpreters (Ubuntu 22.04 ships 3.10) resolve them from
- the archive root and reject python-build-standalone's terminfo links."""
- import tarfile
- if member.issym():
- if os.path.isabs(member.linkname):
- raise tarfile.AbsoluteLinkError(member)
- name = member.name.rstrip("/")
- placed = os.path.realpath(os.path.join(dest, name))
- link_dir = os.path.dirname(name)
- target = os.path.realpath(os.path.join(dest, link_dir, member.linkname))
- for path in (placed, target):
- if os.path.commonpath([path, dest]) != dest:
- raise tarfile.LinkOutsideDestinationError(member, path)
- return member.replace(deep=False, uid=None, gid=None, uname=None, gname=None, mode=None)
- return tarfile.data_filter(member, dest)
- def extract_tar(archive: Path | IO[bytes], dest: Path) -> None:
- """Extract a tarball (a path, or an open stream such as a .deb's data.tar)
- with the one containment policy every PM tar consumer shares. Unsafe
- members raise tarfile.FilterError.
- """
- import tarfile
- dest.mkdir(parents=True, exist_ok=True)
- real_dest = os.path.realpath(dest)
- opened = tarfile.open(archive) if isinstance(archive, (str, os.PathLike)) else tarfile.open(fileobj=archive)
- with opened as tf:
- tf.extractall(dest, filter=lambda member, path: _tar_filter(member, real_dest))
- def extract(archive: Path, dest: Path) -> None:
- shutil.rmtree(dest, ignore_errors=True)
- dest.mkdir(parents=True, exist_ok=True)
- name = archive.name.lower()
- if name.endswith((".tar.gz", ".tgz", ".tar.xz", ".txz", ".tar.bz2")):
- extract_tar(archive, dest)
- elif name.endswith(".zip"):
- _extract_zip(archive, dest)
- else:
- raise ValueError(f"unsupported archive: {archive.name}")
- def _extract_zip(archive: Path, dest: Path) -> None:
- import zipfile
- with zipfile.ZipFile(archive) as zf:
- symlinks: list[tuple] = []
- for info in zf.infolist():
- mode = info.external_attr >> 16
- if stat.S_ISLNK(mode):
- symlinks.append((info, zf.read(info).decode("utf-8")))
- continue
- written = Path(zf.extract(info, dest))
- if mode & 0o111 and written.is_file():
- written.chmod(mode & 0o777)
- for info, target in symlinks:
- _zip_symlink(info.filename, target, dest)
- def _zip_symlink(member: str, target: str, dest: Path) -> None:
- root = dest.resolve()
- link = (root / member).resolve()
- if not link.is_relative_to(root):
- return
- if Path(target).is_absolute():
- return
- resolved = (link.parent / target).resolve()
- if not resolved.is_relative_to(root):
- return
- link.parent.mkdir(parents=True, exist_ok=True)
- try:
- link.symlink_to(target)
- except OSError:
- link.write_text(target, encoding="utf-8")
- def _is_os_metadata(path: Path) -> bool:
- """Only inert regular metadata files are excluded, never links or directories.
- A directory or executable that happens to carry one of these names is
- payload, not metadata: skipping it would hide real package bytes and let a
- tree digest change under a caller that only added a .DS_Store.
- """
- if path.name not in JUNK_NAMES and not path.name.startswith("._"):
- return False
- mode = path.lstat().st_mode
- return stat.S_ISREG(mode) and not mode & 0o111
- def flatten_single_dir(dest: Path) -> None:
- """Hoist a lone top-level dir's contents unless it IS the layout
- (bin/, cmd/, lib/...). Refuses on name collisions. The tree was extracted
- seconds ago, when a Windows scanner still holds it (#131884)."""
- keep = {"bin", "cmd", "lib", "libexec", "share", "etc", "usr"}
- entries = [entry for entry in dest.iterdir() if not _is_os_metadata(entry)]
- if len(entries) != 1 or not entries[0].is_dir() or entries[0].name in keep:
- return
- inner = entries[0]
- items = list(inner.iterdir())
- # Check the whole move before modifying the layout. Duplicate desktop
- # metadata is harmless; every other collision must preserve both trees.
- for item in items:
- target = dest / item.name
- if os.path.lexists(target) and not (_is_os_metadata(item) and _is_os_metadata(target)):
- return
- for item in items:
- target = dest / item.name
- if os.path.lexists(target):
- retry_held(target.unlink) # both entries were verified as regular metadata
- retry_held(lambda: item.rename(target))
- retry_held(inner.rmdir)
- def merge_tree(src: Path, dst: Path) -> None:
- """Move src's tree into dst, keeping both layouts. A file present in
- both is unresolvable — two archives disagreeing about one file cannot
- be settled by extraction order, so it fails loudly instead."""
- for item in sorted(src.rglob("*")):
- if item.is_dir():
- continue
- rel = item.relative_to(src)
- target = dst / rel
- if target.exists():
- raise FileExistsError(f"archives disagree about {rel}")
- target.parent.mkdir(parents=True, exist_ok=True)
- retry_held(lambda: item.replace(target))
- def tree_digest(root: Path) -> str:
- return _tree_digest(root, skip_metadata=True)
- def digest_matches(root: Path, recorded: str | None) -> bool:
- """Facts recorded before desktop metadata was skipped hashed it as payload."""
- return recorded is not None and (
- tree_digest(root) == recorded or _tree_digest(root, skip_metadata=False) == recorded)
- def _tree_digest(root: Path, *, skip_metadata: bool) -> str:
- """Deterministic sha256 over a directory tree: walk every file, sort
- by posix relpath, hash `relpath\\0<content>` per entry. No mtimes, no
- mode bits. Symlinks contribute their LINK TARGET TEXT (os.readlink),
- not the target's bytes — the link is the data. Directory symlinks and
- junctions are not followed.
- ``__pycache__`` directories are skipped: CPython writes .pyc caches
- into them the first time the staged interpreter runs (uv venv/uv sync
- in a bundle build; first boot of a shipped app), so they are runtime
- state, not package bytes — the digest is over what pm published."""
- import hashlib
- files: list[tuple[str, Path]] = []
- for dirpath, dirnames, filenames in os.walk(root):
- descend = []
- for name in sorted(dirnames):
- path = Path(dirpath) / name
- if path.is_symlink() or is_junction(path):
- files.append((path.relative_to(root).as_posix(), path))
- elif name != "__pycache__":
- descend.append(name)
- dirnames[:] = descend
- for fname in filenames:
- path = Path(dirpath) / fname
- if skip_metadata and _is_os_metadata(path):
- continue
- files.append((path.relative_to(root).as_posix(), path))
- files.sort(key=lambda item: item[0])
- digest = hashlib.sha256()
- for rel, path in files:
- digest.update(rel.encode("utf-8"))
- digest.update(b"\0")
- if path.is_symlink() or is_junction(path):
- digest.update(os.readlink(path).encode("utf-8"))
- else:
- with open(path, "rb") as f:
- for block in iter(lambda: f.read(1024 * 1024), b""):
- digest.update(block)
- return digest.hexdigest()
- class Store:
- """One directory of immutable published entries plus a scratch area.
- Hash-keyed archives survive failed installs. Publication releases them."""
- def __init__(self, root: Path):
- self.root = long_root(root)
- def entry(self, name: str) -> Path:
- return self.root / name
- def published(self, name: str) -> bool:
- return self.entry(name).is_dir()
- def fetch(self, url: str, sha256: str, scratch: Path, progress=None, *, pause_event: threading.Event | None = None) -> Path:
- tick = (lambda done, total, ranges: progress(done, total)) if progress is not None else None
- return self.fetch_many([{"url": url, "sha256": sha256}], scratch,
- progress=tick, pause_event=pause_event)[0]
- def fetch_many(self, artifacts: list[dict], scratch: Path, *, progress=None,
- pause_event: threading.Event | None = None) -> list[Path]:
- """Fetch a pinned plan with one aggregate progress stream.
- Completed archives enter the cache even if a later source pauses.
- The downloader owns partial bytes outside this disposable scratch.
- """
- from pm.downloader import Download, DownloadPaused
- from pm.artifact_mirror import pinned_source
- sources = []
- for artifact in artifacts:
- if pause_event is not None and pause_event.is_set():
- raise DownloadPaused("download paused")
- url, digest = artifact["url"], artifact["sha256"]
- entry_name = f"fetch-{digest}"
- entry = self.entry(entry_name)
- files = list(entry.iterdir()) if entry.is_dir() else []
- if len(files) == 1 and files[0].is_file():
- destination = files[0]
- else:
- if entry.exists():
- shutil.rmtree(entry)
- destination = scratch / entry_name / url.rsplit("/", 1)[-1]
- sources.append(pinned_source(url, destination, digest))
- # Progress keeps the lockfile URL even when the transport is a mirror (#123132).
- urls = {str(source.dest): artifact["url"] for source, artifact in zip(sources, artifacts)}
- def tick(done, total, ranges):
- if progress is not None:
- progress(done, total, {urls[key]: rows for key, rows in ranges.items()})
- try:
- Download(sources, pause_event=pause_event).run(progress=tick)
- finally:
- # Only finalized, hash-verified files can exist at these paths.
- for source in sources:
- if source.dest.is_relative_to(scratch) and source.dest.is_file():
- self.publish(source.dest.parent, f"fetch-{source.sha256}")
- return [self.entry(f"fetch-{source.sha256}") / source.dest.name for source in sources]
- @contextmanager
- def scratch(self):
- self.root.mkdir(parents=True, exist_ok=True)
- path = Path(tempfile.mkdtemp(prefix=".staging-", dir=self.root))
- try:
- yield path
- finally:
- shutil.rmtree(path, ignore_errors=True)
- def publish(self, staged: Path, name: str) -> Path:
- """Atomic rename into place, retried for Windows file-lock holds
- (Defender, indexers). A concurrent winner's entry is kept."""
- target = self.entry(name)
- delay = 0.5
- for _ in range(5):
- try:
- os.replace(staged, target)
- return target
- except OSError:
- if target.is_dir():
- return target
- time.sleep(delay)
- delay *= 2
- try:
- os.replace(staged, target)
- except OSError:
- if not target.is_dir():
- raise
- return target
- @contextmanager
- def install_lock(self):
- """Serialize writers using the same advisory lock as runtime publication."""
- from pm.filesystem import lock_fd
- self.root.mkdir(parents=True, exist_ok=True)
- lock = self.root / ".install.lock"
- fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600)
- try:
- # A second `hermes pm install` behind an sdist build otherwise sits
- # silent for minutes; say what it is waiting on.
- if not lock_fd(fd, wait=True, timeout=2):
- print(f"waiting for {lock} (another PM operation holds it)", file=sys.stderr, flush=True)
- lock_fd(fd, wait=True)
- yield
- finally:
- os.close(fd)
|