package.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. """Package definitions: what a package IS. No versions, no hashes — those
  2. live in lock.json, written by `pm lock`."""
  3. from __future__ import annotations
  4. import os
  5. import subprocess
  6. from pathlib import Path, PurePosixPath
  7. from typing import TYPE_CHECKING, Optional
  8. from pm.filesystem import native
  9. if TYPE_CHECKING:
  10. from pm.store import Store
  11. class InstallError(RuntimeError):
  12. def __init__(self, package: str, cause: str, remedy: str = ""):
  13. self.package = package
  14. self.cause = cause
  15. self.remedy = remedy or "retry, or run `hermes pm doctor`"
  16. super().__init__(f"{package}: {cause} — {self.remedy}")
  17. def compose_env(diffs: list[dict], base: Optional[dict] = None) -> dict[str, str]:
  18. """Dependents win over their dependencies for every key: diffs arrive
  19. deps-first, later ones take precedence — npm's pinned shim must shadow
  20. the npm bundled inside node, and a package's exports beat inherited env.
  21. 'PATH' values are lists of dirs, prepended.
  22. Diff values are paths under a store root, which PM spells extended-length
  23. on Windows; a child environment gets their ordinary spelling."""
  24. env = dict(os.environ if base is None else base)
  25. path_dirs: list[str] = []
  26. for diff in reversed(diffs):
  27. for key, value in diff.items():
  28. if key == "PATH":
  29. dirs = value if isinstance(value, list) else [value]
  30. path_dirs.extend(native(d) for d in dirs if native(d) not in path_dirs)
  31. for diff in diffs:
  32. for key, value in diff.items():
  33. if key != "PATH":
  34. env[key] = native(str(value))
  35. if path_dirs:
  36. key = next((k for k in env if k.upper() == "PATH"), "PATH")
  37. existing = env.get(key, "")
  38. prefix = os.pathsep.join(path_dirs)
  39. env[key] = f"{prefix}{os.pathsep}{existing}" if existing else prefix
  40. return env
  41. class Package:
  42. """Subclass and register. Declarative for the common case; override
  43. fetch_url()/unpack()/stage()/verify()/env()/migrate() for the rest.
  44. name: unique id.
  45. deps: packages installed before this one.
  46. optional: not part of the root closure; installed on demand.
  47. default: an optional package the default install also carries (installers,
  48. bare `pm install`, `hermes update`) unless the user declined it
  49. (pm/defaults.py). It stays optional: a failed download warns instead of
  50. failing the install, and its absence never blocks PATH activation.
  51. internal: tooling PM uses inside install/build steps — never on PATH
  52. and never selected as an application root. Dependencies and explicit
  53. build requests can select it. node/npm are shipped runtime tools
  54. (TUI, plugins), not install machinery.
  55. on_path: contributes PATH dirs.
  56. url: template with {version} and {target} holes. override fetch_url()
  57. when a platform needs a completely different url.
  58. gaps: targets this package does NOT exist for, with the reason
  59. (upstream ships no artifact). Everything else is available.
  60. """
  61. name: str = ""
  62. deps: tuple[str, ...] = ()
  63. optional: bool = False
  64. default: bool = False
  65. internal: bool = False
  66. on_path: bool = True
  67. url: str = ""
  68. gaps: dict[str, str] = {}
  69. # How the lockfile `version` field labels this package for `pm update`:
  70. # "semver" — one shared version across targets (node 26.7.0, uv 0.12.3).
  71. # "minor" — the version label is major.minor; each target's exact
  72. # patch lives in ITS artifact urls (ffmpeg's posix vs win32
  73. # builds have no shared release cadence).
  74. version_style: str = "semver"
  75. # Targets where this package's binary is the x64 build run under
  76. # Windows ARM64 built-in emulation (no native arm64 artifact exists).
  77. # The arch guard accepts the x64 PE on these targets.
  78. emulated_arch_targets: frozenset[str] = frozenset()
  79. def missing_reason(self, target: str) -> Optional[str]:
  80. return self.gaps.get(target)
  81. def latest_versions(self, target: str, locked: Optional[str] = None) -> list[str]:
  82. """Newest-first candidate versions for `target` — the "how do I find
  83. latest" hook for `hermes pm update`. Empty list = this package has
  84. no auto-update source (chromium is bumped by hand with
  85. `hermes pm lock --bump`; venv is a state).
  86. Subclasses resolve their upstream's real release index here; the
  87. update driver intersects across targets (per version_style) and
  88. only ever pins a version every relevant target can serve. ``locked``
  89. is the current lockfile version when the resolver needs it (python
  90. keeps its minor line and takes patch/build identity from the asset)."""
  91. return []
  92. def fetch_url(self, version: str, target: str) -> str:
  93. if not self.url:
  94. raise InstallError(self.name, "package has no download url")
  95. return self.url.format(version=version, target=target)
  96. def fetch_urls(self, version: str, target: str) -> list[str]:
  97. """Every archive this target is built from, in extraction order.
  98. Almost every package is one archive; override this (instead of
  99. fetch_url) when upstream splits a runtime across downloads that
  100. have to land in one directory."""
  101. return [self.fetch_url(version, target)]
  102. def store_entry(self, version: str, target: str) -> str:
  103. return f"{self.name}-{version}-{target}"
  104. def known_sha256(self, version: str, url: str) -> Optional[str]:
  105. """A digest the upstream already publishes, so `pm lock` does not
  106. have to stream the artifact to learn it. Override where a release
  107. API serves digests (GitHub's does); returning None means hash it."""
  108. return None
  109. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  110. """Turn the verified archive into the staged tree. Default: extract.
  111. Override for self-extractors or install-style unpacks (npm).
  112. Called once per artifact; extract() empties its destination, so a
  113. multi-archive package receives each later archive in a scratch dir
  114. that pm merges into the staged tree.
  115. """
  116. from pm.store import extract
  117. extract(archive, staged)
  118. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  119. """Post-unpack fixups inside the scratch dir. Default: nothing."""
  120. def binary(self, entry: Path, target: str) -> Optional[Path]:
  121. return None
  122. def verify(self, entry: Path, target: str) -> str:
  123. """Return '' when the entry is usable on target, else why not.
  124. Every subclass check (probe, marker) must keep this shape: '' is
  125. the verified answer, anything else is the diagnosis."""
  126. binary = self.binary(entry, target)
  127. if binary is None:
  128. return ""
  129. return self._binary_reason(binary, entry, target)
  130. def repair_staged_verification(self, entry: Path, target: str, reason: str) -> tuple[str, str]:
  131. """Install-time remediation for a failed staged verification.
  132. verify() is also used by read-only inspection paths such as
  133. hermes pm doctor and must stay side-effect free. The installer
  134. calls this hook only after a freshly staged entry fails verification.
  135. Return (remaining reason, remedy): an empty reason when remediation
  136. made the staged entry usable; an empty remedy keeps InstallError's.
  137. """
  138. return reason, ""
  139. def _binary_reason(self, binary: Path, entry: Path, target: str) -> str:
  140. """'' when the binary is present and arch-plausible on target."""
  141. if not binary.is_file():
  142. return _missing_reason(binary, entry)
  143. if machine_matches_binary(binary, target) is False and target not in self.emulated_arch_targets:
  144. return f"{binary.name} is not a {target} binary"
  145. return ""
  146. def env(self, entry: Path, target: str) -> dict:
  147. diff: dict = {}
  148. if self.on_path:
  149. binary = self.binary(entry, target)
  150. if binary is not None:
  151. diff["PATH"] = [str(binary.parent)]
  152. return diff
  153. class DebPackage(Package):
  154. """A .deb artifact staged by ar+tar extraction (never dpkg, never
  155. executing package content on the host).
  156. For cross-target interpreter/runtime .debs (Termux's bionic python):
  157. the staged binaries cannot exec on the host that stages them, so
  158. verify() is FILE EVIDENCE -- the control stanza carried inside the
  159. .deb (Package + Version) must match the pin. Digest verification of
  160. the downloaded bytes happens in the store, as for every package.
  161. unpack() pulls data.tar out of the ar container and hands it to
  162. pm.store.extract_tar, the one containment policy for every PM tarball
  163. (symlinks allowed with in-root targets; devices/fifos refused).
  164. """
  165. # The control field this package's .deb must declare as its name
  166. # (defaults to the package's own name).
  167. deb_package: str = ""
  168. # Where the staged tree keeps the payload: Termux .debs carry the
  169. # full $PREFIX path, data/data/com.termux/files/usr. Overridable.
  170. prefix_rel: str = "data/data/com.termux/files/usr"
  171. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  172. unpack_deb(self.name, archive, staged)
  173. def verify(self, entry: Path, target: str) -> str:
  174. """'' when the staged tree is plausible on target: the expected
  175. main binary is present. No exec (cross-target), no arch probe --
  176. the digest already proved the bytes."""
  177. expected = entry / self.prefix_rel / self.main_rel(target)
  178. if not expected.is_file() and not expected.is_symlink():
  179. return f"{expected.relative_to(entry)} missing under {native(entry)}"
  180. return ""
  181. def main_rel(self, target: str) -> str:
  182. raise NotImplementedError
  183. def deb_data_tar(name: str, archive: Path) -> bytes:
  184. """A .deb's data.tar, ready for tarfile. Ubuntu compresses it with zstd
  185. (default since 21.10), decompressed here with the stdlib module PM's
  186. 3.14 runtime carries; lzma (the pre-xz format) is refused."""
  187. raw = archive.read_bytes()
  188. if raw[:8] != b"!<arch>\n":
  189. raise InstallError(name, f"not an ar archive: {archive.name}")
  190. offset = 8
  191. while offset + 60 <= len(raw):
  192. hdr = raw[offset:offset + 60]
  193. member = hdr[0:16].decode("ascii", "replace").rstrip()
  194. try:
  195. size = int(hdr[48:58].decode("ascii", "replace").strip())
  196. except ValueError:
  197. raise InstallError(name, f"bad ar member size in {archive.name}")
  198. start = offset + 60
  199. data = raw[start:start + size]
  200. if member.startswith("data.tar"):
  201. if member.endswith(".lzma"):
  202. raise InstallError(name, f"unsupported data compression {member} in {archive.name}")
  203. if member.endswith(".zst"):
  204. try:
  205. from compression import zstd
  206. except ImportError:
  207. raise InstallError(name, f"{member} in {archive.name} needs Python 3.14+ (zstd)")
  208. return zstd.decompress(data)
  209. return data
  210. offset = start + size + (size % 2)
  211. raise InstallError(name, f"no data.tar member in {archive.name}")
  212. def unpack_deb(name: str, archive: Path, staged: Path) -> None:
  213. """Stage a .deb by ar+tar extraction: never dpkg, never package scripts.
  214. The tar goes through pm.store.extract_tar, the one containment policy for
  215. every PM tarball (symlinks allowed with in-root targets; devices/fifos refused)."""
  216. _untar_deb_payload(name, deb_data_tar(name, archive), staged)
  217. def _untar_deb_payload(name: str, payload: bytes, staged: Path) -> None:
  218. import io
  219. import stat as stat_mod
  220. import tarfile
  221. from pm.store import extract_tar
  222. try:
  223. extract_tar(io.BytesIO(payload), staged)
  224. except tarfile.FilterError as exc:
  225. member = exc.tarinfo.name if exc.tarinfo is not None else "?"
  226. raise InstallError(name, f"unsafe member {member!r}: {exc}") from exc
  227. real_staged = os.path.realpath(staged)
  228. # Termux debs carry owner-only modes across the whole tree (700 on
  229. # binaries n libs, 600 on stdlib .py files) -- postinst would
  230. # normalize on a real phone, but pm extracts without postinst, and
  231. # any uid-hostile mode breaks non-owner consumers: the dynamic
  232. # linker cannot read a 700 lib, the interpreter cannot read a 600
  233. # encoding module. Normalize EVERYTHING: a+r on all regular files,
  234. # a+X on anything that was executable. The deb's bytes are pinned
  235. # by digest; modes are not part of the pin. Symlinks are skipped:
  236. # chmod through one would follow it outside the staged tree.
  237. for root, dirs, files in os.walk(staged):
  238. dirs[:] = [d for d in dirs if not (Path(root) / d).is_symlink()]
  239. for file_name in files:
  240. f = Path(root) / file_name
  241. try:
  242. mode = f.lstat().st_mode
  243. except OSError:
  244. continue
  245. if stat_mod.S_ISLNK(mode):
  246. continue
  247. if not os.path.realpath(f).startswith(real_staged + os.sep):
  248. continue
  249. wanted = 0o644 | (0o111 if mode & 0o111 else 0)
  250. try:
  251. f.chmod(wanted)
  252. except OSError:
  253. # Best effort by design: a file we cannot chmod keeps its
  254. # extracted mode, and verify() still judges the tree.
  255. continue
  256. class StatePackage(Package):
  257. """A package that is a STATE of this install (the python venv), not a
  258. store entry. Verified by comparing a stamp; made true by apply()."""
  259. on_path = False
  260. def expected_stamp(self, extras: list[str]) -> str:
  261. raise NotImplementedError
  262. def apply(self, extras: list[str], *, explicit: bool = False) -> None:
  263. raise NotImplementedError
  264. def machine_matches_binary(binary: Path, target: str) -> Optional[bool]:
  265. """Does this executable's architecture match the target? Reads the
  266. PE/ELF/Mach-O header directly. None = unknown format (scripts, shims),
  267. which is not a mismatch."""
  268. import struct
  269. arch = target.split("-")[1]
  270. try:
  271. with open(binary, "rb") as f:
  272. head = f.read(64)
  273. if head[:2] == b"MZ":
  274. f.seek(int.from_bytes(head[60:64], "little"))
  275. sig = f.read(6)
  276. if sig[:4] != b"PE\0\0":
  277. return None
  278. machine = int.from_bytes(sig[4:6], "little")
  279. return machine == {"x64": 0x8664, "arm64": 0xAA64}.get(arch)
  280. if head[:4] == b"\x7fELF":
  281. machine = int.from_bytes(head[18:20], "little")
  282. return machine == {"x64": 0x3E, "arm64": 0xB7}.get(arch)
  283. if head[:4] in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf"):
  284. cpu = struct.unpack("<I", head[4:8])[0]
  285. return cpu == {"x64": 0x01000007, "arm64": 0x0100000C}.get(arch)
  286. if head[:4] in (b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca"):
  287. return True # universal binary carries both
  288. except OSError:
  289. return None
  290. return None
  291. def _entry_listing(entry: Path, limit: int = 12) -> str:
  292. """Top-level names of a store entry, for verification diagnoses."""
  293. if not entry.is_dir():
  294. return "store entry does not exist"
  295. names = sorted(p.name for p in entry.iterdir())
  296. shown = ", ".join(names[:limit])
  297. if len(names) > limit:
  298. shown += f", … ({len(names)} entries)"
  299. return shown
  300. def _missing_reason(binary: Path, entry: Path) -> str:
  301. """Why a package's expected binary is not where it should be — the
  302. diagnosis that tells you whether the pin's layout is wrong."""
  303. rel = binary.relative_to(entry).as_posix()
  304. return f"{rel} missing under {native(entry)}; {_entry_listing(entry)}"
  305. def _probe_reason(binary: Path, proc: subprocess.CompletedProcess) -> str:
  306. """Why a --version probe failed: the exit code plus output tail."""
  307. out = (proc.stdout or b"") + (proc.stderr or b"")
  308. tail = out.decode(errors="replace").strip()[-300:]
  309. return f"{native(binary)} --version exited {proc.returncode}" + (f": {tail}" if tail else "")
  310. class Runner:
  311. """What ensure() hands back: a composed environment and a run mirror."""
  312. def __init__(self, name: str, env: dict[str, str]):
  313. self.name = name
  314. self.env = env
  315. def run(self, cmd, **kwargs) -> subprocess.CompletedProcess:
  316. kwargs.setdefault("env", self.env)
  317. return subprocess.run(cmd, **kwargs) # noqa: PLW1510 -- forwarding wrapper: callers pass subprocess kwargs via **kwargs (check may arrive through it); an explicit check=False would raise TypeError