packages.py 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351
  1. """Package definitions for the tools hermes manages. Versions and hashes
  2. live in pm/lock.json (written by `pm lock`), never here."""
  3. from __future__ import annotations
  4. import logging
  5. import os
  6. import shutil
  7. import subprocess
  8. import sys
  9. import tempfile
  10. import uuid
  11. from pathlib import Path
  12. from typing import Optional
  13. from pm.filesystem import long_root, native
  14. from pm.package import (
  15. DebPackage,
  16. InstallError,
  17. Package,
  18. StatePackage,
  19. _entry_listing,
  20. _probe_reason,
  21. unpack_deb,
  22. )
  23. from pm.registry import register
  24. from pm.store import ALL_TARGETS, MUSL_TARGETS, Store, current_target, flatten_single_dir, merge_tree
  25. from pm.update import (
  26. btbn_index,
  27. btbn_versions,
  28. github_release_tags,
  29. llama_app_bucket_versions,
  30. llama_app_latest,
  31. martin_riedl_index,
  32. martin_riedl_versions,
  33. node_latest_versions,
  34. npm_dist_tags,
  35. pbs_versions,
  36. )
  37. LOG = logging.getLogger(__name__)
  38. _RUST_TRIPLE = {
  39. "win32-x64": "x86_64-pc-windows-msvc",
  40. "win32-arm64": "aarch64-pc-windows-msvc",
  41. "linux-x64": "x86_64-unknown-linux-gnu",
  42. "linux-arm64": "aarch64-unknown-linux-gnu",
  43. "linux-x64-musl": "x86_64-unknown-linux-musl",
  44. "linux-arm64-musl": "aarch64-unknown-linux-musl",
  45. "darwin-x64": "x86_64-apple-darwin",
  46. "darwin-arm64": "aarch64-apple-darwin",
  47. }
  48. _NODE_PLAT = {
  49. "win32-x64": "win-x64",
  50. "win32-arm64": "win-arm64",
  51. "linux-x64": "linux-x64",
  52. "linux-arm64": "linux-arm64",
  53. "linux-x64-musl": "linux-x64-musl",
  54. "linux-arm64-musl": "linux-arm64-musl",
  55. "darwin-x64": "darwin-x64",
  56. "darwin-arm64": "darwin-arm64",
  57. }
  58. class BinaryPackage(Package):
  59. """A downloaded archive exposing one binary. Covers most tools."""
  60. binary_rel: dict[str, str] = {}
  61. flatten = True
  62. probe_version = True
  63. # argv after the binary for the smoke probe. A package whose binary
  64. # rejects the GNU double-dash form (ffmpeg's BtbN autobuild) overrides.
  65. probe_args: list[str] = ["--version"]
  66. # Run the probe with cwd=binary.parent: dlopen'd backends (llama.cpp's
  67. # cudart) resolve their shared libraries from the working directory.
  68. probe_cwd = False
  69. def _rel(self, target: str) -> Optional[str]:
  70. win = target.startswith("win32")
  71. return self.binary_rel.get(target) or self.binary_rel.get(
  72. "win32" if win else "posix"
  73. )
  74. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  75. if self.flatten:
  76. flatten_single_dir(staged)
  77. def binary(self, entry: Path, target: str) -> Optional[Path]:
  78. rel = self._rel(target)
  79. return entry / rel if rel else None
  80. def verify(self, entry: Path, target: str) -> str:
  81. """Check file/architecture evidence for every target, plus a smoke
  82. probe only on the native target. Never execute cross-staged bytes."""
  83. binary = self.binary(entry, target)
  84. if binary is None:
  85. return "no binary_rel for this target"
  86. reason = self._binary_reason(binary, entry, target)
  87. if reason:
  88. return reason
  89. if not self.probe_version or target != current_target():
  90. return ""
  91. try:
  92. proc = subprocess.run(
  93. [native(binary), *self.probe_args],
  94. capture_output=True,
  95. timeout=60,
  96. cwd=native(binary.parent) if self.probe_cwd else None,
  97. env=self._probe_env(),
  98. check=False,
  99. )
  100. except OSError as e:
  101. return f"could not exec {native(binary)} {' '.join(self.probe_args)}: {e}"
  102. except subprocess.TimeoutExpired:
  103. return f"{native(binary)} {' '.join(self.probe_args)} timed out after 60s"
  104. if proc.returncode != 0:
  105. return _probe_reason(binary, proc)
  106. return ""
  107. def _probe_env(self) -> dict:
  108. """Deps' env composed in: npm's shim is `#!/usr/bin/env node` and
  109. must find the node it extends on PATH."""
  110. if not self.deps:
  111. return dict(os.environ)
  112. from pm.install import env_for
  113. return env_for(*self.deps)
  114. @register
  115. class Dmgbuild(BinaryPackage):
  116. """Build-only DMG supplier, independently pinned by PM rather than dmg-builder.
  117. The lock version is <release>+<bundle revision>. Updates are manual: review
  118. the official electron-builder-binaries bundle and re-pin both Darwin targets.
  119. Keep the paired Python tree intact for the launcher and diagnostic hook.
  120. """
  121. name = "dmgbuild"
  122. internal = True
  123. on_path = False
  124. flatten = False
  125. probe_version = False
  126. binary_rel = {"posix": "dmgbuild"}
  127. gaps = {target: "DMG creation requires macOS" for target in ALL_TARGETS if not target.startswith("darwin-")}
  128. def fetch_url(self, version: str, target: str) -> str:
  129. release, _, revision = version.partition("+")
  130. arch = {"darwin-arm64": "arm64", "darwin-x64": "x86_64"}[target]
  131. return (
  132. "https://github.com/electron-userland/electron-builder-binaries/releases/download/"
  133. f"dmg-builder@{release}/dmgbuild-bundle-{arch}-{revision}.tar.gz"
  134. )
  135. def verify(self, entry: Path, target: str) -> str:
  136. return super().verify(entry, target) or self._binary_reason(entry / "python/bin/python3", entry, target)
  137. class _BionicDebArm:
  138. """Shared bionic-arm behavior for the termux tool packages: extract as a
  139. .deb on the bionic target (DebPackage's hardened ar+tar), as the binary
  140. package otherwise, and never host-exec-probe a bionic binary (it cannot
  141. run on the staging host)."""
  142. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  143. if target == "linux-arm64-bionic":
  144. DebPackage.unpack(self, archive, staged, target)
  145. else:
  146. BinaryPackage.unpack(self, archive, staged, target)
  147. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  148. if target != "linux-arm64-bionic":
  149. BinaryPackage.stage(self, store, staged, version, target)
  150. def binary(self, entry: Path, target: str) -> Optional[Path]:
  151. if target == "linux-arm64-bionic":
  152. # File evidence, not exec: the staged .deb's main binary.
  153. # Cross-target verify() never probes it; consumers (env PATH,
  154. # bundle layout) need the real path.
  155. return entry / self.prefix_rel / self.main_rel(target)
  156. return BinaryPackage.binary(self, entry, target)
  157. def verify(self, entry: Path, target: str) -> str:
  158. # MRO order puts BinaryPackage.verify (exec-probe semantics) ahead of
  159. # DebPackage.verify (file-evidence semantics); bionic needs the
  160. # latter -- one dispatch here replaces the per-class copies.
  161. if target == "linux-arm64-bionic":
  162. return DebPackage.verify(self, entry, target)
  163. return BinaryPackage.verify(self, entry, target)
  164. @register
  165. class Uv(_BionicDebArm, BinaryPackage, DebPackage):
  166. """astral's prebuilt tarballs for glibc/musl/mac/win; the Termux main-repo
  167. uv .deb for bionic (termux builds uv from source -- no astral bionic
  168. artifact exists). The bionic arm is a runtime tool on the phone (lazy
  169. plugin installs) and the wheelhouse's resolver in the build container."""
  170. name = "uv"
  171. deps = ("python",)
  172. internal = True
  173. on_path = False
  174. binary_rel = {"win32": "uv.exe", "posix": "uv"}
  175. # The staged .deb's main binary: DebPackage.verify checks it.
  176. main_bin_rel = "bin/uv"
  177. def main_rel(self, target: str) -> str:
  178. return self.main_bin_rel
  179. deb_package = "uv"
  180. def fetch_url(self, version: str, target: str) -> str:
  181. if target == "linux-arm64-bionic":
  182. return f"https://packages.termux.dev/apt/termux-main/pool/main/u/uv/uv_{version}_aarch64.deb"
  183. triple = _RUST_TRIPLE[target]
  184. ext = "zip" if target.startswith("win32") else "tar.gz"
  185. return f"https://github.com/astral-sh/uv/releases/download/{version}/uv-{triple}.{ext}"
  186. def latest_versions(self, target: str, locked=None) -> list[str]:
  187. return github_release_tags("astral-sh/uv")
  188. # The POSIX interpreter name the pinned Windows Python also answers to (see Python.stage).
  189. WINDOWS_PYTHON3 = "python3.exe"
  190. @register
  191. class Python(_BionicDebArm, BinaryPackage, DebPackage):
  192. """The pinned interpreter for launchers and every PM-managed uv command.
  193. Optional when provisioning unrelated tools; required by uv's closure.
  194. """
  195. name = "python"
  196. optional = True
  197. probe_version = False
  198. binary_rel = {"win32": "python.exe", "posix": "bin/python3"}
  199. # The staged .deb's main binary: DebPackage.verify checks it.
  200. main_bin_rel = "bin/python3.14"
  201. def main_rel(self, target: str) -> str:
  202. return self.main_bin_rel
  203. # termux-main (official termux repo) python deb. It lags python-build-
  204. # standalone by one patch (3.14.6-1 vs 3.14.7), so the bionic row is a
  205. # manual pin -- never derived from the node version, and pm update leaves
  206. # it alone (no bionic resolver).
  207. deb_package = "python"
  208. _BIONIC_URL = (
  209. "https://packages.termux.dev/apt/termux-main/pool/main/p/python/"
  210. "python_3.14.6-1_aarch64.deb"
  211. )
  212. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  213. super().stage(store, staged, version, target)
  214. binary = self.binary(staged, target)
  215. if binary is not None and sys.platform == "darwin":
  216. from hermes_cli.macos_signing import sign_managed_python
  217. sign_managed_python(binary)
  218. # python-build-standalone ships the x64 VC runtime (vcruntime140_1.dll)
  219. # beside ARM64 Windows Python; it cannot load on ARM64 and would fail
  220. # the arch guard. Drop it HERE, before publish: the tree digest is
  221. # recorded over the shipped bytes, so a post-publish deletion (as the
  222. # old bundle-time drop did) makes doctor's re-hash mismatch.
  223. if target == "win32-arm64":
  224. (staged / "vcruntime140_1.dll").unlink(missing_ok=True)
  225. # python-build-standalone's Windows build ships python.exe only, so `python3` (npm and
  226. # vitest helpers, POSIX-minded scripts) resolves past the activated PATH to the Microsoft
  227. # Store's App Execution Alias stub, which installs nothing and fails. python.exe is a
  228. # small launcher of pythonXY.dll that finds its home from its own directory, so a copy
  229. # under the POSIX name is the same interpreter. Staged before publish, so the recorded
  230. # tree digest covers it.
  231. if target.startswith("win32"):
  232. shutil.copy2(staged / "python.exe", staged / WINDOWS_PYTHON3)
  233. def verify(self, entry: Path, target: str) -> str:
  234. reason = super().verify(entry, target)
  235. if reason or not target.startswith("win32"):
  236. return reason
  237. # An entry staged before python3.exe existed fails here, so an explicit install
  238. # (activation's setup) restages it once; startup's facts check never calls verify.
  239. alias = entry / WINDOWS_PYTHON3
  240. try:
  241. if alias.read_bytes() == (entry / "python.exe").read_bytes():
  242. return ""
  243. except OSError:
  244. return f"{WINDOWS_PYTHON3} is missing beside python.exe"
  245. return f"{WINDOWS_PYTHON3} differs from python.exe"
  246. def fetch_url(self, version: str, target: str) -> str:
  247. if target == "linux-arm64-bionic":
  248. return self._BIONIC_URL
  249. # lock version is "<python>+<release tag>", e.g. "3.14.7+20260901"
  250. pyver, _, tag = version.partition("+")
  251. if not tag:
  252. raise InstallError(self.name, f"version {version!r} needs the +<release> tag")
  253. triple = _RUST_TRIPLE[target]
  254. return (
  255. "https://github.com/astral-sh/python-build-standalone/releases/download/"
  256. f"{tag}/cpython-{pyver}+{tag}-{triple}-install_only.tar.gz"
  257. )
  258. def latest_versions(self, target: str, locked=None) -> list[str]:
  259. # Keep the locked minor line. The advertised asset owns patch and build.
  260. # Bionic remains a manual pin from a separate supplier.
  261. if target == "linux-arm64-bionic" or not locked or "+" not in locked:
  262. return []
  263. pyver = locked.partition("+")[0]
  264. minor = ".".join(pyver.split(".")[:2])
  265. return pbs_versions(minor, _RUST_TRIPLE[target])
  266. def _uv_lock_digest(path: Path) -> bytes:
  267. """sha256 of uv.lock, cached on (mtime_ns, size) — check() runs at
  268. every startup and uv.lock is megabyte-class."""
  269. import hashlib
  270. stat = path.stat()
  271. key = (stat.st_mtime_ns, stat.st_size)
  272. cached = _uv_lock_digest_cache.get(path)
  273. if cached and cached[0] == key:
  274. return cached[1]
  275. digest = hashlib.sha256(path.read_bytes()).digest()
  276. _uv_lock_digest_cache[path] = (key, digest)
  277. return digest
  278. _uv_lock_digest_cache: dict[Path, tuple] = {}
  279. def _copy_seed_file(src: str | os.PathLike, dst: str | os.PathLike) -> str | os.PathLike:
  280. """Copy one payload cache file unless an earlier seed already copied it whole.
  281. A copy cut short leaves the file in place with too few bytes, so a matching
  282. size is what counts as done; a finished file is not rewritten.
  283. """
  284. try:
  285. if os.stat(dst).st_size == os.stat(src).st_size:
  286. return dst
  287. except FileNotFoundError:
  288. pass
  289. # Other profiles share this cache: publish only a complete copy, without
  290. # truncating the live entry or leaving a failed copy visible to readers.
  291. fd, staging = tempfile.mkstemp(prefix=".seed-", dir=Path(dst).parent)
  292. os.close(fd)
  293. try:
  294. shutil.copy2(src, staging)
  295. os.replace(staging, dst)
  296. finally:
  297. Path(staging).unlink(missing_ok=True)
  298. return dst
  299. def _copy_seed_link(src: str | os.PathLike, dst: str | os.PathLike) -> None:
  300. """Recreate one payload cache symlink, healing a directory left by an older seed.
  301. uv caches a wheel as a symlink into ``archive-v0/``; following it (the
  302. ``copytree`` default) leaves a directory where uv later renames its own
  303. entry into place, which fails with EISDIR. The payload's targets are
  304. relative, so the target is copied verbatim, never resolved.
  305. """
  306. linkto = os.readlink(src)
  307. dst = Path(dst)
  308. staging = dst.parent / f".seed-link-{uuid.uuid4().hex}"
  309. os.symlink(linkto, staging)
  310. try:
  311. if dst.is_dir() and not dst.is_symlink():
  312. # A pre-fix seed copied the link target's contents here; uv cannot replace it.
  313. shutil.rmtree(dst)
  314. os.replace(staging, dst) # atomic: replaces a file or an existing symlink
  315. finally:
  316. staging.unlink(missing_ok=True)
  317. def _seed_tree(src: Path, dst: Path) -> None:
  318. """Merge one payload bucket, preserving uv's link entries.
  319. ``is_symlink()`` must be tested before ``is_dir()``: ``is_dir()`` follows
  320. links, and following them is exactly the bug this walker replaced.
  321. """
  322. dst.mkdir(parents=True, exist_ok=True)
  323. for child in src.iterdir():
  324. target = dst / child.name
  325. if child.is_symlink():
  326. _copy_seed_link(child, target)
  327. elif child.is_dir():
  328. _seed_tree(child, target)
  329. else:
  330. _copy_seed_file(child, target)
  331. def uv_cache_path() -> Path:
  332. """Where the hermes-owned uv cache lives; unlike uv_cache_dir(), never seeds or writes."""
  333. from hermes_constants import get_default_hermes_root
  334. return get_default_hermes_root() / "cache" / "uv"
  335. def uv_cache_dir() -> Path:
  336. """The hermes-owned uv cache: machine-scoped and shared (keyed by
  337. content — two profiles reuse one cache), anchored to the DEFAULT
  338. hermes root like partials_root(). A bundle ships a seeded copy at
  339. the payload root (uv-cache/); the first call under each uv.lock
  340. merges it into the writable machine cache (the read-only payload
  341. can't serve uv's working cache), and a warm `uv sync --offline`
  342. from it is near-free (probed: 0.4s vs 1.2s cold) — the blow-away-
  343. on-update contract depends on it. uv's default cache location is
  344. per-user/platform-opinionated and never used by pm."""
  345. from pm.paths import repo_root, store_root
  346. machine_cache = uv_cache_path()
  347. marker = machine_cache / ".seeded"
  348. # The marker names the lock whose payload was merged. An update ships a new payload cache for
  349. # a new lock; a bare "done" marker skipped it forever, so plugin rebuilds fetched the new pins
  350. # from the network (and built sdists where the index has no wheel for the target).
  351. lock = repo_root() / "uv.lock"
  352. seed = _uv_lock_digest(lock).hex() if lock.is_file() else "1"
  353. # The marker also records whether a payload cache was present. An image without one still
  354. # writes the marker for its lock, and a later image with the same lock must still merge its
  355. # seed. Do not drop the suffix: the lock alone cannot tell those two images apart.
  356. payload_cache = long_root(store_root().parent / "uv-cache")
  357. if payload_cache.is_dir():
  358. # The suffix names the seed's copy semantics, not just "a payload exists": it moved to
  359. # ``payload2`` when the seed stopped following links, so an existing volume re-runs the
  360. # merge once and the directory-shaped entries the old copy left are healed in place.
  361. seed += "+payload2"
  362. try:
  363. current = marker.read_text(encoding="utf-8-sig").strip() == seed
  364. except OSError:
  365. current = False
  366. if not current:
  367. # Seed from a shipped bundle cache when present (payload root =
  368. # store_root().parent on a sealed install). Record completion ONLY after a clean copy: a
  369. # partial seed that marked itself done would never be retried, and every later offline
  370. # sync that needs the missing entries fails closed.
  371. try:
  372. # uv's cache trees run deep; both roots get the long spelling so
  373. # the copy is not cut at MAX_PATH. The returned path stays ordinary.
  374. if payload_cache.is_dir():
  375. seeded = long_root(machine_cache)
  376. seeded.mkdir(parents=True, exist_ok=True)
  377. # archive-v0 first: a wheel link published before its bucket points at nothing,
  378. # for another profile's uv and, if the seed dies there, until the retry.
  379. for entry in sorted(payload_cache.iterdir(), key=lambda entry: entry.name != "archive-v0"):
  380. if entry.name == ".seeded":
  381. continue
  382. dest = seeded / entry.name
  383. # Merge into whatever an earlier failed or killed seed left: skipping
  384. # an existing directory would record that partial copy as complete.
  385. if entry.is_symlink():
  386. _copy_seed_link(entry, dest)
  387. elif entry.is_dir():
  388. _seed_tree(entry, dest)
  389. else:
  390. _copy_seed_file(entry, dest)
  391. except OSError as exc:
  392. LOG.warning("uv cache seed incomplete, retrying on the next install: %s", exc)
  393. else:
  394. try:
  395. marker.parent.mkdir(parents=True, exist_ok=True)
  396. marker.write_text(seed, encoding="utf-8")
  397. except OSError:
  398. pass
  399. return machine_cache
  400. @register
  401. class Venv(StatePackage):
  402. """The project venv: pyproject.toml + uv.lock + enabled extras.
  403. Made true by `uv sync --frozen`; uv is its internal dependency."""
  404. name = "venv"
  405. deps = ("uv",)
  406. def __init__(self, project_root: Path | None = None):
  407. self._project_root = project_root
  408. def project_root(self) -> Path:
  409. from pm.paths import repo_root
  410. return repo_root() if self._project_root is None else self._project_root
  411. def venv_dir(self) -> Path:
  412. from pm.environments import selected_venv
  413. return selected_venv(self.project_root())
  414. def expected_stamp(self, extras: list[str], *, plugin_dirs=None) -> str:
  415. import hashlib
  416. import json
  417. from pm.lock import Lockfile
  418. from pm.paths import lockfile_path
  419. from pm.store import current_target
  420. lock = Lockfile(lockfile_path())
  421. target = current_target()
  422. python = (lock.version("python"), target,
  423. [artifact["sha256"] for artifact in lock.artifacts("python", target)])
  424. h = hashlib.sha256()
  425. h.update(_uv_lock_digest(self.project_root() / "uv.lock"))
  426. h.update(",".join(sorted(extras)).encode())
  427. h.update(json.dumps(python).encode())
  428. # Plugin members union into the venv — a changed member set must
  429. # re-sync even when extras and core lock are unchanged.
  430. from pm.workspace import build_member_dirs, members_stamp
  431. h.update(members_stamp(build_member_dirs(extras) if plugin_dirs is None else plugin_dirs).encode())
  432. return h.hexdigest()
  433. def apply(self, extras: list[str], *, plugin_dirs=None, repair: bool = False, explicit: bool = False,
  434. skip_invalid_secondary: bool = False) -> dict:
  435. """Prepare one complete environment; the caller commits its selection.
  436. ``skip_invalid_secondary`` is the update's contract: an unreadable secondary profile
  437. is left out (the caller reports it) instead of refusing the whole graph.
  438. """
  439. import uuid
  440. from pm.environments import install_state_dir, runtime_facts_path
  441. from pm.environment import managed_environment
  442. from pm.lock import Facts
  443. from pm.native_build import source_build_environment
  444. from pm.workspace import build_member_dirs, lock_and_sync
  445. project = self.project_root()
  446. generation = install_state_dir(project) / "environments" / uuid.uuid4().hex
  447. candidate = generation / "venv"
  448. environment = managed_environment(candidate, env=source_build_environment(project),
  449. explicit=explicit or repair, output=sys.stderr)
  450. if not repair:
  451. # Inspection may skip a broken secondary profile, but publishing a replacement
  452. # graph must not silently evict its recorded members (including passed candidates).
  453. # An update does evict them, loudly: it must not fail on another profile's config.
  454. from pm.plugins_state import enabled_plugins_ordered
  455. enabled_plugins_ordered(skip_invalid_secondary=skip_invalid_secondary)
  456. members = [] if repair else (build_member_dirs(extras, environment.python) if plugin_dirs is None
  457. else plugin_dirs)
  458. try:
  459. generation.mkdir(parents=True)
  460. (generation / ".lease-managed").touch()
  461. environment.create()
  462. prior = Facts(runtime_facts_path(project), strict=repair).get("venv") or {}
  463. replay = None
  464. if repair and ("environment" in prior or "resolved_lock" in prior):
  465. if not all(isinstance(prior.get(key), str) and prior[key] for key in ("environment", "resolved_lock")):
  466. raise InstallError(self.name, "recorded dependency paths are incomplete; refusing to drop plugins")
  467. recorded = Path(prior["resolved_lock"]).resolve()
  468. previous = Path(prior["environment"]).resolve().parent
  469. generations = install_state_dir(project) / "environments"
  470. if (previous.parent != generations.resolve() or recorded != previous / "workspace" / "uv.lock"
  471. or not recorded.is_file()):
  472. raise InstallError(self.name, "recorded dependency lock is missing; refusing to drop plugins")
  473. replay = recorded.parent
  474. seed = (Path(prior["resolved_lock"]) if members and prior.get("resolved_lock")
  475. else project / "uv.lock")
  476. lock_and_sync(members, extras, root=generation / "workspace",
  477. seed_lock=seed, frozen=repair or not members, replay=replay,
  478. source=project, environment=environment)
  479. resolved_lock = generation / "workspace" / "uv.lock"
  480. environment.check(resolved_lock)
  481. if repair:
  482. from pm.environments import venv_command
  483. from pm.recovery import validate_environment
  484. validate_environment(venv_command(project, candidate, ("-I",)),
  485. env=dict(environment.env), cwd=resolved_lock.parent)
  486. except BaseException:
  487. shutil.rmtree(generation, ignore_errors=True)
  488. raise
  489. (generation / ".lease-managed").touch()
  490. return {"environment": candidate, "resolved_lock": resolved_lock}
  491. @register
  492. class Nodejs(_BionicDebArm, BinaryPackage, DebPackage):
  493. """nodejs.org tarballs for glibc/mac/win, unofficial-builds for musl;
  494. the Termux main-repo nodejs .deb for bionic (same major line, termux-built)."""
  495. name = "node"
  496. binary_rel = {"win32": "node.exe", "posix": "bin/node"}
  497. # The staged .deb's main binary: DebPackage.verify checks it.
  498. main_bin_rel = "bin/node"
  499. def main_rel(self, target: str) -> str:
  500. return self.main_bin_rel
  501. deb_package = "nodejs"
  502. def fetch_url(self, version: str, target: str) -> str:
  503. if target == "linux-arm64-bionic":
  504. # termux's deb carries a -1 revision after the upstream version
  505. return f"https://packages.termux.dev/apt/termux-main/pool/main/n/nodejs/nodejs_{version}-1_aarch64.deb"
  506. plat = _NODE_PLAT[target]
  507. ext = "zip" if target.startswith("win32") else "tar.xz"
  508. base = (
  509. "https://unofficial-builds.nodejs.org/download/release"
  510. if target in MUSL_TARGETS
  511. else "https://nodejs.org/dist"
  512. )
  513. return f"{base}/v{version}/node-v{version}-{plat}.{ext}"
  514. def latest_versions(self, target: str, locked=None) -> list[str]:
  515. # Keep one Node version across targets. If unofficial musl publication
  516. # lags nodejs.org, the later artifact pin/download fails before the
  517. # lockfile is written rather than selecting glibc bytes on musl.
  518. return node_latest_versions()
  519. def repair_staged_verification(self, entry: Path, target: str, reason: str) -> tuple[str, str]:
  520. """Repair the host library needed by official Linux Node, then re-probe.
  521. Keep this out of verify(): doctor/status checks call verify and must
  522. never gain permission to install host packages.
  523. """
  524. if target != current_target() or not target.startswith("linux") or "libatomic.so.1" not in reason:
  525. return reason, ""
  526. from pm.libatomic import try_install_libatomic
  527. installed, remedy = try_install_libatomic()
  528. if not installed:
  529. return reason, remedy
  530. retried = self.verify(entry, target)
  531. if "libatomic.so.1" in retried:
  532. return retried, "the libatomic package installed, but Node still cannot load libatomic.so.1; check the loader path"
  533. return retried, ""
  534. @register
  535. class TermuxDocker(Package):
  536. """The termux/termux-docker container image, pinned by registry digest.
  537. The image is never downloaded or unpacked by pm -- docker pulls it by
  538. digest reference at build time. The lock row exists so the digest is
  539. pinned in the single pin authority beside every other third-party
  540. artifact: consumers read the digest string from the lock's url field
  541. (termux/termux-docker@sha256:...). verify() is presence-shaped: this
  542. package stages nothing.
  543. """
  544. name = "termux-docker"
  545. optional = True
  546. # Pure pin: no bytes are staged, so stage_only()/install skip the store
  547. # entirely -- the digest's consumers (docker pull) verify it.
  548. pin_only = True
  549. def missing_reason(self, target: str) -> Optional[str]:
  550. return None if target == "linux-arm64-bionic" else "docker image target is linux-arm64-bionic"
  551. def fetch_url(self, version: str, target: str) -> str:
  552. return f"docker://termux/termux-docker@{version}"
  553. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  554. raise InstallError(self.name, "a docker image digest is a pin, not a downloadable artifact")
  555. def verify(self, entry: Path, target: str) -> str:
  556. return ""
  557. def npm_env(cache_dir: Path, base_env: Optional[dict] = None) -> dict[str, str]:
  558. """Keep ambient Node and npm options out of PM's child process."""
  559. env = {
  560. key: value
  561. for key, value in (os.environ if base_env is None else base_env).items()
  562. if not key.lower().startswith("npm_config_")
  563. and key.upper() not in ("NODE_OPTIONS", "NODE_PATH", "NODE_ENV")
  564. }
  565. env["npm_config_cache"] = str(cache_dir)
  566. return env
  567. @register
  568. class Npm(BinaryPackage):
  569. name = "npm"
  570. deps = ("node",)
  571. binary_rel = {"win32": "npm.cmd", "posix": "bin/npm"}
  572. flatten = False
  573. probe_version = False
  574. url = "https://registry.npmjs.org/npm/-/npm-{version}.tgz"
  575. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  576. """npm installs itself using the node it extends: a plain unpack
  577. resolves the cli from dirname(process.execPath) and finds node's
  578. bundled npm instead. --offline pins the bytes to the verified
  579. tarball; --ignore-scripts + a sanitized env keep user npm/node
  580. config out of the staging."""
  581. if target == "linux-arm64-bionic":
  582. from pm.store import extract
  583. unpacked = staged / ".unpacked"
  584. extract(archive, unpacked)
  585. package = unpacked / "package"
  586. lib = staged / "lib/node_modules/npm"
  587. lib.parent.mkdir(parents=True, exist_ok=True)
  588. package.rename(lib)
  589. unpacked.rmdir()
  590. bindir = staged / "bin"
  591. bindir.mkdir()
  592. for name in ("npm", "npx"):
  593. wrapper = bindir / name
  594. wrapper.write_text(
  595. "#!/data/data/com.termux/files/usr/bin/sh\n"
  596. 'here="$(cd "$(dirname "$0")" && pwd)"\n'
  597. f'exec node "$here/../lib/node_modules/npm/bin/{name}-cli.js" "$@"\n',
  598. encoding="utf-8",
  599. )
  600. wrapper.chmod(0o755)
  601. return
  602. from pm.install import _installed_location, _lockfile
  603. from pm.registry import get_package
  604. node = get_package("node")
  605. location = _installed_location(node, _lockfile(), target)
  606. if location is None:
  607. raise InstallError(self.name, "npm extends node, which is not installed")
  608. facts, store = location
  609. node_fact = facts.get("node")
  610. if node_fact is None:
  611. raise InstallError(self.name, "npm extends node, which is not installed")
  612. node_bin = node.binary(store.entry(node_fact["entry"]), target)
  613. if node_bin is None or not node_bin.is_file():
  614. raise InstallError(self.name, "node's entry is missing its binary")
  615. win = target.startswith("win32")
  616. bundled_cli = (
  617. node_bin.parent / "node_modules" / "npm" / "bin" / "npm-cli.js"
  618. if win
  619. else node_bin.parent.parent / "lib" / "node_modules" / "npm" / "bin" / "npm-cli.js"
  620. )
  621. if not bundled_cli.is_file():
  622. raise InstallError(self.name, "node's entry is missing its bundled npm-cli.js")
  623. staged.mkdir(parents=True, exist_ok=True)
  624. # npm caches the tarball it installs. The archive's directory is the
  625. # store's download entry, which must hold only the archive: a cache
  626. # there turns its removal into a tree delete that fails on Windows
  627. # while Defender still holds the fresh copy (WinError 145). Cleanup
  628. # of this throwaway cache must never fail the install.
  629. with tempfile.TemporaryDirectory(prefix="hermes-npm-cache-", ignore_cleanup_errors=True) as cache:
  630. proc = subprocess.run(
  631. [
  632. native(node_bin), native(bundled_cli), "install", "--global",
  633. "--prefix", native(staged), "--offline", "--ignore-scripts",
  634. "--no-audit", "--no-fund", native(archive),
  635. ],
  636. capture_output=True,
  637. text=True,
  638. timeout=900,
  639. env=npm_env(Path(cache)),
  640. check=False,
  641. )
  642. if proc.returncode != 0:
  643. raise InstallError(
  644. self.name, f"self-install exited {proc.returncode}: {proc.stderr[-400:]}"
  645. )
  646. def latest_versions(self, target: str, locked=None) -> list[str]:
  647. latest = npm_dist_tags("npm").get("latest")
  648. return [latest] if latest else []
  649. # The pinned git artifact is a self-extracting 7z: only hosts where PE images
  650. # execute can stage it (tests patch this flag).
  651. _HOST_IS_WINDOWS = os.name == "nt"
  652. @register
  653. class Git(BinaryPackage):
  654. """Windows only: Git for Windows carries the bash.exe contract. POSIX
  655. uses the system git - a deliberate gap, not an oversight. The pin is the
  656. PortableGit self-extracting 7z: it carries its own extractor (stock
  657. Windows 10 tar.exe has no bzip2), shows a progress window (-y only drops
  658. prompts), and runs the vendor post-install, so the staged tree is
  659. post-install output."""
  660. name = "git"
  661. optional = True
  662. binary_rel = {"win32": "cmd/git.exe"}
  663. flatten = False
  664. gaps = {
  665. "linux-x64": "POSIX uses system git by choice",
  666. "linux-arm64": "POSIX uses system git by choice",
  667. "linux-x64-musl": "POSIX uses system git by choice",
  668. "linux-arm64-musl": "POSIX uses system git by choice",
  669. "darwin-x64": "POSIX uses system git by choice",
  670. "darwin-arm64": "POSIX uses system git by choice",
  671. }
  672. def fetch_url(self, version: str, target: str) -> str:
  673. tag, build = version.split("+")
  674. arch = "arm64" if target.endswith("arm64") else "64-bit"
  675. return (
  676. f"https://github.com/git-for-windows/git/releases/download/"
  677. f"v{tag}.windows.{build}/PortableGit-{tag}.{build}-{arch}.7z.exe"
  678. )
  679. def latest_versions(self, target: str, locked=None) -> list[str]:
  680. # Tag is v2.53.0.windows.3; the lock version is 2.53.0+3.
  681. out = []
  682. for tag in github_release_tags("git-for-windows/git", strip_prefix="v"):
  683. if ".windows." in tag:
  684. base, _, build = tag.partition(".windows.")
  685. out.append(f"{base}+{build}")
  686. return out
  687. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  688. if not _HOST_IS_WINDOWS:
  689. raise InstallError(
  690. self.name,
  691. "the pinned artifact is a PortableGit self-extracting 7z, which "
  692. "only runs on Windows",
  693. "stage win32 git on a Windows host; POSIX hosts use the system git",
  694. )
  695. shutil.rmtree(staged, ignore_errors=True)
  696. staged.mkdir(parents=True)
  697. # Execute a scratch copy, never the cached fetch-<sha> bytes: an executed
  698. # PE can stay handle-held (on-execute AV scan, the RunProgram children)
  699. # past pm's download-cleanup retry and fail the install (WinError 32).
  700. with tempfile.TemporaryDirectory(
  701. prefix=".sfx-", dir=staged.parent, ignore_cleanup_errors=True
  702. ) as work:
  703. exe = Path(work) / archive.name
  704. shutil.copy2(archive, exe)
  705. # No pipes: RunProgram children would inherit them and hold run()
  706. # open past the stub's exit. Under -y the stub prints nothing anyway.
  707. try:
  708. proc = subprocess.run(
  709. [native(exe), f"-o{native(staged)}", "-y"],
  710. stdin=subprocess.DEVNULL,
  711. stdout=subprocess.DEVNULL,
  712. stderr=subprocess.DEVNULL,
  713. timeout=600,
  714. check=False,
  715. )
  716. except subprocess.TimeoutExpired:
  717. raise InstallError(
  718. self.name, "PortableGit self-extractor did not finish in 600 s"
  719. ) from None
  720. if proc.returncode:
  721. raise InstallError(
  722. self.name,
  723. f"PortableGit self-extractor exited {proc.returncode} (it reports "
  724. "nothing under -y; usual causes: disk full, path-length limit, "
  725. "antivirus lock)",
  726. )
  727. def env(self, entry: Path, target: str) -> dict:
  728. return {"PATH": [str(entry / "cmd"), str(entry / "usr" / "bin")]}
  729. @register
  730. class Gh(BinaryPackage):
  731. name = "gh"
  732. optional = True
  733. binary_rel = {"win32": "bin/gh.exe", "posix": "bin/gh"}
  734. def fetch_url(self, version: str, target: str) -> str:
  735. # GitHub CLI's Linux release matrix is built with CGO_ENABLED=0,
  736. # so the generic Linux archive is libc-independent.
  737. lookup_target = target.removesuffix("-musl") if target in MUSL_TARGETS else target
  738. osname, arch = lookup_target.split("-")
  739. plat = {"win32": "windows", "linux": "linux", "darwin": "macOS"}[osname]
  740. arch = {"x64": "amd64", "arm64": "arm64"}[arch]
  741. ext = "zip" if osname in ("win32", "darwin") else "tar.gz"
  742. return (
  743. f"https://github.com/cli/cli/releases/download/v{version}/"
  744. f"gh_{version}_{plat}_{arch}.{ext}"
  745. )
  746. def latest_versions(self, target: str, locked=None) -> list[str]:
  747. return github_release_tags("cli/cli", strip_prefix="v")
  748. @register
  749. class Ffmpeg(_BionicDebArm, BinaryPackage, DebPackage):
  750. """FFmpeg builds for supported targets; BtbN Linux archives require glibc.
  751. optional=False: ffmpeg is a required runtime tool where supported. Sealed bundles ship
  752. it baked into the payload; every `hermes update` and `hermes pm install`
  753. re-ensures it from the new lockfile before the venv sync
  754. (pm.client.ensure_tools_for_sync), so a pin bump lands. Windows + Linux:
  755. BtbN/FFmpeg-Builds (month-end autobuild tag, kept two years; ships ffprobe too).
  756. macOS: ffmpeg.martin-riedl.de (uniform ZIP, published sha256;
  757. single-binary — no ffprobe).
  758. Linux deliberately does NOT use martin-riedl: that build is compiled
  759. without x11grab (confirmed on the pinned 9.0.1 amd64 binary), and
  760. x11grab is how screen capture on X11 works."""
  761. name = "ffmpeg"
  762. deb_package = "ffmpeg"
  763. optional = False
  764. gaps = {target: "BtbN Linux builds link glibc dynamically" for target in MUSL_TARGETS}
  765. def main_rel(self, target: str) -> str:
  766. return "bin/ffmpeg"
  767. # macOS (martin-riedl, the only remaining posix stream) and Windows/Linux
  768. # (BtbN) have no shared release cadence — they drift in PATCH. The lockfile
  769. # version label is major.minor; each target's exact patch lives in its
  770. # artifact urls.
  771. version_style = "minor"
  772. # martin-riedl (macOS) zips are a single `ffmpeg` at the zip root; BtbN
  773. # ships bin/ffmpeg (Linux, .tar.xz) and bin/ffmpeg.exe (Windows, .zip)
  774. # under one top-level dir that flatten hoists.
  775. binary_rel = {
  776. "win32": "bin/ffmpeg.exe",
  777. "linux-x64": "bin/ffmpeg",
  778. "linux-arm64": "bin/ffmpeg",
  779. "posix": "ffmpeg",
  780. }
  781. flatten = True
  782. # BtbN autobuild n9.0.1-11-ge47273f4d9 rejects `--version`
  783. # ("Unrecognized option '-version'", exit 2880417800); `-version` works
  784. # and is accepted by every ffmpeg build.
  785. probe_args = ["-version"]
  786. def fetch_url(self, version: str, target: str) -> str:
  787. if target == "linux-arm64-bionic":
  788. return f"https://packages.termux.dev/apt/termux-main/pool/main/f/ffmpeg/ffmpeg_{version}_aarch64.deb"
  789. if target in MUSL_TARGETS:
  790. raise InstallError(self.name, f"unavailable on {target}: {self.missing_reason(target)}")
  791. osname, arch = target.split("-")
  792. if osname in ("win32", "linux"):
  793. artifact = btbn_index().get(target, {}).get(version)
  794. if artifact is not None:
  795. tag, asset = artifact
  796. return f"https://github.com/BtbN/FFmpeg-Builds/releases/download/{tag}/{asset}"
  797. else:
  798. epoch = martin_riedl_index().get(target, {}).get(version)
  799. if epoch is not None:
  800. source_arch = "amd64" if arch == "x64" else arch
  801. return f"https://ffmpeg.martin-riedl.de/download/macos/{source_arch}/{epoch}_{version}/ffmpeg.zip"
  802. # Existing installs read exact URLs from the lockfile. Re-pinning
  803. # must never silently substitute a different version or target.
  804. raise InstallError(self.name, f"no advertised {version} artifact for {target}",
  805. "retry when the upstream index is available, or keep the existing pin")
  806. def latest_versions(self, target: str, locked=None) -> list[str]:
  807. if target in MUSL_TARGETS:
  808. return []
  809. if target in ("win32-x64", "win32-arm64", "linux-x64", "linux-arm64"):
  810. return btbn_versions(target)
  811. return martin_riedl_versions(target)
  812. @register
  813. class Ripgrep(BinaryPackage):
  814. name = "ripgrep"
  815. binary_rel = {"win32": "rg.exe", "posix": "rg"}
  816. # The ARM64 Windows release links the VC++ runtime dynamically (the x64
  817. # one embeds it), and a fresh Windows has no vcruntime140.dll: rg.exe
  818. # dies with STATUS_DLL_NOT_FOUND. Our pinned Python ships the ARM64 DLL.
  819. deps = ("python",)
  820. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  821. super().stage(store, staged, version, target)
  822. if target != "win32-arm64":
  823. return
  824. from pm.install import _installed_location, _lockfile
  825. from pm.registry import get_package
  826. python = get_package("python")
  827. location = _installed_location(python, _lockfile(), target)
  828. fact = location[0].get("python") if location is not None else None
  829. if fact is None:
  830. raise InstallError(self.name, "ARM64 rg.exe needs vcruntime140.dll from python, which is not installed")
  831. runtime = location[1].entry(fact["entry"]) / "vcruntime140.dll"
  832. if not runtime.is_file():
  833. raise InstallError(self.name, f"python's entry has no {runtime.name}")
  834. # App-local copy: the loader searches the exe's own folder first, and
  835. # the entry digest is recorded over it.
  836. shutil.copy2(runtime, staged / runtime.name)
  837. def verify(self, entry: Path, target: str) -> str:
  838. if target == "linux-arm64-bionic":
  839. return Package.verify(self, entry, target)
  840. return super().verify(entry, target)
  841. def fetch_url(self, version: str, target: str) -> str:
  842. if target == "linux-arm64-bionic":
  843. target = "linux-arm64" # This upstream artifact is a static musl executable.
  844. triple = _RUST_TRIPLE[target].replace("-unknown-linux-gnu", "-unknown-linux-musl")
  845. ext = "zip" if target.startswith("win32") else "tar.gz"
  846. return (
  847. f"https://github.com/BurntSushi/ripgrep/releases/download/{version}/"
  848. f"ripgrep-{version}-{triple}.{ext}"
  849. )
  850. def latest_versions(self, target: str, locked=None) -> list[str]:
  851. return github_release_tags("BurntSushi/ripgrep")
  852. @register
  853. class CuaDriver(BinaryPackage):
  854. name = "cua-driver"
  855. optional = True
  856. # Computer use's only OS path, so the default install carries it.
  857. default = True
  858. gaps = {**{target: "cua-driver does not publish a musl build" for target in MUSL_TARGETS},
  859. "linux-arm64-bionic": "cua-driver does not publish an Android build"}
  860. binary_rel = {
  861. "darwin-arm64": "CuaDriver.app/Contents/MacOS/cua-driver",
  862. "darwin-x64": "CuaDriver.app/Contents/MacOS/cua-driver",
  863. "win32": "cua-driver.exe",
  864. "posix": "cua-driver",
  865. }
  866. def fetch_url(self, version: str, target: str) -> str:
  867. arch = {
  868. "darwin-x64": "darwin-universal",
  869. "darwin-arm64": "darwin-universal",
  870. "linux-x64": "linux-x86_64",
  871. "linux-arm64": "linux-arm64",
  872. "win32-x64": "windows-x86_64",
  873. "win32-arm64": "windows-arm64",
  874. }[target]
  875. ext = "zip" if target.startswith("win32") else "tar.gz"
  876. # Only the directory archive contains the signed macOS app identity
  877. # needed by TCC and private sessions. Other targets' binary archives
  878. # already carry their runtime helpers (including Windows UIAccess).
  879. variant = "" if target.startswith("darwin") else "-binary"
  880. return (
  881. f"https://github.com/trycua/cua/releases/download/cua-driver-rs-v{version}/"
  882. f"cua-driver-rs-{version}-{arch}{variant}.{ext}"
  883. )
  884. def latest_versions(self, target: str, locked=None) -> list[str]:
  885. return github_release_tags("trycua/cua", strip_prefix="cua-driver-rs-v")
  886. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  887. flatten_single_dir(staged)
  888. sdk = staged / "sdk"
  889. if sdk.is_dir():
  890. shutil.rmtree(sdk, ignore_errors=True)
  891. def _probe_env(self) -> dict:
  892. """The probe IS a first run: it must not mint telemetry state."""
  893. try:
  894. from tools.computer_use.cua_backend import cua_driver_child_env
  895. return cua_driver_child_env()
  896. except Exception:
  897. return dict(os.environ, CUA_DRIVER_RS_TELEMETRY_ENABLED="0")
  898. @register
  899. class Chromium(Package):
  900. """Playwright resolves browsers by DIRECTORY NAME under one root:
  901. `<name with '-'→'_'>-<revision>`, no target suffix. The env points
  902. PLAYWRIGHT_BROWSERS_PATH at the store root itself. The entry carries
  903. the INSTALLATION_COMPLETE marker playwright checks.
  904. The version is `<playwright revision>+<chrome version>` — most targets
  905. download from the Chrome-for-Testing CDN by chrome version, and the
  906. targets CfT doesn't build (linux-arm64) come from playwright's own
  907. mirror by revision. The store entry is named by revision only, which
  908. is all playwright's resolver reads."""
  909. name = "chromium"
  910. optional = True
  911. # browser_exec's launcher drives this build, and its readiness check
  912. # installs it only once per process, so default-install it like the
  913. # other OS tools; `install.sh --skip-browser` declines it.
  914. default = True
  915. on_path = False
  916. # Neither Chrome-for-Testing nor Playwright's mirror builds for Android.
  917. gaps = {
  918. "linux-arm64-bionic": "no Chromium build for Android/Termux",
  919. **{target: "Playwright/Chrome-for-Testing publishes no musl build" for target in MUSL_TARGETS},
  920. }
  921. emulated_arch_targets = frozenset({"win32-arm64"})
  922. _CDN = "https://cdn.playwright.dev"
  923. # Chrome-for-Testing platform names; targets absent here fall back to
  924. # playwright's dbazure mirror with its own platform names.
  925. # win32-arm64 uses the win64 (x64) build: CfT publishes no native
  926. # win-arm64 chromium, and Windows ARM64 runs x64 binaries via built-in
  927. # emulation.
  928. _CFT = {
  929. "linux-x64": "linux64",
  930. "darwin-x64": "mac-x64",
  931. "darwin-arm64": "mac-arm64",
  932. "win32-x64": "win64",
  933. "win32-arm64": "win64",
  934. }
  935. _MIRROR = {"linux-arm64": "linux-arm64"}
  936. def store_entry(self, version: str, target: str) -> str:
  937. revision = version.partition("+")[0]
  938. return f"{self.name.replace('-', '_')}-{revision}"
  939. def fetch_url(self, version: str, target: str) -> str:
  940. revision, _, chrome = version.partition("+")
  941. plat = self._CFT.get(target)
  942. if plat and chrome:
  943. return f"{self._CDN}/builds/cft/{chrome}/{plat}/chrome-{plat}.zip"
  944. mirror_plat = self._MIRROR[target]
  945. return (
  946. f"{self._CDN}/dbazure/download/playwright/builds/chromium/"
  947. f"{revision}/chromium-{mirror_plat}.zip"
  948. )
  949. def binary(self, entry: Path, target: str) -> Optional[Path]:
  950. # CfT and Playwright's ARM Linux archive use different enclosing
  951. # directories. Resolve the executable within the selected entry.
  952. names = {"chrome.exe"} if target.startswith("win32") else {"chrome", "chromium", "Google Chrome for Testing", "Chromium"}
  953. return next((p for p in sorted(entry.rglob("*")) if p.name in names and p.is_file()), None)
  954. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  955. (staged / "INSTALLATION_COMPLETE").write_text("", encoding="utf-8")
  956. def verify(self, entry: Path, target: str) -> str:
  957. marker = entry / "INSTALLATION_COMPLETE"
  958. if not marker.is_file():
  959. return f"INSTALLATION_COMPLETE missing under {native(entry)}; {_entry_listing(entry)}"
  960. binary = self.binary(entry, target)
  961. if binary is None:
  962. return f"Chromium executable missing under {native(entry)}"
  963. return self._binary_reason(binary, entry, target)
  964. def env(self, entry: Path, target: str) -> dict:
  965. binary = self.binary(entry, target)
  966. if binary is None:
  967. raise InstallError(self.name, f"Chromium executable missing under {native(entry)}")
  968. # Playwright launches without an executable_path (e.g. the
  969. # har-derived-api-client skill's `chromium.launch(channel="chromium")`)
  970. # find this build through the root; Hermes's own launcher asks
  971. # hermes_cli.browser_runtime.chromium_executable() instead.
  972. return {"PLAYWRIGHT_BROWSERS_PATH": str(entry.parent)}
  973. class LlamaCpp(BinaryPackage):
  974. """One llama.cpp backend build. Backends are dlopen'd plugins, so a
  975. usable engine is one archive per (target, backend) — plus, for CUDA, the
  976. cudart archive (end users have no CUDA toolkit), and on Linux the libgomp
  977. .deb. Every library lands beside llama-server: Windows resolves DLLs from
  978. the executable's directory, and the Linux builds' RUNPATH is $ORIGIN.
  979. Backend is a HARDWARE choice, not a target, so each backend is its own
  980. optional package and the runtime asks for the one this machine can
  981. use. Version is llama.cpp's rolling release tag without the `b`.
  982. """
  983. optional = True
  984. on_path = False
  985. binary_rel = {"win32": "llama-server.exe", "posix": "llama-server"}
  986. flatten = False
  987. # --version is llama-server's liveness proof AND the check that the
  988. # backend's shared libraries resolve: a CUDA build with no cudart
  989. # beside it fails here rather than at first chat.
  990. probe_cwd = True
  991. backend: str = ""
  992. # Release-asset infix per target, or absent where upstream ships none.
  993. assets: dict[str, str] = {}
  994. # Upstream's Linux builds (CPU included) link the system OpenMP runtime,
  995. # which minimal hosts (WSL, containers) lack, and a normal install never
  996. # touches the system package manager. The $ORIGIN RUNPATH loads this copy
  997. # ahead of the host's, so its glibc floor must stay at or below every
  998. # engine's. Ubuntu 22.04's needs glibc 2.34, the floor of the x64 CPU and
  999. # Vulkan builds, and provides every GOMP version the builds reference. The
  1000. # release-pocket file stays in the pool until 22.04's EOL; the artifact
  1001. # mirror serves the pinned bytes after that.
  1002. _LIBGOMP = {
  1003. "linux-x64": "https://archive.ubuntu.com/ubuntu/pool/main/g/gcc-12/libgomp1_12-20220319-1ubuntu1_amd64.deb",
  1004. "linux-arm64": "https://ports.ubuntu.com/ubuntu-ports/pool/main/g/gcc-12/libgomp1_12-20220319-1ubuntu1_arm64.deb",
  1005. }
  1006. @property
  1007. def gaps(self) -> dict[str, str]: # type: ignore[override]
  1008. return {
  1009. target: f"llama.cpp publishes no {self.backend} build for {target}"
  1010. for target in ALL_TARGETS
  1011. if target not in self.assets
  1012. }
  1013. def _asset_names(self, version: str, target: str) -> list[str]:
  1014. ext = "zip" if target.startswith("win32") else "tar.gz"
  1015. return [f"llama-b{version}-bin-{self.assets[target]}.{ext}"]
  1016. def fetch_urls(self, version: str, target: str) -> list[str]:
  1017. return [
  1018. f"https://github.com/ggml-org/llama.cpp/releases/download/b{version}/{asset}"
  1019. for asset in self._asset_names(version, target)
  1020. ] + ([self._LIBGOMP[target]] if target in self._LIBGOMP else [])
  1021. def fetch_url(self, version: str, target: str) -> str:
  1022. return self.fetch_urls(version, target)[0]
  1023. def latest_versions(self, target: str, locked=None) -> list[str]:
  1024. # Resolve from the llama.app installer bucket (the installer's own
  1025. # updater pointer + version index — no API token, no rate limit):
  1026. # the `latest` pointer is the authoritative "next version"; the
  1027. # bucket tree supplies the full candidate list. Artifacts still
  1028. # come from the llama.cpp GitHub releases (1:1 tag correspondence).
  1029. latest = llama_app_latest()
  1030. if latest is not None:
  1031. return [latest, *llama_app_bucket_versions()]
  1032. return llama_app_bucket_versions() or github_release_tags(
  1033. "ggml-org/llama.cpp", strip_prefix="b"
  1034. )
  1035. def known_sha256(self, version: str, url: str) -> Optional[str]:
  1036. """GitHub's release API serves every asset's digest, so pinning a
  1037. 280 MB engine costs one API call instead of the download."""
  1038. return _github_release_digests("ggml-org/llama.cpp", f"b{version}").get(
  1039. url.rsplit("/", 1)[-1]
  1040. )
  1041. def unpack(self, archive: Path, staged: Path, target: str) -> None:
  1042. if archive.name.endswith(".deb"):
  1043. unpack_deb(self.name, archive, staged)
  1044. else:
  1045. super().unpack(archive, staged, target)
  1046. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  1047. """Some archives nest the binaries under build/bin; hoist them so
  1048. binary_rel is one path for every target. The libgomp .deb unpacks
  1049. in its filesystem layout (usr/lib/<triplet>/); only the library
  1050. itself moves beside llama-server."""
  1051. server = self.binary(staged, target).name
  1052. if not (staged / server).is_file():
  1053. found = sorted(staged.rglob(server))
  1054. if not found:
  1055. raise InstallError(self.name, "archive contains no llama-server")
  1056. merge_tree(found[0].parent, staged)
  1057. # The pin decides whether the .deb arrives (test_llamacpp_pins); verify()'s --version
  1058. # probe is what proves the libraries resolve.
  1059. libs = sorted((staged / "usr" / "lib").glob("*/libgomp.so.1"))
  1060. if libs:
  1061. shutil.copyfile(libs[0], staged / "libgomp.so.1")
  1062. shutil.rmtree(staged / "usr")
  1063. def _github_release_digests(repo: str, tag: str) -> dict[str, str]:
  1064. from pm.update import _get_json
  1065. cached = _release_digest_cache.get((repo, tag))
  1066. if cached is not None:
  1067. return cached
  1068. url = f"https://api.github.com/repos/{repo}/releases/tags/{tag}"
  1069. try:
  1070. release = _get_json(url)
  1071. except Exception:
  1072. return {}
  1073. digests = {}
  1074. for asset in release.get("assets", []):
  1075. digest = (asset.get("digest") or "").partition("sha256:")[2]
  1076. if digest:
  1077. digests[asset["name"]] = digest
  1078. _release_digest_cache[(repo, tag)] = digests
  1079. return digests
  1080. _release_digest_cache: dict[tuple, dict] = {}
  1081. @register
  1082. class LlamaCppCuda(LlamaCpp):
  1083. """CUDA 13.4 everywhere upstream builds it: 13.x drivers run any 13.x
  1084. runtime (minor-version compatibility), and 13.4 is the only line upstream
  1085. ships for win-arm64. Linux archives first appeared after b10964, and
  1086. upstream dropped win-cuda-13.3 at the same time.
  1087. The engine archive carries no CUDA libraries (end users have no toolkit),
  1088. so each target pins a second cudart archive too."""
  1089. name = "llamacpp-cuda"
  1090. backend = "cuda"
  1091. assets = {
  1092. "win32-x64": "win-cuda-13.4-x64",
  1093. "win32-arm64": "win-cuda-13.4-arm64",
  1094. "linux-x64": "ubuntu-cuda-13.4-x64",
  1095. "linux-arm64": "ubuntu-cuda-13.4-arm64",
  1096. }
  1097. def _asset_names(self, version: str, target: str) -> list[str]:
  1098. # Upstream names the Windows cudart zip without the build tag and the
  1099. # Linux one with it.
  1100. infix = self.assets[target]
  1101. cudart = (
  1102. f"cudart-llama-bin-{infix}.zip"
  1103. if target.startswith("win32")
  1104. else f"cudart-llama-b{version}-bin-{infix}.tar.gz"
  1105. )
  1106. return super()._asset_names(version, target) + [cudart]
  1107. def stage(self, store: Store, staged: Path, version: str, target: str) -> None:
  1108. """The Linux cudart tarball unpacks into its own top-level dir, but the
  1109. engine's RUNPATH is $ORIGIN, so its libraries must sit beside
  1110. llama-server. The Windows zip is flat and needs no hoist."""
  1111. super().stage(store, staged, version, target)
  1112. for extra in sorted(staged.glob("cudart-*")):
  1113. if extra.is_dir():
  1114. merge_tree(extra, staged)
  1115. shutil.rmtree(extra)
  1116. @register
  1117. class LlamaCppVulkan(LlamaCpp):
  1118. name = "llamacpp-vulkan"
  1119. backend = "vulkan"
  1120. assets = {
  1121. "win32-x64": "win-vulkan-x64",
  1122. "linux-x64": "ubuntu-vulkan-x64",
  1123. "linux-arm64": "ubuntu-vulkan-arm64",
  1124. }
  1125. @register
  1126. class LlamaCppHip(LlamaCpp):
  1127. name = "llamacpp-hip"
  1128. backend = "hip"
  1129. # Upstream renamed the ROCm archives to 10.0 at b10767 (cff184438e).
  1130. assets = {
  1131. "win32-x64": "win-rocm-10.0-x64",
  1132. "linux-x64": "ubuntu-rocm-10.0-x64",
  1133. }
  1134. @register
  1135. class LlamaCppMetal(LlamaCpp):
  1136. """macOS archives are unified builds with Metal compiled in."""
  1137. name = "llamacpp-metal"
  1138. backend = "metal"
  1139. assets = {"darwin-x64": "macos-x64", "darwin-arm64": "macos-arm64"}
  1140. @register
  1141. class LlamaCppCpu(LlamaCpp):
  1142. name = "llamacpp-cpu"
  1143. backend = "cpu"
  1144. assets = {
  1145. "win32-x64": "win-cpu-x64",
  1146. "win32-arm64": "win-cpu-arm64",
  1147. "linux-x64": "ubuntu-x64",
  1148. "linux-arm64": "ubuntu-arm64",
  1149. "darwin-x64": "macos-x64",
  1150. "darwin-arm64": "macos-arm64",
  1151. }
  1152. @register
  1153. class WhisperCppCpu(BinaryPackage):
  1154. """Native local STT for Windows ARM64, where faster-whisper has no wheel."""
  1155. name = "whispercpp-cpu"
  1156. optional = True
  1157. gaps = {target: "uses the existing faster-whisper provider" for target in ALL_TARGETS
  1158. if target != "win32-arm64"}
  1159. binary_rel = {"win32-arm64": "whisper-cli.exe"}
  1160. probe_args = ["--help"]
  1161. url = "https://github.com/ggml-org/whisper.cpp/releases/download/{version}/whisper-bin-win-cpu-arm64.zip"