binaries.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. """Hardware selection for PM-owned llama.cpp binaries; mutable runtime state stays separate."""
  2. from __future__ import annotations
  3. import json
  4. import logging
  5. import os
  6. import re
  7. import threading
  8. from contextlib import contextmanager, suppress
  9. from dataclasses import dataclass
  10. from pathlib import Path
  11. from typing import Callable, Iterator
  12. import pm
  13. from pm.downloader import ProgressFn
  14. logger = logging.getLogger(__name__)
  15. BACKEND_PACKAGES = {
  16. "cuda": "llamacpp-cuda",
  17. "vulkan": "llamacpp-vulkan",
  18. "metal": "llamacpp-metal",
  19. "hip": "llamacpp-hip",
  20. "cpu": "llamacpp-cpu",
  21. }
  22. class BinaryResolutionError(RuntimeError):
  23. """The requested backend has no pinned or installed engine."""
  24. @dataclass(frozen=True)
  25. class Engine:
  26. backend: str
  27. tag: str
  28. binary: Path
  29. def runtimes_root() -> Path:
  30. """Machine-scoped presets and server state. Binaries belong to PM's store."""
  31. from hermes_constants import get_default_hermes_root
  32. return get_default_hermes_root() / "runtimes" / "llamacpp"
  33. def pinned_tag(backend: str) -> str:
  34. version = pm.Lockfile(pm.paths.lockfile_path()).version(BACKEND_PACKAGES[backend])
  35. if version is None:
  36. raise BinaryResolutionError(f"llama.cpp {backend} has no PM version pin")
  37. return f"b{version}"
  38. def select_backend(gpu_vendor: str | None, os_name: str | None = None) -> str:
  39. if os_name is None:
  40. os_name = "macos" if pm.current_target().startswith("darwin-") else "other"
  41. if os_name == "macos":
  42. return "metal"
  43. vendor = (gpu_vendor or "").lower()
  44. if "nvidia" in vendor:
  45. return "cuda"
  46. if any(name in vendor for name in ("amd", "intel", "radeon", "arc")):
  47. return "vulkan"
  48. return "cpu"
  49. def _candidates(requested: str, gpu_vendor: str | None, target: str) -> tuple[str, ...]:
  50. if requested != "auto":
  51. if requested not in BACKEND_PACKAGES:
  52. raise BinaryResolutionError(f"unknown backend {requested}")
  53. return (requested,)
  54. preferred = select_backend(gpu_vendor, "macos" if target.startswith("darwin-") else "other")
  55. fallback = ("vulkan", "cpu") if gpu_vendor else ("cpu",)
  56. return tuple(dict.fromkeys((preferred, *fallback)))
  57. def unavailable_reason(backend: str, target: str | None = None) -> str | None:
  58. name = BACKEND_PACKAGES.get(backend)
  59. if name is None:
  60. return f"unknown backend {backend}"
  61. target = target or pm.current_target()
  62. reason = pm.get_package(name).missing_reason(target)
  63. if reason:
  64. return reason
  65. lock = pm.Lockfile(pm.paths.lockfile_path())
  66. if not lock.version(name) or not lock.artifacts(name, target):
  67. return f"{name} is not pinned for {target}"
  68. return None
  69. def resolve_backend(requested: str = "auto", *, gpu_vendor: str | None = None,
  70. target: str | None = None) -> str:
  71. """Explicit choices are strict. Auto only falls back to compatible pinned builds."""
  72. if requested == "auto" and target is None and gpu_vendor is None:
  73. from hermes_cli.local_runtime.bootstrap import _detect_gpu_vendor
  74. gpu_vendor = _detect_gpu_vendor()
  75. target = target or pm.current_target()
  76. reasons = []
  77. for backend in _candidates(requested, gpu_vendor, target):
  78. reason = unavailable_reason(backend, target)
  79. if reason is None:
  80. return backend
  81. reasons.append(reason)
  82. raise BinaryResolutionError("; ".join(reasons))
  83. def _installed(candidates: tuple[str, ...], allow_outdated: bool) -> Engine | None:
  84. for candidate in candidates:
  85. found = pm.installed_package(BACKEND_PACKAGES[candidate], allow_outdated=allow_outdated)
  86. if found is not None and found.binary is not None:
  87. return Engine(candidate, f"b{found.version}", found.binary)
  88. return None
  89. def installed_engine(backend: str = "auto", *, allow_outdated: bool = True) -> Engine | None:
  90. """Boot may retain a prior PM pin and moves a pre-PM engine into PM's store once; it never downloads."""
  91. vendor = None
  92. if backend == "auto":
  93. from hermes_cli.local_runtime.bootstrap import _detect_gpu_vendor
  94. vendor = _detect_gpu_vendor()
  95. candidates = _candidates(backend, vendor, pm.current_target())
  96. engine = _installed(candidates, allow_outdated)
  97. if engine is None and any(adopt_legacy_engine(candidate) for candidate in candidates):
  98. engine = _installed(candidates, allow_outdated)
  99. return engine
  100. # Before PM owned binaries, Hermes installed each engine to runtimes/llamacpp/b<tag>/<backend>/
  101. # and wrote a manifest.json with the archive digests and the llama-server --version it saw.
  102. _SHA256 = re.compile(r"[0-9a-f]{64}")
  103. _ADOPTION_LOCK = threading.Lock()
  104. # Installs that failed to move stay put for this process; the pane polls status every few seconds.
  105. _LEFT_IN_PLACE: set[Path] = set()
  106. def _legacy_installs(backend: str) -> list[tuple[str, Path, dict]]:
  107. """Verified pre-PM installs of ``backend`` as (version, dir, manifest), newest first."""
  108. found = []
  109. for manifest_path in runtimes_root().glob(f"b*/{backend}/manifest.json"):
  110. tag = manifest_path.parent.parent.name
  111. try:
  112. number = int(tag.removeprefix("b"))
  113. manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
  114. except (OSError, ValueError):
  115. continue
  116. if (isinstance(manifest, dict) and manifest.get("tag") == tag and manifest.get("backend") == backend
  117. and manifest.get("verified_version") and isinstance(manifest.get("assets"), dict)
  118. and manifest_path.parent not in _LEFT_IN_PLACE):
  119. found.append((number, str(number), manifest_path.parent, manifest))
  120. return [(version, path, manifest) for _, version, path, manifest in sorted(found, reverse=True)]
  121. def _legacy_artifacts(package, version: str, target: str, assets: dict) -> list[str] | None:
  122. """The manifest's archive digests in PM's archive order, or None when one is missing.
  123. The pre-PM installer fetched only llama.cpp's own release assets. On Linux PM also pins the
  124. libgomp .deb, so an adopted engine there records fewer archives than the lock and counts as
  125. outdated: usable now (its manifest proves it ran against the host's libgomp), update offered."""
  126. shas = [assets.get(name) for name in package._asset_names(version, target)]
  127. if not shas or not all(isinstance(sha, str) and _SHA256.fullmatch(sha) for sha in shas):
  128. return None
  129. return shas
  130. @contextmanager
  131. def _store_lock(root: Path) -> Iterator[bool]:
  132. """PM's store lock, or False when another PM operation holds it (a download can take minutes)."""
  133. from pm.filesystem import lock_fd
  134. root.mkdir(parents=True, exist_ok=True)
  135. fd = os.open(root / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600)
  136. try:
  137. yield lock_fd(fd, wait=True, timeout=2)
  138. finally:
  139. os.close(fd)
  140. def adopt_legacy_engine(backend: str) -> bool:
  141. """Move the newest pre-PM install of ``backend`` into PM's store and record it as installed.
  142. A machine that already ran a Hermes-installed engine keeps it across the update to PM. The
  143. manifest's archive digests become the PM identity, so a tag that matches the pin counts as
  144. current and an older tag counts as outdated, which offers the update. ``os.rename`` only:
  145. instant on one volume, and a store on another volume leaves the engine where it is. Never
  146. raises; returns whether an engine was moved.
  147. """
  148. try:
  149. candidates = _legacy_installs(backend)
  150. if not candidates:
  151. return False
  152. from pm import paths as pm_paths
  153. name = BACKEND_PACKAGES[backend]
  154. package = pm.get_package(name)
  155. target = pm.current_target()
  156. root = pm_paths.writable_store_root()
  157. facts_path = pm_paths.facts_path() if root == pm_paths.store_root() else root / "facts.json"
  158. with _ADOPTION_LOCK, _store_lock(root) as held:
  159. if not held or pm.installed_package(name, allow_outdated=True) is not None:
  160. return False
  161. for version, source, manifest in candidates:
  162. if _adopt(package, version, target, source, manifest, root, facts_path):
  163. return True
  164. _LEFT_IN_PLACE.add(source)
  165. except Exception as exc:
  166. logger.warning("could not move the pre-PM llama.cpp %s engine into the PM store: %s", backend, exc)
  167. return False
  168. def _adopt(package, version: str, target: str, source: Path, manifest: dict, root: Path,
  169. facts_path: Path) -> bool:
  170. from pm.store import tree_digest
  171. artifacts = _legacy_artifacts(package, version, target, manifest["assets"])
  172. entry_name = package.store_entry(version, target)
  173. entry = root / entry_name
  174. if artifacts is None or entry.exists() or entry.is_symlink():
  175. return False
  176. reason = package.verify(source, target)
  177. if reason:
  178. logger.warning("pre-PM llama.cpp at %s left in place: %s", source, reason)
  179. return False
  180. try:
  181. os.rename(source, entry)
  182. except OSError as exc:
  183. logger.warning("pre-PM llama.cpp at %s left in place: %s", source, exc)
  184. return False
  185. try:
  186. pm.Facts(facts_path).record(package.name, version, entry_name, package.env(entry, target), root,
  187. target=target, artifacts=artifacts, digest=tree_digest(entry))
  188. except BaseException:
  189. with suppress(OSError):
  190. os.rename(entry, source)
  191. raise
  192. with suppress(OSError):
  193. source.parent.rmdir()
  194. logger.info("moved llama.cpp b%s (%s) from %s into the PM store", version, package.name, source)
  195. return True
  196. def ensure_engine(backend: str, *, progress: Callable[[str, int, int, str], None] | None = None,
  197. pause_event: threading.Event | None = None,
  198. download_progress: ProgressFn | None = None) -> Engine:
  199. """Only deliberate install/update jobs call this. PM verifies every archive and publishes."""
  200. resolved = resolve_backend(backend)
  201. pm.ensure(BACKEND_PACKAGES[resolved], explicit=True, progress=progress,
  202. pause_event=pause_event, download_progress=download_progress)
  203. engine = installed_engine(resolved, allow_outdated=False)
  204. if engine is None:
  205. raise BinaryResolutionError(f"llama.cpp {resolved} install has no usable pinned binary")
  206. return engine