build_operations.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. """Build/CI operations with caller-owned inputs, independent of live selection."""
  2. from __future__ import annotations
  3. from collections.abc import Mapping, Sequence
  4. from dataclasses import dataclass
  5. from pathlib import Path
  6. import sys
  7. import time
  8. from types import MappingProxyType
  9. from pm.filesystem import long_root, native
  10. from pm.install import InstalledPackage
  11. from pm.lock import Lockfile
  12. from pm.package import InstallError
  13. def check_project_lock(
  14. source: Path, *, python: Path | None = None, cache: Path | None = None,
  15. env: Mapping[str, str] | None = None, offline: bool = False, explicit: bool = False,
  16. quiet: bool = False,
  17. ) -> None:
  18. """Reject a missing or stale lock without rewriting source or creating a venv.
  19. ``quiet`` captures uv's output instead of reporting it: for a caller that
  20. expects a stale lock and says so itself, the red failure tail is noise.
  21. The error still carries uv's message.
  22. """
  23. from pm.environment import managed_environment
  24. from pm.operations import _require_install_allowed
  25. if not offline:
  26. _require_install_allowed(explicit)
  27. source = Path(source).absolute()
  28. if not (source / "pyproject.toml").is_file():
  29. raise InstallError("venv", f"project manifest is missing: {source}")
  30. environment = managed_environment(
  31. source / ".venv", python=Path(python) if python is not None else None,
  32. cache=Path(cache) if cache is not None else None, env=env,
  33. offline=offline, explicit=explicit, output=None if quiet else sys.stderr,
  34. )
  35. environment.check_lock(source)
  36. def export_requirements(
  37. source: Path, out: Path, *, extras: Sequence[str] = (), python: Path | None = None,
  38. cache: Path | None = None, env: Mapping[str, str] | None = None, explicit: bool = False,
  39. ) -> None:
  40. """Export locked runtime requirements, preserving markers and direct URL pins."""
  41. from pm.environment import managed_environment
  42. from pm.operations import _require_install_allowed
  43. _require_install_allowed(explicit)
  44. source, out = Path(source).absolute(), Path(out).absolute()
  45. if not (source / "pyproject.toml").is_file():
  46. raise InstallError("venv", f"project manifest is missing: {source}")
  47. if not (source / "uv.lock").is_file():
  48. raise InstallError("venv", f"frozen export requires a lock: {source / 'uv.lock'}")
  49. environment = managed_environment(
  50. source / ".venv", python=Path(python) if python is not None else None,
  51. cache=Path(cache) if cache is not None else None, env=env,
  52. explicit=explicit, output=sys.stderr,
  53. )
  54. out.parent.mkdir(parents=True, exist_ok=True)
  55. environment.export_requirements(source, out, extras=extras)
  56. def build_requirements_environment(
  57. requirements: Sequence[str], *, out: Path, python: Path | None = None,
  58. cache: Path | None = None, env: Mapping[str, str] | None = None,
  59. wheelhouse: Path | None = None, offline: bool = False, sealed: bool = False,
  60. explicit: bool = False, lock: Path | None = None,
  61. ) -> Path:
  62. """Create and check a fresh environment. Never mutate an existing destination.
  63. Wheelhouse builds disable indexes and source builds. Failure removes only
  64. this invocation's exclusively claimed output, not prior builds. *lock* is the
  65. uv.lock the requirements were exported from: its overrides apply to the
  66. install and the closing check.
  67. """
  68. from pm.environment import _fresh_build, managed_environment
  69. from pm.operations import _require_install_allowed, _requirements
  70. _require_install_allowed(explicit)
  71. requirements = _requirements(requirements) if requirements or isinstance(requirements, str) else []
  72. out = Path(out).absolute()
  73. lock = Path(lock).absolute() if lock is not None else None
  74. wheelhouse = Path(wheelhouse).absolute() if wheelhouse is not None else None
  75. if wheelhouse is not None and not wheelhouse.is_dir():
  76. raise InstallError("venv", f"wheelhouse directory is missing: {wheelhouse}")
  77. if out.exists() or out.is_symlink():
  78. raise FileExistsError(f"environment destination already exists: {out}")
  79. environment = managed_environment(
  80. out, python=Path(python) if python is not None else None,
  81. cache=Path(cache) if cache is not None else None, env=env,
  82. offline=offline, explicit=explicit, output=sys.stderr,
  83. )
  84. with _fresh_build(environment, sealed=sealed, lock=lock):
  85. environment.install_requirements(requirements, wheelhouse=wheelhouse, lock=lock)
  86. return environment.executable
  87. def prepare_tools(names: Sequence[str], *, out: Path, target: str,
  88. cache: Path | None = None) -> Path:
  89. """Realize an explicit tool closure in a build-owned store, not live state."""
  90. from pm.install import _install, _lockfile
  91. from pm.lock import Facts
  92. from pm.package import StatePackage
  93. from pm.registry import walk
  94. from pm.store import Store
  95. if isinstance(names, str):
  96. raise TypeError("names must be a sequence, not a string")
  97. packages = walk(list(names))
  98. if any(isinstance(package, StatePackage) for package in packages):
  99. raise ValueError("prepare_tools accepts tools, not application state")
  100. out = Path(out).resolve()
  101. store, lock = Store(out), _lockfile()
  102. with store.install_lock():
  103. facts = Facts(out / "facts.json", strict=True)
  104. for package in packages:
  105. _install(package, lock, facts, store, target, _lock_held=True)
  106. return out
  107. def _copy_links(package, entry: Path) -> None:
  108. import os
  109. from pm.filesystem import is_junction
  110. for directory, dirs, files in os.walk(entry):
  111. for name in dirs + files:
  112. path = Path(directory) / name
  113. if path.is_symlink() or is_junction(path):
  114. if (Path(os.readlink(path)).is_absolute() or not path.exists()
  115. or not path.resolve().is_relative_to(entry)):
  116. raise InstallError(package.name, f"tool copy link escapes entry: {native(path)}")
  117. @dataclass(frozen=True)
  118. class VerifiedTools:
  119. entries: Mapping[str, InstalledPackage]
  120. _envs: tuple[dict, ...]
  121. def environment(self, base: Mapping[str, str]) -> dict[str, str]:
  122. from pm.package import compose_env
  123. return compose_env(list(self._envs), base=dict(base))
  124. def verified_tools(names: Sequence[str], *, source_store: Path, target: str,
  125. lock: Lockfile | None = None) -> VerifiedTools:
  126. """Read a pinned tool closure without writes, acquisition or binary execution.
  127. Publication already ran package verification. Admission binds its recorded
  128. digest to the canonical entry, lock and declared environment; it does not
  129. rerun arbitrary version probes just to read a previously published tool.
  130. """
  131. from pm.install import _identity, _lockfile
  132. from pm.lock import Facts
  133. from pm.package import StatePackage
  134. from pm.registry import walk
  135. from pm.store import digest_matches
  136. if isinstance(names, str):
  137. raise TypeError("names must be a sequence, not a string")
  138. packages = walk(list(names))
  139. if any(isinstance(package, StatePackage) for package in packages):
  140. raise ValueError("verified_tools accepts tool packages, not application state")
  141. source_store = Path(source_store).resolve()
  142. if not (source_store / "facts.json").is_file():
  143. raise InstallError("tools", f"source facts missing: {source_store}")
  144. # Admission walks and digests whole tool trees: spell the root like Store does.
  145. source_store = long_root(source_store)
  146. facts = Facts(source_store / "facts.json", strict=True)
  147. lock = lock if lock is not None else _lockfile()
  148. entries, envs = {}, []
  149. for package in packages:
  150. if getattr(package, "pin_only", False):
  151. continue
  152. version = lock.version(package.name)
  153. identity = _identity(lock, package.name, target)
  154. if not version or identity is None:
  155. raise InstallError(package.name, "tool source failed verification: missing pin")
  156. fact = facts.get(package.name)
  157. expected = package.store_entry(version, target)
  158. entry = source_store / expected
  159. if (not fact or fact.get("entry") != expected
  160. or not entry.is_dir() or not entry.resolve().is_relative_to(source_store)
  161. or not facts.installed(package.name, version, source_store, identity)
  162. or not digest_matches(entry, fact.get("digest"))):
  163. raise InstallError(package.name, "tool source failed verification", "run preparation again")
  164. _copy_links(package, entry)
  165. binary = package.binary(entry, target)
  166. env = package.env(entry, target)
  167. if ((binary is not None and (not binary.is_file() or not binary.resolve().is_relative_to(entry)))
  168. or facts.env_for(package.name, source_store) != env):
  169. raise InstallError(package.name, "tool source failed verification: binary or environment")
  170. # The selection leaves PM (bundle scripts compare and execute it): ordinary spelling.
  171. entries[package.name] = InstalledPackage(Path(native(entry)), version,
  172. Path(native(binary)) if binary is not None else None)
  173. envs.append(env)
  174. return VerifiedTools(MappingProxyType(entries), tuple(envs))
  175. def stage_tools(names: Sequence[str], *, source_store: Path, out: Path, target: str) -> Path:
  176. """Copy a current, verified tool closure without acquisition or live selection.
  177. Stores must be disjoint. Both publication locks cover admission and copying;
  178. the ordinary installer owns verification, independent copies and fresh facts.
  179. """
  180. from contextlib import ExitStack
  181. from pm.install import _entry_verified, _install, _lockfile
  182. from pm.lock import Facts
  183. from pm.package import StatePackage
  184. from pm.registry import walk
  185. from pm.store import Store
  186. if isinstance(names, str):
  187. raise TypeError("names must be a sequence, not a string")
  188. source_store, out = Path(source_store).resolve(), Path(out).resolve()
  189. if source_store.is_relative_to(out) or out.is_relative_to(source_store):
  190. raise ValueError("tool stores must be disjoint")
  191. if not (source_store / "facts.json").is_file():
  192. raise InstallError("tools", f"source facts missing: {source_store}")
  193. packages = walk(list(names))
  194. if any(isinstance(package, StatePackage) for package in packages):
  195. raise ValueError("stage_tools accepts tool packages, not application state")
  196. source, destination = Store(source_store), Store(out)
  197. lock = _lockfile()
  198. with ExitStack() as stack:
  199. for store in sorted((source, destination), key=lambda store: str(store.root)):
  200. stack.enter_context(store.install_lock())
  201. selection = verified_tools(names, source_store=source_store, target=target, lock=lock)
  202. facts = Facts(source_store / "facts.json", strict=True)
  203. for package in packages:
  204. if package.name not in selection.entries:
  205. continue
  206. fact = facts.get(package.name)
  207. assert fact is not None # Admission requires the selected fact under the same store lock.
  208. if not _entry_verified(package, fact, source, target):
  209. raise InstallError(package.name, "tool copy source failed verification", "run preparation again")
  210. staged = Facts(out / "facts.json", strict=True)
  211. for package in packages:
  212. started = time.monotonic()
  213. _install(package, lock, staged, destination, target,
  214. copy_from=(facts, source), _lock_held=True, _fresh_copy=True)
  215. # Copy plus re-digest of a large tree runs silently for minutes on Windows.
  216. print(f" copied {package.name} ({time.monotonic() - started:.1f}s)", flush=True)
  217. return out
  218. def prune_cache(cache: Path, *, ci: bool = False) -> None:
  219. """Prune unused cache entries; CI mode also discards downloaded wheels."""
  220. from pm.environment import managed_environment
  221. cache = Path(cache).absolute()
  222. environment = managed_environment(cache, cache=cache, realize=False, output=sys.stderr)
  223. environment.prune_cache(ci=ci)