operations.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331
  1. """Python dependency operations executed by the isolated package manager.
  2. Callers describe a dependency graph and its lifetime, never a uv command.
  3. Only the private environment engine knows how to obtain or invoke uv.
  4. """
  5. from __future__ import annotations
  6. from collections.abc import Callable, Mapping, Sequence
  7. import hashlib
  8. import json
  9. import os
  10. from pathlib import Path
  11. import re
  12. import shutil
  13. import sys
  14. import uuid
  15. from pm.package import InstallError
  16. def _require_install_allowed(explicit: bool) -> None:
  17. from pm.install import _refuse_lazy, lazy_installs_allowed
  18. if not explicit and not lazy_installs_allowed():
  19. raise _refuse_lazy("venv", "Python dependency operation requires an explicit request")
  20. def build_environment(
  21. *, source: Path, out: Path, python: Path | None = None,
  22. cache: Path | None = None, env: Mapping[str, str] | None = None,
  23. extras: Sequence[str] = (), groups: Sequence[str] = (),
  24. all_extras: bool = False, no_install_project: bool = False,
  25. frozen: bool = True, sealed: bool = False, offline: bool = False,
  26. explicit: bool = False, timeout: int = 1800,
  27. ) -> Path:
  28. """Build and validate a fresh destination; never overwrite an existing tree.
  29. A build has no profile/plugin discovery or application selection side effects.
  30. Failure removes only the destination exclusively created by this invocation.
  31. Sealed builds prune only the .pth files that refer to build-time state.
  32. """
  33. from pm.environment import _fresh_build, managed_environment
  34. from pm.native_build import source_build_environment
  35. source, out = Path(source).absolute(), Path(out).absolute()
  36. if not (source / "pyproject.toml").is_file():
  37. raise InstallError("venv", f"project manifest is missing: {source}")
  38. if frozen and not (source / "uv.lock").is_file():
  39. raise InstallError("venv", f"frozen build requires a lock: {source / 'uv.lock'}")
  40. if out.exists() or out.is_symlink():
  41. raise FileExistsError(f"environment destination already exists: {out}")
  42. _require_install_allowed(explicit)
  43. # A caller-supplied env is already the build environment (bundle staging
  44. # prepares its own, shared with its Node builds).
  45. if env is None:
  46. env = source_build_environment(source)
  47. environment = managed_environment(
  48. out, python=Path(python) if python is not None else None,
  49. cache=Path(cache) if cache is not None else None, env=env,
  50. offline=offline, explicit=explicit, output=sys.stderr,
  51. )
  52. with _fresh_build(environment, sealed=sealed, lock=source / "uv.lock"):
  53. environment.sync(source, extras=extras, groups=groups, all_extras=all_extras,
  54. no_install_project=no_install_project, frozen=frozen, timeout=timeout)
  55. return environment.executable
  56. def lock_project(
  57. source: Path, *, upgrade: bool = False, python: Path | None = None,
  58. env: Mapping[str, str] | None = None, cache: Path | None = None,
  59. offline: bool = False, explicit: bool = False,
  60. ) -> None:
  61. """Refresh the project's resolution without creating or selecting a venv."""
  62. from pm.environment import managed_environment
  63. source = Path(source).absolute()
  64. if not (source / "pyproject.toml").is_file():
  65. raise InstallError("venv", f"project manifest is missing: {source}")
  66. _require_install_allowed(explicit)
  67. environment = managed_environment(
  68. source / ".venv", python=Path(python) if python is not None else None,
  69. cache=Path(cache) if cache is not None else None, env=env,
  70. offline=offline, explicit=explicit, output=sys.stderr,
  71. )
  72. environment.lock(source, upgrade=upgrade)
  73. def stage_manager_runtime(
  74. *, python: Path, destination: Path, project: Path | None = None,
  75. offline: bool = False, wheelhouse: Path | None = None, cache: Path | None = None,
  76. ) -> Path:
  77. """Stage PM's independent dependency graph using its private toolchain."""
  78. from pm._uv import _toolchain
  79. from pm.runtime_stage import stage_runtime
  80. destination, python = Path(destination).absolute(), Path(python).absolute()
  81. if destination.exists() or destination.is_symlink():
  82. raise FileExistsError(f"environment destination already exists: {destination}")
  83. # Tools are staged before this bootstrap: acquiring them here would need
  84. # the TLS dependencies in the very runtime we are constructing.
  85. tools = _toolchain(realize=False)
  86. if tools is None:
  87. raise InstallError("pm-runtime", "stage pinned tools before building the manager runtime")
  88. destination.mkdir(parents=True)
  89. try:
  90. executable = stage_runtime(tools[0], python, destination,
  91. project=Path(project) if project is not None else None,
  92. offline=offline, wheelhouse=Path(wheelhouse) if wheelhouse is not None else None,
  93. cache=Path(cache) if cache is not None else None)
  94. from pm.lock import _write
  95. _write(destination / "pm-runtime.json", {})
  96. return executable
  97. except BaseException:
  98. shutil.rmtree(destination, ignore_errors=True)
  99. raise
  100. def _environment_root(name: str, root: Path | None) -> Path:
  101. if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", name):
  102. raise ValueError("environment name must be a simple package name")
  103. if root is not None:
  104. return Path(root).absolute()
  105. from hermes_constants import get_hermes_home
  106. return get_hermes_home() / "environments" / name
  107. def _python(environment: Path) -> Path:
  108. from pm.environments import venv_python
  109. return venv_python(environment)
  110. def _selection(root: Path) -> dict:
  111. try:
  112. record = json.loads((root / "active.json").read_text(encoding="utf-8-sig"))
  113. except FileNotFoundError:
  114. return {}
  115. except (OSError, ValueError) as exc:
  116. raise InstallError("venv", f"cannot read environment selection: {root}") from exc
  117. if (not isinstance(record, dict) or not isinstance(record.get("generation"), str)
  118. or not re.fullmatch(r"gen-[0-9a-f]{32}", record["generation"])):
  119. raise InstallError("venv", f"invalid environment selection: {root}")
  120. return record
  121. def environment_python(name: str, *, root: Path | None = None) -> Path | None:
  122. """Read the selected interpreter without acquiring tools or writing state."""
  123. root = _environment_root(name, root)
  124. selected = _selection(root)
  125. if not selected:
  126. return None
  127. environment = root / selected["generation"] / "venv"
  128. if not environment.resolve().is_relative_to(root.resolve()):
  129. raise InstallError(name, "selected environment escapes its root")
  130. python = _python(environment)
  131. return python if python.is_file() and (environment / "pyvenv.cfg").is_file() else None
  132. def _tool(python: Path, executable: str) -> Path | None:
  133. if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", executable):
  134. raise ValueError("tool executable must be a filename, not a path")
  135. directory = python.parent
  136. suffixes = (".exe", ".cmd", "") if os.name == "nt" else ("",)
  137. return next((directory / (executable + suffix) for suffix in suffixes
  138. if (directory / (executable + suffix)).is_file()), None)
  139. def python_tool(name: str, executable: str, *, root: Path | None = None) -> Path | None:
  140. """Return a selected tool entrypoint, never an installer executable."""
  141. python = environment_python(name, root=root)
  142. return _tool(python, executable) if python is not None else None
  143. def _requirements(requirements: Sequence[str]) -> list[str]:
  144. from packaging.requirements import InvalidRequirement, Requirement
  145. if isinstance(requirements, str) or not requirements:
  146. raise ValueError("an isolated environment needs a nonempty sequence of requirements")
  147. try:
  148. return sorted({str(Requirement(requirement)) for requirement in requirements})
  149. except (InvalidRequirement, TypeError) as exc:
  150. raise ValueError(f"invalid environment requirement: {exc}") from exc
  151. def ensure_environment(
  152. name: str, requirements: Sequence[str], *, root: Path | None = None,
  153. explicit: bool = False, timeout: int = 1800,
  154. executable: str | None = None,
  155. ) -> Path:
  156. """Make an isolated dependency set current, then atomically select it.
  157. An optional tool entrypoint is validated before publication, not afterwards.
  158. """
  159. root = _environment_root(name, root)
  160. requirements = _requirements(requirements)
  161. if executable is not None:
  162. _tool(Path("unused/python"), executable) # Validate before any write.
  163. def build(generation: Path, base_python: Path) -> Path:
  164. (generation / "pyproject.toml").write_text(
  165. '[project]\nname = "hermes-side-environment"\nversion = "0"\n'
  166. 'requires-python = ">=3.11"\ndependencies = '
  167. + json.dumps(requirements) + '\n[tool.uv]\npackage = false\n', encoding="utf-8",
  168. )
  169. previous = _selection(root)
  170. if previous:
  171. seed = root / previous["generation"] / "uv.lock"
  172. if seed.is_file():
  173. shutil.copyfile(seed, generation / "uv.lock")
  174. return build_environment(source=generation, out=generation / "venv", python=base_python,
  175. frozen=False, explicit=explicit, timeout=timeout)
  176. return _ensure_generation(name, root, {"requirements": requirements}, build,
  177. record={"requirements": requirements}, explicit=explicit,
  178. executable=executable)
  179. def ensure_project_environment(
  180. name: str, project: Path, *, extras: Sequence[str] = (), groups: Sequence[str] = (),
  181. root: Path | None = None, explicit: bool = False, timeout: int = 1800,
  182. ) -> Path:
  183. """Make an isolated environment of a project's LOCKED dependencies current.
  184. The project itself is not installed, and nothing reaches PM facts or the
  185. selected application generation: this serves side environments such as the
  186. test suite's. The identity covers the lock and manifest bytes, so any edit
  187. that can change the resolved set selects a fresh generation.
  188. """
  189. root = _environment_root(name, root)
  190. project = Path(project).absolute()
  191. extras, groups = sorted(set(extras)), sorted(set(groups))
  192. manifests = {}
  193. for manifest in ("pyproject.toml", "uv.lock"):
  194. try:
  195. manifests[manifest] = hashlib.sha256((project / manifest).read_bytes()).hexdigest()
  196. except FileNotFoundError as exc:
  197. raise InstallError("venv", f"locked project environment needs {project / manifest}") from exc
  198. def build(generation: Path, base_python: Path) -> Path:
  199. return build_environment(source=project, out=generation / "venv", python=base_python,
  200. extras=extras, groups=groups, no_install_project=True,
  201. frozen=True, explicit=explicit, timeout=timeout)
  202. return _ensure_generation(name, root, {"manifests": manifests, "extras": extras, "groups": groups},
  203. build, record={"extras": extras, "groups": groups}, explicit=explicit)
  204. def _ensure_generation(
  205. name: str, root: Path, inputs: dict, build: Callable[[Path, Path], Path], *, record: dict, explicit: bool,
  206. executable: str | None = None,
  207. ) -> Path:
  208. """Select the generation whose inputs match, building one only when none does.
  209. Build at the final path: Windows launchers and scripts embed that path.
  210. The prior generation survives both successful replacement and failed builds.
  211. """
  212. from pm.filesystem import lock_fd, native
  213. from pm._uv import _toolchain
  214. from pm.install import _refuse_lazy, lazy_installs_allowed
  215. from pm.lock import Lockfile, _write
  216. from pm import paths
  217. from pm.store import current_target
  218. lock = Lockfile(paths.lockfile_path())
  219. target = current_target()
  220. inputs = {**inputs, "python": lock.version("python"), "target": target,
  221. "artifacts": [item["sha256"] for item in lock.artifacts("python", target)]}
  222. def selected_python() -> Path | None:
  223. tools = _toolchain(realize=False)
  224. return tools[1].resolve() if tools is not None else None
  225. def identity(base_python: Path) -> str:
  226. # The same pinned artifact can live in different stores. A venv's
  227. # pyvenv.cfg keeps the original interpreter path, not just its version.
  228. # The store spells its root extended-length on Windows; the identity must not.
  229. return hashlib.sha256(json.dumps({**inputs, "interpreter": native(base_python)},
  230. sort_keys=True).encode()).hexdigest()
  231. def current(base_python: Path | None) -> Path | None:
  232. if base_python is None:
  233. return None
  234. selected = _selection(root)
  235. python = environment_python(name, root=root)
  236. if (selected.get("inputs") == identity(base_python) and python is not None
  237. and (executable is None or _tool(python, executable) is not None)):
  238. return python
  239. return None
  240. existing = current(selected_python())
  241. if existing is not None:
  242. return existing
  243. if not explicit and not lazy_installs_allowed():
  244. raise _refuse_lazy(name, "isolated Python environment is missing or outdated")
  245. root.mkdir(parents=True, exist_ok=True)
  246. with (root / ".install.lock").open("a+b") as mutex:
  247. lock_fd(mutex.fileno(), wait=True)
  248. base_python = selected_python()
  249. existing = current(base_python)
  250. if existing is not None:
  251. return existing
  252. if base_python is None:
  253. tools = _toolchain(explicit=explicit)
  254. if tools is None:
  255. raise InstallError(name, "PM's pinned toolchain is unavailable")
  256. base_python = tools[1].resolve()
  257. generation = root / f"gen-{uuid.uuid4().hex}"
  258. generation.mkdir()
  259. try:
  260. python = build(generation, base_python)
  261. if executable is not None and _tool(python, executable) is None:
  262. raise InstallError(name, f"installed requirements do not provide {executable!r}")
  263. from pm.environments import flush_before_selecting
  264. flush_before_selecting()
  265. _write(root / "active.json", {"generation": generation.name,
  266. "inputs": identity(base_python), **record})
  267. except BaseException:
  268. shutil.rmtree(generation, ignore_errors=True)
  269. raise
  270. return python
  271. def ensure_python_tool(
  272. name: str, requirements: Sequence[str], executable: str, *,
  273. root: Path | None = None, explicit: bool = False, timeout: int = 1800,
  274. ) -> Path:
  275. python = ensure_environment(name, requirements, root=root, explicit=explicit,
  276. timeout=timeout, executable=executable)
  277. tool = _tool(python, executable)
  278. assert tool is not None # Validated under the publication lock above.
  279. return tool