"""PM's private Python engine: tool routing, isolation and command execution. Operations own source, destination and publication. Bootstrap may inject its staged toolchain directly without recursing through the worker it is building. """ from __future__ import annotations import codecs from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass, replace import io import os from pathlib import Path import shutil import subprocess import sys import tempfile import time from typing import TextIO from pm.filesystem import native from pm.package import InstallError from pm.progress import LiveTail, TextSink, verbose_output # Slow steps get a status line; unlisted quick ones (venv, export, pip check) # stay silent unless they fail. _UV_LABELS: dict[object, str] = { "sync": "Installing Python dependencies", "lock": "Resolving Python dependencies", ("pip", "install"): "Installing Python packages", "cache": "Pruning the uv cache", } # Deliberately narrow: a fetch timeout or index outage must not be misread as a # conflict — and regardless of classification, nothing here ever disables a # plugin; the caller decides. Lives here (stdlib-only imports) because the # bootstrap runner streams uv output from a pre-3.11 system python where # pm.workspace's tomllib import cannot load. _RESOLVER_MARKERS = ( "no solution found", "conflicting requirements", "conflicting urls", "because only the following versions", "and your pyproject depends on", ) # A package's own build ran and failed; a fetch/download failure never prints this. _BUILD_MARKERS = ("the build backend returned an error",) # The index omitted upload times (no PEP 700 metadata), so the release-age cutoff # cannot date the file. uv words it as "no solution", but the index is at fault, # not the requested set, so it must never read as a dependency conflict (#135589). # A seeded lock says "has no publish time"; a cold one only warns the date is missing # and hints the version was filtered, so that case needs both lines. _UNDATED_PIN = "has no publish time" _UNDATED_FILE = ("is missing an upload date", "was filtered by `exclude-newer") UNDATED_INDEX_HINT = ("the package index did not report upload times, which locking Hermes's " "dependencies needs for its 3-day release-age check. Use https://pypi.org/simple " "or a mirror that serves upload times (pip.conf index-url / UV_INDEX_URL), then retry.") # uv's own English diagnostic. A localized Windows tail (os error 1224) rides # along, but the marker that means "packages may be installed, only the # optional compile step failed" is this line. #124268 _BYTECODE_COMPILE_MARKER = "failed to bytecode-compile" # Streamed uv output keeps only the last 2000 characters. Resolver markers are # stitched back if a later verbose flood evicts them; the bytecode marker has # to ride the same path or repair/bootstrap never sees the retry signal, and so # does the build marker: a failed native build's traceback runs far past 2000 # characters, and without it the failure reads as a fetch error, not a verdict. _STREAM_TAIL_MARKERS = _RESOLVER_MARKERS + (_BYTECODE_COMPILE_MARKER, *_BUILD_MARKERS) # Retries after purging corrupt cached packages. Each round must name a package # not purged before, so this bounds a cache with several damaged entries, never # a loop on one broken wheel. _CACHE_RECOVERY_ROUNDS = 3 class ResolutionConflict(InstallError): """uv's resolver proved the union has no valid solution.""" class BuildFailure(InstallError): """A package's build backend ran and failed.""" def _is_bytecode_compile_failure(result: subprocess.CompletedProcess) -> bool: """True when uv failed in the optional bytecode-compile step, not the install. ``--compile-bytecode`` exists so the first user-facing import does not pay compile cost (#100461). It is not required for a usable environment. Windows real-time AV can lock uv's temporary compile script (os error 1224) after the packages are already on disk (#124268). """ if not result.returncode: return False text = f"{result.stderr or ''}\n{result.stdout or ''}".lower() return _BYTECODE_COMPILE_MARKER in text def classify_uv_failure(stage: str, returncode: int, output: str) -> InstallError: """Turn a failed `uv ` into the right classified error. Resolver-conflict output → ResolutionConflict; a build backend that ran and failed → BuildFailure; anything else (fetch, tooling) → plain InstallError with the tail of the output. """ cause = f"uv {stage} exited {returncode}: {output.strip()[-600:]}" lowered = output.lower() if _UNDATED_PIN in lowered or all(marker in lowered for marker in _UNDATED_FILE): return InstallError("venv", cause, UNDATED_INDEX_HINT) if any(marker in lowered for marker in _RESOLVER_MARKERS): return ResolutionConflict("venv", cause) if any(marker in lowered for marker in _BUILD_MARKERS): from pm.build_compilers import missing_compiler_remedy return BuildFailure("venv", cause, missing_compiler_remedy(output)) return InstallError("venv", cause) def _project_name(source: Path) -> str: """``[project].name`` of *source*'s pyproject, on any Python that can run the bootstrap. tomllib is 3.11+; the bootstrap runner stages PM's runtime from a system python that may be older, so the one field this module reads is parsed with the stdlib parser when it exists and a table-scoped regex otherwise. """ text = (source / "pyproject.toml").read_text(encoding="utf-8-sig") try: import tomllib except ModuleNotFoundError: import re table = re.search(r"(?ms)^\[project\]\s*$(.*?)(?=^\[)", text + "\n[") match = table and re.search(r'(?m)^name\s*=\s*"([^"]+)"', table.group(1)) if not match: raise InstallError("venv", f"{source / 'pyproject.toml'} has no [project].name") return match.group(1) return tomllib.loads(text)["project"]["name"] def prune_site_pth(venv_dir: Path) -> None: """Drop .pth files that must never execute inside a shipped payload. ``uv sync`` leaves two behind: ``_virtualenv.pth`` (repoints ``sys.prefix`` at the venv) and the project's ``__editable__`` pointer (names the BUILD machine — the payload wires the repo snapshot itself, see scripts/build/launcher_wrapper.py). Bundled launchers process every other .pth with ``site.addsitedir()``; pywin32.pth is load-bearing on Windows (win32\\lib on sys.path is what makes ``import pywintypes`` resolve, which portalocker/concurrent-log-handler need to write logs). """ if (venv_dir / "Scripts").is_dir(): sites = [venv_dir / "Lib" / "site-packages"] else: lib = venv_dir / "lib" sites = sorted(lib.glob("python*/site-packages")) if lib.is_dir() else [] for site_dir in sites: if not site_dir.is_dir(): continue for pth in site_dir.glob("*.pth"): if pth.name == "_virtualenv.pth" or pth.name.startswith("__editable__"): try: pth.unlink() except OSError: pass def _read_pipe(fd: int) -> bytes: size = 65536 if sys.platform == "win32": import _winapi import msvcrt # Bootstrap can run on Python 3.11, before Windows os.set_blocking # exists. Peek keeps the sole reader bounded without changing pipe mode. try: available, _ = _winapi.PeekNamedPipe(msvcrt.get_osfhandle(fd), 0) except BrokenPipeError: return b"" if not available: raise BlockingIOError size = min(size, available) return os.read(fd, size) def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int, output: TextSink) -> subprocess.CompletedProcess: """Keep CI progress live, a bounded diagnostic tail, and a wall-clock timeout.""" deadline = time.monotonic() + timeout proc = subprocess.Popen(command, cwd=native(cwd), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=0) pipe = proc.stdout assert isinstance(pipe, io.TextIOWrapper) # Popen was given stdout=PIPE and text=True. tail = "" conflict = "" try: # A descendant can keep stdout open after proc exits. Nonblocking reads # bound that drain without leaving a thread stuck in readline()/close(). if sys.platform != "win32": os.set_blocking(pipe.fileno(), False) decoder = io.IncrementalNewlineDecoder( codecs.getincrementaldecoder(pipe.encoding)(errors="replace"), translate=True, ) while True: remaining = deadline - time.monotonic() if remaining <= 0: raise subprocess.TimeoutExpired(command, timeout, stderr=tail) try: data = _read_pipe(pipe.fileno()) except BlockingIOError: time.sleep(min(.05, remaining)) continue text = decoder.decode(data, final=not data) if text: # Preserve an observed resolver marker even after verbose output # evicts it. Scan across read boundaries, never retain the full log. if not conflict: lowered = (tail + text).lower() conflict = next((marker for marker in _STREAM_TAIL_MARKERS if marker in lowered), "") tail = (tail + text)[-2000:] output.write(text) output.flush() if not data: break # EOF can precede process exit; it does not grant another timeout budget. try: code = proc.wait(timeout=max(0, deadline - time.monotonic())) except subprocess.TimeoutExpired: raise subprocess.TimeoutExpired(command, timeout, stderr=tail) from None except BaseException: proc.kill() proc.wait(timeout=5) raise finally: pipe.close() if conflict and conflict not in tail.lower(): tail = conflict + "\n" + tail[-(2000 - len(conflict) - 1):] return subprocess.CompletedProcess(command, code, "", tail) def _base_environment(env: Mapping[str, str] | None = None) -> dict[str, str]: """Ambient UV settings never select the project, interpreter or cache. Index and transport settings are the exception (pm.index_config): without them mirrored and air-gapped networks cannot resolve anything. """ from pm.index_config import bridged_index_settings, is_forwarded source = os.environ if env is None else env base = {key: value for key, value in source.items() if not key.startswith("PYTHON") and key != "VIRTUAL_ENV" and (not key.startswith("UV_") or is_forwarded(key))} if env is None: base.update(bridged_index_settings(os.environ)) return base def managed_environment(destination: Path, *, python: Path | None = None, cache: Path | None = None, env: Mapping[str, str] | None = None, offline: bool = False, explicit: bool = False, output: TextIO | None = None, realize: bool = True) -> PythonEnvironment: from pm._uv import _toolchain from pm.packages import uv_cache_dir tools = _toolchain(explicit=explicit, realize=realize) if tools is None: raise InstallError("venv", "PM's pinned toolchain is unavailable") uv, pinned_python = tools python = pinned_python if python is None else python.absolute() build_env = _base_environment(env) if python.resolve() == pinned_python.resolve(): # Only our interpreter: a separately selected one (e.g. Nix) keeps its own sysconfig. from pm.build_compilers import pinned_build_defaults for key, value in pinned_build_defaults(python, build_env).items(): build_env.setdefault(key, value) return PythonEnvironment( uv=uv, python=python, destination=destination.absolute(), cache=uv_cache_dir() if cache is None else cache.absolute(), env=build_env, offline=offline, output=output, ) @contextmanager def _fresh_build(environment: PythonEnvironment, *, sealed: bool, lock: Path | None = None) -> Iterator[None]: """Own only the output claimed by this build, including failed validation.""" out = environment.destination # A concurrent creator wins intact: never enter cleanup before mkdir succeeds. out.mkdir(parents=True) try: environment.create() yield environment.check(lock) if sealed: prune_site_pth(out) except BaseException: shutil.rmtree(out, ignore_errors=True) raise @dataclass(frozen=True, kw_only=True) class PythonEnvironment: uv: Path python: Path destination: Path cache: Path env: Mapping[str, str] offline: bool = False output: TextIO | None = None no_config: bool = False @property def executable(self) -> Path: from pm.environments import venv_python return venv_python(self.destination) def _run(self, args: list[str], *, cwd: Path, timeout: int) -> subprocess.CompletedProcess: # Explicit index credentials survive, but cannot redirect the project, # interpreter or cache selected by the operation. env = _base_environment(self.env) if self.no_config: # --no-config only removes config files; the pip-mirror bridge # (pm.index_config) reaches uv as env vars. The pinned runtime # builder asked to be isolated from ambient index configuration: # re-resolving its official-index lockfile against a mirror trips # `--locked` (#124418), and a fully-pinned graph never needs a # mirror to resolve. Index-REDIRECT settings are dropped; transport # knobs (UV_NATIVE_TLS / UV_INSECURE_HOST / UV_HTTP_TIMEOUT) survive # — corporate networks need them to reach the pinned URLs at all. from pm.index_config import is_index_redirect env = {key: value for key, value in env.items() if not is_index_redirect(key)} # uv is a child: it gets the ordinary spelling of store paths (pm.filesystem.native), # or the venv it writes would record the extended-length one in pyvenv.cfg. env.update(UV_PYTHON=native(self.python), UV_PROJECT_ENVIRONMENT=native(self.destination), UV_CACHE_DIR=native(self.cache), UV_PYTHON_DOWNLOADS="never") # CMake 4 removed compatibility with cmake_minimum_required(VERSION < 3.5), # so an sdist carrying an old minimum dies at CONFIGURE time: python-olm # (mautrix[encryption], the matrix extra) vendors libolm declaring 3.4. # Every source build uv drives inherits this env; older CMake ignores # the variable, and a value the caller or user already set wins. env.setdefault("CMAKE_POLICY_VERSION_MINIMUM", "3.5") with tempfile.TemporaryDirectory(prefix="pm-uv-config-") as config: env.update(XDG_CONFIG_HOME=config, XDG_CONFIG_DIRS=config) command = [native(self.uv), *map(native, args)] if self.no_config and "--no-config" not in command: command.append("--no-config") if self.offline: command.append("--offline") try: if self.output is not None and verbose_output(): # uv hides build-backend output until failure without verbose mode, # but --verbose alone is uv's DEBUG level: ~200 lines of interpreter # and cache internals on every streamed run. RUST_LOG scopes it to the # build frontend, so only the backend's own lines reach the user. command.append("--verbose") env.setdefault("RUST_LOG", "uv_build_frontend=debug") return _run_streaming(command, cwd=cwd, env=env, timeout=timeout, output=self.output) if self.output is not None: # uv prints backend output on failure anyway; interactive users see # one live status line and the captured tail only if it fails. tail = LiveTail(_UV_LABELS.get(tuple(args[:2]), _UV_LABELS.get(args[0])), self.output, indent=" ") try: result = _run_streaming(command, cwd=cwd, env=env, timeout=timeout, output=tail) except BaseException: tail.close(False) raise if "--compile-bytecode" in args and _is_bytecode_compile_failure(result): # Caller retries without the flag. A failure banner here # would report a recovered install as failed. tail.discard() else: tail.close(result.returncode == 0) return result return subprocess.run(command, cwd=native(cwd), env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False) except subprocess.TimeoutExpired as exc: from pm.index_config import TIMEOUT_HINT # A silent stall against an unreachable index is the #95608 shape; # name the mirror knobs instead of surfacing a raw TimeoutExpired. raise InstallError("venv", f"uv {args[0]} timed out after {timeout}s", TIMEOUT_HINT) from exc def create(self) -> None: """Create at the final destination; callers must not move a live venv.""" self.destination.parent.mkdir(parents=True, exist_ok=True) result = self._run( ["venv", "--relocatable", "--no-project", "--no-config", "--python", str(self.python), str(self.destination)], cwd=self.destination.parent, timeout=120, ) if result.returncode: raise InstallError("venv", f"uv venv failed: {result.stderr[-600:]}") def lock(self, source: Path, *, upgrade: bool = False, timeout: int = 1800) -> None: command = ["lock", "--python", str(self.python)] if upgrade: command.append("--upgrade") result = self._run(command, cwd=source, timeout=timeout) if result.returncode: raise classify_uv_failure("lock", result.returncode, result.stderr or result.stdout) def check_lock(self, source: Path) -> None: result = self._run(["lock", "--check", "--python", str(self.python)], cwd=source, timeout=1800) if result.returncode: raise classify_uv_failure("lock", result.returncode, result.stderr or result.stdout) def sync(self, source: Path, *, extras: Sequence[str] = (), groups: Sequence[str] = (), timeout: int = 1800, frozen: bool = True, all_extras: bool = False, no_install_project: bool = False, locked: bool = False, no_default_groups: bool = False) -> None: """Install the root and every member; resolve only in a writable workspace. ``frozen=False`` is reserved for the caller-owned generated workspace, never the original project's lock. Seed/replay policy belongs to PM. """ if not frozen: self.lock(source, timeout=timeout) # Locking members alone is insufficient: plain sync only installs root deps. # uv writes no __pycache__ (pip does): without --compile-bytecode the first import # of every module in the foreground of a user request compiles it (#100461). command = ["sync", "--locked" if locked else "--frozen", "--all-packages", "--python", str(self.python), "--compile-bytecode"] if no_default_groups: command.append("--no-default-groups") if all_extras: from pm.features import opt_in_extras command.append("--all-extras") for extra in opt_in_extras(source): command += ["--no-extra", extra] if no_install_project: # --all-packages has no single selected project in uv, so # --no-install-project alone does not exclude the root. Name it # explicitly without dropping member dependencies or installations. command += ["--no-install-project", "--no-install-package", _project_name(source)] for extra in sorted(set(extras)): command += ["--extra", extra] for group in sorted(set(groups)): command += ["--group", group] result = self._with_cache_recovery( "sync", lambda: self._sync_attempt(command, cwd=source, timeout=timeout)) if result.returncode: raise classify_uv_failure("sync", result.returncode, result.stderr or result.stdout) def _sync_attempt(self, command: list[str], *, cwd: Path, timeout: int) -> subprocess.CompletedProcess: result = self._run(command, cwd=cwd, timeout=timeout) if _is_bytecode_compile_failure(result): # Compile is an optimization. Retry once without the flag so a # locked temp script cannot fail pm repair/bootstrap. Imports # compile lazily on first use. if self.output is not None: self.output.write( "Bytecode compile failed; retrying the install without it. " "Modules will compile on first import.\n" ) fallback = [arg for arg in command if arg != "--compile-bytecode"] result = self._run(fallback, cwd=cwd, timeout=timeout) return result def _with_cache_recovery(self, stage: str, attempt: Callable[[], subprocess.CompletedProcess], ) -> subprocess.CompletedProcess: """Run *attempt*; if a corrupt cached package failed it, purge it and retry. An install interrupted by a machine stop can leave a cache entry whose files are NUL bytes, and uv reuses it on every sync, so a plain retry never helps. Only uv's install-step corruption shapes trigger this (pm.uv_cache_health); every other failure returns untouched for the caller to classify. Each purge also takes every entry that fails the same validation, so one retry normally covers several damaged packages. uv stops at the first broken entry, though, and a dangling pointer is invisible to the scan, so a retry that trips over a DIFFERENT package purges that one too. A package that fails again after its purge ends the loop: the source itself is serving a broken wheel. """ from pm.uv_cache_health import corrupt_cached_packages, failed_cached_packages purged: set[str] = set() result = attempt() for _ in range(_CACHE_RECOVERY_ROUNDS): if not result.returncode: return result failed = failed_cached_packages(f"{result.stderr or ''}\n{result.stdout or ''}") if not failed or failed <= purged: break names = sorted((failed | corrupt_cached_packages(self.cache)) - purged) if self.output is not None: self.output.write(f"The cached copy of {', '.join(names)} is corrupt; " "removing it from the uv cache and retrying the install.\n") try: self.purge_cached_packages(names) except InstallError as exc: # Keep the install failure that started this; the clean error alone hides it. original = f"{result.stderr or ''}\n{result.stdout or ''}".strip()[-600:] raise InstallError(exc.package, f"{exc.cause}; uv {stage} had failed with: {original}", exc.remedy) from exc purged.update(names) result = attempt() if not result.returncode or not purged: return result text = f"{result.stderr or ''}\n{result.stdout or ''}" cache = native(self.cache) still = failed_cached_packages(text) if still: remedy = ("the package source is serving a broken wheel; retry later or pin another version" if still <= purged else "run `hermes pm doctor --repair`, then retry") raise InstallError( "venv", f"uv {stage} exited {result.returncode}: the cached copy of {', '.join(sorted(still))} is " f"still invalid after removing {', '.join(sorted(purged))} from the uv cache at {cache}: " f"{text.strip()[-600:]}", remedy, ) if self.offline: raise InstallError( "venv", f"uv {stage} exited {result.returncode}: removed the corrupt cached copy of " f"{', '.join(sorted(purged))} from {cache}, and offline mode cannot fetch it again: " f"{text.strip()[-600:]}", "retry with network access", ) return result def purge_cached_packages(self, names: Sequence[str]) -> None: """Remove *names* from the uv cache. ``uv cache clean`` waits for other uv processes holding the cache, so another profile's sync never loses files it is reading. Archives it leaves behind that still fail validation are then deleted (pm.uv_cache_health.remove_corrupt_entries). """ from pm.uv_cache_health import remove_corrupt_entries cache = native(self.cache) try: result = replace(self, output=None)._run(["cache", "clean", *names], cwd=self.cache.parent, timeout=900) except InstallError as exc: raise InstallError("uv", f"could not clean the uv cache at {cache}: {exc.cause}", "stop other Hermes installs and retry") from exc if result.returncode: raise InstallError("uv", f"could not remove {', '.join(names)} from the uv cache at " f"{cache}: {(result.stderr or result.stdout)[-600:]}") remove_corrupt_entries(self.cache, set(names)) def export_requirements(self, source: Path, out: Path, *, extras: Sequence[str] = (), timeout: int = 1800) -> None: command = ["export", "--frozen", "--python", str(self.python), "--no-default-groups", "--no-emit-project", "--no-hashes", "--no-annotate", "--no-header", "--format", "requirements-txt", "--output-file", str(out)] for extra in sorted(set(extras)): command += ["--extra", extra] result = self._run(command, cwd=source, timeout=timeout) if result.returncode: raise classify_uv_failure("export", result.returncode, result.stderr or result.stdout) def install_requirements(self, requirements: Sequence[str], *, wheelhouse: Path | None = None, lock: Path | None = None) -> None: if not requirements: return # A file avoids command-line length limits and shell/marker quoting. with tempfile.TemporaryDirectory(prefix="pm-requirements-") as temporary: requirements_file = Path(temporary) / "requirements.txt" requirements_file.write_text("\n".join(requirements) + "\n", encoding="utf-8") self._install_requirements_file(requirements_file, wheelhouse=wheelhouse, lock=lock) def _install_requirements_file(self, requirements: Path, *, wheelhouse: Path | None = None, lock: Path | None = None, timeout: int = 1800) -> None: """Install *requirements*; *lock* is the uv.lock they were exported from. `uv pip install` re-resolves the export, so the lock's manifest overrides must come back as `--overrides` or the graph the lock resolved is rejected. """ command = ["pip", "install", "--no-config", "--python", str(self.executable), "--requirements", str(requirements)] if lock is not None: from pm.pip_check import override_requirements overrides = override_requirements(lock) if overrides: overrides_file = requirements.with_name("overrides.txt") overrides_file.write_text("\n".join(overrides) + "\n", encoding="utf-8") command += ["--overrides", str(overrides_file)] if wheelhouse is not None: command += ["--no-index", "--only-binary", ":all:", "--find-links", str(wheelhouse.absolute())] result = self._with_cache_recovery( "pip", lambda: self._run(command, cwd=requirements.parent, timeout=timeout)) if result.returncode: raise classify_uv_failure("pip", result.returncode, result.stderr or result.stdout) def install_wheelhouse(self, source: Path, wheelhouse: Path, *, timeout: int = 1800) -> None: """Install rebuilt wheels whose hashes the bundle manifest owns, not uv.lock.""" requirements = source / "requirements.txt" self.export_requirements(source, requirements, timeout=timeout) self._install_requirements_file(requirements, wheelhouse=wheelhouse, lock=source / "uv.lock", timeout=timeout) self.check(source / "uv.lock") def prune_cache(self, *, ci: bool = False) -> None: command = ["cache", "prune", "--no-config"] if ci: command += ["--ci", "--force"] result = self._run(command, cwd=Path.cwd(), timeout=1800) if result.returncode: raise InstallError("uv", f"cache pruning failed: {result.stderr[-600:]}") def check(self, lock: Path | None = None) -> None: """`uv pip check` the environment; *lock* is the uv.lock it was installed from. Mismatches the lock's manifest overrides account for are expected (pm.pip_check). The check is a quiet step, so it runs captured even for a streaming environment: a streamed run keeps only the output's tail, which can drop the report's head. """ result = replace(self, output=None)._run( # Forced colour (FORCE_COLOR/CLICOLOR_FORCE) would wrap the report in ANSI pm.pip_check cannot read. ["pip", "check", "--no-config", "--color", "never", "--python", str(self.executable)], cwd=self.destination.parent, timeout=60, ) if not result.returncode: return from pm.pip_check import unexpected_incompatibilities unexpected = unexpected_incompatibilities(f"{result.stdout or ''}\n{result.stderr or ''}", lock) if unexpected is None: raise InstallError("venv", f"dependency validation failed: {result.stderr[-600:]}") if unexpected: raise InstallError("venv", "dependency validation failed: " + "\n".join(unexpected)[-600:])