runtime.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. """The dependency manager's runtime, independent of the application graph.
  2. Only the bootstrap below runs in the caller's interpreter. It never imports
  3. application dependencies or adds the manager's dependencies to that process.
  4. """
  5. from __future__ import annotations
  6. import hashlib
  7. import json
  8. import os
  9. from pathlib import Path
  10. import shutil
  11. import subprocess
  12. import sys
  13. from typing import Callable
  14. import uuid
  15. from pm.filesystem import native
  16. from pm.package import InstallError
  17. def runtime_environment() -> dict[str, str]:
  18. """Do not let an activated application or a uv caller select PM's imports."""
  19. from hermes_constants import get_hermes_home
  20. from pm.paths import store_root
  21. from pm.environment import _base_environment
  22. env = _base_environment()
  23. env["HERMES_HOME"] = str(get_hermes_home())
  24. env["HERMES_RUNTIME_DIR"] = native(store_root())
  25. return env
  26. def _python(environment: Path) -> Path:
  27. from pm.environments import venv_python
  28. return venv_python(environment)
  29. def _inputs(project: Path, python: Path, *, as_spelled: bool = False) -> str:
  30. digest = hashlib.sha256()
  31. for name in ("pyproject.toml", "uv.lock"):
  32. digest.update((project / name).read_bytes())
  33. digest.update(b"\0")
  34. # A different interpreter must not reuse a venv pointing at the old one, but
  35. # the same one reached through a symlinked store (a per-task HERMES_HOME whose
  36. # tools/ links back) must. Only the directories resolve: the pinned
  37. # bin/python3 is itself a symlink, so resolving it would re-key every install.
  38. # A store spells its root extended-length on Windows; the identity must not.
  39. python = Path(native(python)).absolute()
  40. digest.update(str(python if as_spelled else python.parent.resolve() / python.name).encode())
  41. return digest.hexdigest()
  42. def is_runtime() -> bool:
  43. if (Path(sys.prefix) / "pm-runtime.json").is_file():
  44. return True
  45. resident = _resident_runtime()
  46. return resident is not None and str(resident[1]) in sys.path
  47. def _resident_runtime() -> tuple[Path, Path] | None:
  48. from pm.paths import repo_root
  49. project = repo_root()
  50. payload = project.parent if (project.parent / "manifest.json").is_file() else None
  51. if payload is not None:
  52. runtime = payload / "pm-runtime"
  53. else:
  54. from pm.paths import install_stamp_path
  55. stamp_path = install_stamp_path(project)
  56. try:
  57. stamp = json.loads(stamp_path.read_text(encoding="utf-8-sig"))
  58. except FileNotFoundError:
  59. return None
  60. except (OSError, ValueError) as exc:
  61. raise InstallError("pm-runtime", "invalid package install stamp", "reinstall this application") from exc
  62. if stamp.get("distribution") not in ("nix", "docker"):
  63. return None
  64. value = stamp.get("pmRuntime")
  65. if not isinstance(value, str) or not Path(value).is_absolute():
  66. raise InstallError("pm-runtime", "packaged PM runtime is missing", "reinstall this application")
  67. runtime = Path(value)
  68. try:
  69. marker = json.loads((runtime / "pm-runtime.json").read_text(encoding="utf-8"))
  70. python = (runtime / marker["python"]).resolve()
  71. site = (runtime / marker["sitePackages"]).resolve()
  72. except (OSError, ValueError, KeyError, TypeError) as exc:
  73. raise InstallError("pm-runtime", "packaged PM runtime is missing or invalid", "reinstall this application") from exc
  74. # Native payloads must remain self-contained. Nix's independent derivation
  75. # instead refers to its immutable interpreter/dependencies in /nix/store.
  76. if (not python.is_file() or not site.is_dir()
  77. or (payload is not None and (not python.is_relative_to(payload) or not site.is_relative_to(runtime)))):
  78. raise InstallError("pm-runtime", "packaged PM paths are missing or escape the payload", "reinstall this application")
  79. return python, site
  80. def _validate(python: Path, env: dict[str, str]) -> str:
  81. try:
  82. checked = subprocess.run(
  83. [str(python), "-I", "-B", "-c",
  84. "import packaging, tomli_w, truststore; from ruamel.yaml import YAML"],
  85. env=env, capture_output=True, text=True, timeout=30,
  86. check=False,
  87. )
  88. except (OSError, subprocess.TimeoutExpired) as exc:
  89. return str(exc)
  90. return checked.stderr.strip() or f"exit {checked.returncode}" if checked.returncode else ""
  91. # Generations this process resolved for a child. The lease is taken under
  92. # .prepare.lock, which the collector also holds, so a publish + `pm gc` between
  93. # this return and the child's own lease cannot remove the child's generation.
  94. # It lives as long as this process (one lease per generation, not per call):
  95. # children may still be starting from an older generation after a newer one
  96. # is chosen.
  97. _HELD: dict[Path, Callable[[], None]] = {}
  98. def _hold_for_children(environment: Path) -> None:
  99. from hermes_cli.runtime_state import lease_directory
  100. if environment not in _HELD:
  101. _HELD[environment] = lease_directory(environment)
  102. def prepare_runtime(uv: Path, python: Path, root: Path, *, offline: bool = False,
  103. project: Path | None = None, bootstrap: bool = True,
  104. cache: Path | None = None) -> Path:
  105. """Publish a locked PM environment without resolving the application.
  106. Generations are immutable after publication. Failed preparation leaves the
  107. previous generation intact, including when an old worker is still running.
  108. """
  109. from pm.filesystem import lock_fd
  110. from pm.lock import _write
  111. from pm.runtime_stage import stage_runtime
  112. project = project or Path(__file__).resolve().parent
  113. identity = _inputs(project, python)
  114. env = runtime_environment()
  115. root.mkdir(parents=True, exist_ok=True)
  116. with (root / ".prepare.lock").open("a+b") as lock:
  117. lock_fd(lock.fileno(), wait=True)
  118. selected = root / "selected.json"
  119. try:
  120. fact = json.loads(selected.read_text(encoding="utf-8"))
  121. except FileNotFoundError:
  122. fact = {}
  123. # Records from before canonicalization spell the path as launched; for a
  124. # home under a symlink (/home -> /var/home) that is still this interpreter.
  125. if fact.get("inputs") in (identity, _inputs(project, python, as_spelled=True)):
  126. environment = root / fact["generation"]
  127. if (environment / "pm-runtime.json").is_file() and not _validate(_python(environment), env):
  128. _hold_for_children(environment)
  129. return _python(environment)
  130. if not bootstrap:
  131. raise InstallError("pm-runtime", "not installed or outdated and lazy installs are disabled",
  132. "run `hermes pm install` to prepare the independent PM runtime")
  133. generation = Path("generations") / uuid.uuid4().hex
  134. environment = root / generation
  135. try:
  136. print("Preparing the isolated Hermes runtime…", file=sys.stderr, flush=True)
  137. executable = stage_runtime(uv, python, environment, project=project, offline=offline, cache=cache)
  138. (environment / ".lease-managed").touch()
  139. _write(environment / "pm-runtime.json", {"inputs": identity})
  140. _write(selected, {"inputs": identity, "generation": generation.as_posix()})
  141. except BaseException:
  142. shutil.rmtree(environment, ignore_errors=True)
  143. raise
  144. _hold_for_children(environment)
  145. return executable
  146. def lease_current_runtime() -> None:
  147. """Pin the PM runtime this process runs from so the collector leaves it alone."""
  148. if (Path(sys.prefix) / "pm-runtime.json").is_file():
  149. from hermes_cli.runtime_state import lease_directory
  150. lease_directory(Path(sys.prefix))
  151. def collect_runtime_generations(root: Path) -> list[Path]:
  152. """Remove PM runtime generations nothing can run from any more.
  153. Staging happens under ``.prepare.lock``, so with it held an unpublished generation
  154. (no ``pm-runtime.json``) is an aborted stage. A superseded published generation goes
  155. once every worker launched from it has exited; generations published before leases
  156. existed stay, as the application collector keeps its own.
  157. """
  158. from pm.filesystem import lock_fd
  159. from hermes_cli.runtime_state import leases_held
  160. generations = root / "generations"
  161. removed: list[Path] = []
  162. if not generations.is_dir():
  163. return removed
  164. with (root / ".prepare.lock").open("a+b") as lock:
  165. if not lock_fd(lock.fileno(), wait=False):
  166. return removed # a stage is in flight; maintenance skips rather than queues
  167. try:
  168. selected = json.loads((root / "selected.json").read_text(encoding="utf-8-sig")).get("generation", "")
  169. except FileNotFoundError:
  170. selected = ""
  171. for generation in sorted(generations.iterdir()):
  172. if not generation.is_dir() or generation.is_symlink() or generation == root / selected:
  173. continue
  174. published = (generation / "pm-runtime.json").is_file()
  175. if published and (not (generation / ".lease-managed").is_file() or leases_held(generation)):
  176. continue
  177. shutil.rmtree(generation)
  178. removed.append(generation)
  179. return removed
  180. def runtime_python(*, bootstrap: bool = True, cache: Path | None = None) -> Path:
  181. """Resolve PM without selecting, repairing, or importing the app environment."""
  182. if is_runtime():
  183. return Path(sys.executable)
  184. from pm.environments import install_state_dir
  185. from pm._uv import _toolchain
  186. from pm.paths import repo_root
  187. project = repo_root()
  188. resident = _resident_runtime()
  189. if resident is not None:
  190. return resident[0]
  191. tools = _toolchain(realize=False)
  192. if tools is None:
  193. if not bootstrap:
  194. raise InstallError("pm-runtime", "not installed and lazy installs are disabled",
  195. "run `hermes pm install` to prepare the independent PM runtime")
  196. from pm.lock import Lockfile
  197. from pm.paths import lockfile_path, store_root
  198. from pm.registry import get_package
  199. from pm.store import current_target
  200. # Setup has already verified/extracted uv, but there are no PM facts
  201. # yet. Use it to acquire PM's TLS support BEFORE downloading Python.
  202. package = get_package("uv")
  203. version = Lockfile(lockfile_path()).version("uv")
  204. target = current_target()
  205. staged = package.binary(store_root() / package.store_entry(version, target), target) if version else None
  206. if staged is not None and staged.is_file():
  207. tools = staged, Path(sys.executable)
  208. else:
  209. # Non-shell bootstrap callers (CI) already have a host interpreter.
  210. tools = _toolchain(explicit=True)
  211. if tools is None:
  212. raise InstallError("pm-runtime", "pinned uv and Python are unavailable")
  213. uv, python = tools
  214. return prepare_runtime(uv, python, install_state_dir(project) / "pm-runtime",
  215. bootstrap=bootstrap, cache=cache)
  216. def runtime_command(script: Path, args: tuple[str, ...] | list[str] = (), *,
  217. bootstrap: bool = True, cache: Path | None = None) -> list[str]:
  218. """One launch contract for mutable venvs and resident signed payloads."""
  219. resident = _resident_runtime()
  220. if resident is None:
  221. python = runtime_python(bootstrap=bootstrap, cache=cache)
  222. return [str(python), "-I", "-B", str(script), *args]
  223. python, site = resident
  224. launcher = (
  225. "import runpy,sys; sys.path.insert(0,sys.argv.pop(1)); "
  226. "script=sys.argv.pop(1); sys.argv[0]=script; runpy.run_path(script,run_name='__main__')"
  227. )
  228. return [str(python), "-I", "-S", "-B", "-c", launcher, str(site), str(script), *args]
  229. def run_cli(argv: list[str]) -> int:
  230. result = subprocess.run(runtime_command(Path(__file__).with_name("launch.py"), argv),
  231. env=runtime_environment(), check=False)
  232. return result.returncode