client.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438
  1. """Synchronous PM mutations in an isolated interpreter, never the app's imports."""
  2. from __future__ import annotations
  3. from collections.abc import Mapping, Sequence
  4. import json
  5. from pathlib import Path
  6. import subprocess
  7. import sys
  8. import threading
  9. import uuid
  10. from pm import paths, plugin_inputs
  11. from pm.package import InstallError, Runner, StatePackage
  12. from pm.plugin_inputs import Candidates, Members, PluginInput, Selection
  13. from pm.runtime import is_runtime, runtime_command, runtime_environment
  14. from pm.worker_operations import OPERATIONS
  15. def _missing_or_refuse(name):
  16. from pm.install import _refuse_lazy, current_target, is_installed, lazy_installs_allowed
  17. from pm.registry import get_package, walk
  18. missing = [package.name for package in walk([name]) if not is_installed(package.name)]
  19. target = current_target() if missing else None
  20. for gap in missing:
  21. # A registry gap (git on POSIX: "uses system git by choice") is static: no worker can
  22. # install it, so answer with the worker's own refusal here. Asking one anyway cost a PM
  23. # worker round trip (about a second) on every ``selected_git_env()``, which each
  24. # checkpoint git child calls, so one gateway /rollback or /diff paid it several times.
  25. reason = get_package(gap).missing_reason(target)
  26. if reason is not None:
  27. raise InstallError(gap, f"unavailable on {target}: {reason}", "none")
  28. if missing and not lazy_installs_allowed():
  29. raise _refuse_lazy(name, ", ".join(missing))
  30. return missing
  31. def _refuse_cold_runtime(cold: InstallError, arguments) -> InstallError:
  32. """Record a lazy sync refused because PM's own runtime is missing; return the error to raise."""
  33. from pm import receipt
  34. exc = cold
  35. if arguments.get("extras"):
  36. # The user asked for an extra, not for PM's own runtime: name
  37. # the command that provisions both.
  38. from pm.extras import install_hint
  39. exc = InstallError(cold.package, f"{cold.cause} while enabling {list(arguments['extras'])}",
  40. "run `" + "`, `".join(install_hint(extra) for extra in arguments["extras"]) + "`")
  41. token = receipt.begin("sync")
  42. try:
  43. receipt.record_refusal("lazy-install", str(exc))
  44. receipt.record_step("dependency-sync", False, f"{type(exc).__name__}: {exc}")
  45. finally:
  46. receipt.finalize("failed", 1, token=token)
  47. return exc
  48. def _worker_command(spec, arguments, worker: Path, environment: dict) -> list[str]:
  49. """How to start the worker; may disable lazy installs in *environment*."""
  50. from pm.install import lazy_installs_allowed
  51. from pm.registry import get_package
  52. # Bootstrap precedes dispatch and must share the operation's selected cache.
  53. cache = Path(arguments["cache"]) if arguments.get("cache") is not None else None
  54. state_sync = spec.bootstrap == "policy" or (
  55. spec.bootstrap == "state" and isinstance(get_package(arguments["name"]), StatePackage))
  56. if (state_sync and not arguments.get("explicit") and not arguments.get("repair")
  57. and not lazy_installs_allowed()):
  58. # A ready PM still decides no-op/refusal under its install lock. A cold
  59. # PM is itself a missing prerequisite, not permission to bootstrap tools.
  60. try:
  61. command = runtime_command(worker, bootstrap=False, cache=cache)
  62. except InstallError as cold:
  63. raise _refuse_cold_runtime(cold, arguments) from None
  64. environment["HERMES_DISABLE_LAZY_INSTALLS"] = "1"
  65. return command
  66. if spec.bootstrap == "never":
  67. return runtime_command(worker, bootstrap=False, cache=cache)
  68. return runtime_command(worker, cache=cache)
  69. _WORKER_ERRORS = {"ValueError": ValueError, "TypeError": TypeError, "KeyError": KeyError,
  70. "OSError": OSError, "FileExistsError": FileExistsError,
  71. "FileNotFoundError": FileNotFoundError, "PermissionError": PermissionError}
  72. def _raise_worker_error(error: dict):
  73. """Re-raise a worker failure as the caller-side exception it names."""
  74. if "package" in error:
  75. from pm.workspace import ResolutionConflict
  76. kind = ResolutionConflict if error["type"] == "ResolutionConflict" else InstallError
  77. raise kind(error["package"], error["cause"], error["remedy"])
  78. if error["type"] == "DownloadPaused":
  79. from pm.downloader import DownloadPaused
  80. raise DownloadPaused(error["message"])
  81. raise _WORKER_ERRORS.get(error["type"], RuntimeError)(error["message"])
  82. def _request(operation, arguments, *, callbacks=None, pause_event=None, project_root=None):
  83. from pm import receipt
  84. from pm.registry import package_definitions
  85. request_id = uuid.uuid4().hex
  86. update_id = receipt._ambient_update_id()
  87. callbacks = callbacks or {}
  88. spec = OPERATIONS[operation]
  89. names = list(spec.packages) if spec.packages is not None else (
  90. arguments["names"] if "names" in arguments else [arguments["name"]])
  91. message = {
  92. "id": request_id, "operation": operation, "arguments": arguments,
  93. "update_id": update_id,
  94. "callbacks": list(callbacks),
  95. "packages": package_definitions(names),
  96. "context": {"repo": str(Path(project_root).absolute() if project_root is not None else paths.repo_root()),
  97. "lockfile": str(paths.lockfile_path())},
  98. }
  99. worker = Path(__file__).with_name("worker.py").resolve()
  100. environment = runtime_environment()
  101. command = _worker_command(spec, arguments, worker, environment)
  102. callback_error = None
  103. stopped = threading.Event()
  104. write_lock = threading.Lock()
  105. monitor = None
  106. with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
  107. text=True, encoding="utf-8", env=environment) as process:
  108. assert process.stdin is not None and process.stdout is not None
  109. writer = process.stdin
  110. def send(data):
  111. with write_lock:
  112. cancelled = pause_event is not None and pause_event.is_set()
  113. writer.write(json.dumps({"id": request_id, "cancel": cancelled, **data}) + "\n")
  114. writer.flush()
  115. def watch_pause():
  116. assert pause_event is not None
  117. while not stopped.wait(0.05):
  118. if pause_event.is_set():
  119. try:
  120. send({"type": "cancel"})
  121. except (OSError, ValueError):
  122. return # The final response may already be on its way.
  123. return
  124. try:
  125. send(message)
  126. if pause_event is not None:
  127. monitor = threading.Thread(target=watch_pause, daemon=True)
  128. monitor.start()
  129. while True:
  130. line = process.stdout.readline()
  131. if not line:
  132. raise InstallError("pm", "worker exited without a result", "check the worker diagnostics on stderr")
  133. response = json.loads(line)
  134. if response["id"] != request_id:
  135. raise InstallError("pm", "worker returned a different request id")
  136. if response["type"] == "result":
  137. break
  138. name = response["callback"]
  139. try:
  140. callbacks[name](*response.get("args", []))
  141. send({"type": "callback_result", "call": response["call"], "result": None})
  142. except BaseException as exc:
  143. if callback_error is None:
  144. callback_error = exc
  145. send({"type": "callback_result", "call": response["call"],
  146. "error": f"{type(exc).__name__}: {exc}"})
  147. stopped.set()
  148. if monitor is not None:
  149. monitor.join()
  150. writer.close()
  151. if process.wait(timeout=5):
  152. raise InstallError("pm", "worker exited unsuccessfully")
  153. receipt.accept_worker_receipt(response.get("receipt"), update_id)
  154. if callback_error is not None:
  155. raise callback_error
  156. if "error" in response:
  157. _raise_worker_error(response["error"])
  158. return response["result"]
  159. finally:
  160. stopped.set()
  161. if monitor is not None:
  162. monitor.join()
  163. if process.poll() is None:
  164. process.terminate()
  165. process.wait(timeout=5)
  166. def ensure(name, *, base_env=None, explicit=False, verify=True, progress=None, pause_event=None,
  167. download_progress=None) -> Runner:
  168. from pm.install import env_for
  169. from pm.registry import get_package
  170. if is_runtime():
  171. from pm.install import ensure as direct
  172. return direct(name, base_env=base_env, explicit=explicit, verify=verify, progress=progress,
  173. pause_event=pause_event, download_progress=download_progress)
  174. if not explicit and not isinstance(get_package(name), StatePackage):
  175. if not _missing_or_refuse(name):
  176. return Runner(name, env_for(name, base_env=base_env))
  177. if pause_event is not None and pause_event.is_set():
  178. from pm.downloader import DownloadPaused
  179. raise DownloadPaused("install paused")
  180. callbacks = {}
  181. if progress is not None:
  182. callbacks["progress"] = progress
  183. if download_progress is not None:
  184. callbacks["download_progress"] = lambda done, total, ranges: download_progress(
  185. done, total, {key: [tuple(row) for row in rows] for key, rows in ranges.items()})
  186. # The worker's default is verify=True: only a lowered check travels in the request.
  187. _request("ensure", {"name": name, "explicit": explicit, **({} if verify else {"verify": False})},
  188. callbacks=callbacks, pause_event=pause_event)
  189. return Runner(name, env_for(name, base_env=base_env))
  190. def sync_venv(extras=None, *, explicit=False, plugins: PluginInput | None = None, repair=False,
  191. project_root: Path | None = None, evict_incompatible_plugins: bool = False,
  192. automatic: list[str] | None = None, carried: list[str] | None = None) -> None:
  193. from pm.environments import running_from_selected_environment
  194. if extras and not explicit and not repair and not running_from_selected_environment(
  195. paths.repo_root() if project_root is None else Path(project_root)):
  196. # A lazy extra may only extend the environment this process runs from. From any other
  197. # interpreter (a build_environment test venv, a developer venv, a Nix Python) the sync would
  198. # commit a selection this process never activates while every process booted afterwards
  199. # swaps onto it — a generation without whatever the foreign interpreter carried.
  200. from pm.install import _refuse_lazy
  201. raise _refuse_lazy(
  202. "venv",
  203. f"{list(extras)}: this process is not running from the install's dependency environment "
  204. f"({sys.prefix}); only an explicit install may change what later processes boot into",
  205. )
  206. if isinstance(plugins, Selection) and "expected_config" not in plugins.data:
  207. from pm.filesystem import file_digest
  208. plugins = Selection({**plugins.data,
  209. "expected_config": file_digest(Path(plugins.data["home"]) / "config.yaml") or "missing"}) # config-reader: ok — pm publication: byte-level compare-and-swap rewrite of the local file
  210. foreign = project_root is not None and Path(project_root).resolve() != paths.repo_root().resolve()
  211. if is_runtime() and not foreign:
  212. from pm.install import sync_venv as direct
  213. return direct(extras, explicit=explicit, plugins=plugins, repair=repair,
  214. evict_incompatible_plugins=evict_incompatible_plugins, automatic=automatic, carried=carried)
  215. _request("sync_venv", {"extras": extras, "explicit": explicit, "repair": repair,
  216. "plugins": plugin_inputs.encode(plugins),
  217. "evict_incompatible_plugins": evict_incompatible_plugins, "automatic": automatic,
  218. "carried": carried},
  219. project_root=project_root)
  220. def ensure_tools_for_sync() -> None:
  221. """Publish every required tool in this tree's lockfile, then put them on PATH.
  222. Updates call this before the venv sync: the sync only pulls uv/python in
  223. through its own dependency, so a bumped ripgrep/ffmpeg/node pin was never
  224. installed and activation skipped the managed tool dirs on every start.
  225. Publishing tools first also lets native builds resolve compilers and git
  226. from the pinned store instead of the host (as `hermes pm install` does).
  227. An update is an explicit user action, so the lazy-install policy does not
  228. gate it; a failed download fails the update.
  229. """
  230. from pm.install import activate
  231. from pm.libatomic import install_before_lock
  232. from pm.lock import Lockfile
  233. from pm.registry import tool_roots
  234. install_before_lock()
  235. for name in tool_roots(Lockfile(paths.lockfile_path()).names()):
  236. ensure(name, explicit=True)
  237. problems = activate(allow_incomplete=True)
  238. if problems:
  239. raise RuntimeError(f"tools not on PATH before venv sync: {'; '.join(problems)}")
  240. def stage_only(name, target, *, progress=None) -> Path:
  241. if is_runtime():
  242. from pm.install import stage_only as direct
  243. return direct(name, target, progress=progress)
  244. callbacks = {"progress": progress} if progress is not None else {}
  245. return Path(_request("stage_only", {"name": name, "target": target}, callbacks=callbacks))
  246. def _python_operation(operation: str, arguments: dict):
  247. if is_runtime():
  248. return OPERATIONS[operation].resolve(operation)(**arguments)
  249. payload = {key: str(value.absolute()) if isinstance(value, Path) else value
  250. for key, value in arguments.items()}
  251. return _request(operation, payload)
  252. def build_environment(
  253. *, source: Path, out: Path, python: Path | None = None,
  254. cache: Path | None = None, env: Mapping[str, str] | None = None,
  255. extras: Sequence[str] = (), groups: Sequence[str] = (),
  256. all_extras: bool = False, no_install_project: bool = False,
  257. frozen: bool = True, sealed: bool = False, offline: bool = False,
  258. explicit: bool = False, timeout: int = 1800,
  259. ) -> Path:
  260. """Build a validated Python environment without exposing install machinery."""
  261. return Path(_python_operation("build_environment", {
  262. "source": Path(source), "out": Path(out), "python": python, "cache": cache,
  263. "env": dict(env) if env is not None else None, "extras": list(extras), "groups": list(groups),
  264. "all_extras": all_extras, "no_install_project": no_install_project,
  265. "frozen": frozen, "sealed": sealed, "offline": offline,
  266. "explicit": explicit, "timeout": timeout,
  267. }))
  268. def lock_project(
  269. source: Path, *, upgrade: bool = False, python: Path | None = None,
  270. env: Mapping[str, str] | None = None, cache: Path | None = None,
  271. offline: bool = False, explicit: bool = False,
  272. ) -> None:
  273. """Resolve a project's lock without creating or selecting an environment."""
  274. _python_operation("lock_project", {
  275. "source": Path(source), "upgrade": upgrade, "python": python,
  276. "env": dict(env) if env is not None else None, "cache": cache,
  277. "offline": offline, "explicit": explicit,
  278. })
  279. def stage_manager_runtime(
  280. *, python: Path, destination: Path, project: Path | None = None,
  281. offline: bool = False, wheelhouse: Path | None = None, cache: Path | None = None,
  282. ) -> Path:
  283. # PM cannot dispatch the construction of its own offline runtime through
  284. # that runtime. The stdlib-only bootstrap shares the private build engine.
  285. from pm.operations import stage_manager_runtime as stage
  286. return stage(python=Path(python), destination=Path(destination), project=project,
  287. offline=offline, wheelhouse=wheelhouse, cache=cache)
  288. def ensure_environment(
  289. name: str, requirements: Sequence[str], *, root: Path | None = None,
  290. explicit: bool = False, timeout: int = 1800,
  291. ) -> Path:
  292. """Select a complete isolated dependency generation, retaining the previous one."""
  293. if isinstance(requirements, str):
  294. raise TypeError("requirements must be a sequence, not a string")
  295. return Path(_python_operation("ensure_environment", {
  296. "name": name, "requirements": list(requirements), "root": root,
  297. "explicit": explicit, "timeout": timeout,
  298. }))
  299. def ensure_project_environment(
  300. name: str, project: Path, *, extras: Sequence[str] = (), groups: Sequence[str] = (),
  301. root: Path | None = None, explicit: bool = False, timeout: int = 1800,
  302. ) -> Path:
  303. """Select an isolated environment of the project's locked dependencies."""
  304. return Path(_python_operation("ensure_project_environment", {
  305. "name": name, "project": Path(project), "extras": list(extras), "groups": list(groups),
  306. "root": root, "explicit": explicit, "timeout": timeout,
  307. }))
  308. def ensure_python_tool(
  309. name: str, requirements: Sequence[str], executable: str, *, root: Path | None = None,
  310. explicit: bool = False, timeout: int = 1800,
  311. ) -> Path:
  312. """Install an isolated tool and return its validated executable, not uv/uvx."""
  313. if isinstance(requirements, str):
  314. raise TypeError("requirements must be a sequence, not a string")
  315. return Path(_python_operation("ensure_python_tool", {
  316. "name": name, "requirements": list(requirements), "executable": executable,
  317. "root": root, "explicit": explicit, "timeout": timeout,
  318. }))
  319. def venv_is_current(*, extras: list[str] | None = None, plugins: Members | Candidates | None = None,
  320. project_root: Path | None = None) -> bool:
  321. """Check through a ready PM, never bootstrap dependencies for a probe."""
  322. if is_runtime() and (project_root is None or Path(project_root).resolve() == paths.repo_root().resolve()):
  323. from pm.install import venv_is_current as direct
  324. return direct(extras=extras, plugins=plugins, project_root=project_root)
  325. try:
  326. return bool(_request("venv_is_current", {"extras": extras, "plugins": plugin_inputs.encode(plugins)},
  327. project_root=project_root))
  328. except InstallError as exc:
  329. if exc.package == "pm-runtime":
  330. return False # Without its checker, currency cannot be established.
  331. raise
  332. def check_project_lock(source: Path, *, python: Path | None = None, cache: Path | None = None,
  333. env: Mapping[str, str] | None = None, offline: bool = False,
  334. explicit: bool = False, quiet: bool = False) -> None:
  335. _python_operation("check_project_lock", {
  336. "source": Path(source), "python": python, "cache": cache,
  337. "env": dict(env) if env is not None else None, "offline": offline, "explicit": explicit,
  338. "quiet": quiet,
  339. })
  340. def export_requirements(source: Path, out: Path, *, extras: Sequence[str] = (),
  341. python: Path | None = None, cache: Path | None = None,
  342. env: Mapping[str, str] | None = None, explicit: bool = False) -> None:
  343. _python_operation("export_requirements", {
  344. "source": Path(source), "out": Path(out), "extras": list(extras),
  345. "python": python, "cache": cache, "env": dict(env) if env is not None else None,
  346. "explicit": explicit,
  347. })
  348. def build_requirements_environment(requirements: Sequence[str], *, out: Path,
  349. python: Path | None = None, cache: Path | None = None,
  350. env: Mapping[str, str] | None = None, wheelhouse: Path | None = None,
  351. offline: bool = False, sealed: bool = False,
  352. explicit: bool = False, lock: Path | None = None) -> Path:
  353. if isinstance(requirements, str):
  354. raise TypeError("requirements must be a sequence, not a string")
  355. return Path(_python_operation("build_requirements_environment", {
  356. "requirements": list(requirements), "out": Path(out), "python": python, "cache": cache,
  357. "env": dict(env) if env is not None else None, "wheelhouse": wheelhouse,
  358. "offline": offline, "sealed": sealed, "explicit": explicit, "lock": lock,
  359. }))
  360. def prepare_tools(names: Sequence[str], *, out: Path, target: str,
  361. cache: Path | None = None) -> Path:
  362. """Realize build tools without selecting application or profile state."""
  363. if isinstance(names, str):
  364. raise TypeError("names must be a sequence, not a string")
  365. return Path(_python_operation("prepare_tools", {
  366. "names": list(names), "out": Path(out), "target": target, "cache": cache,
  367. }))
  368. def stage_tools(names: Sequence[str], *, source_store: Path, out: Path, target: str) -> Path:
  369. """Independently copy verified pins through a ready PM; never bootstrap."""
  370. if isinstance(names, str):
  371. raise TypeError("names must be a sequence, not a string")
  372. return Path(_python_operation("stage_tools", {
  373. "names": list(names), "source_store": Path(source_store), "out": Path(out), "target": target,
  374. }))
  375. def prune_cache(cache: Path, *, ci: bool = False) -> None:
  376. _python_operation("prune_cache", {"cache": Path(cache), "ci": ci})