registry.py 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. """Package registry and dependency walk."""
  2. from __future__ import annotations
  3. import importlib
  4. import importlib.util
  5. from pathlib import Path
  6. import sys
  7. from types import ModuleType
  8. from typing import Any
  9. from pm.package import InstallError, Package, StatePackage
  10. _packages: dict[str, Package] = {}
  11. _BUILTIN_MODULES = ("pm.packages", "pm.security_packages")
  12. def _builtins_loaded() -> dict[str, Package]:
  13. """Register the built-in definitions on first read, not on ``import pm``.
  14. Process boot imports ``pm.environments`` before any dependency is importable;
  15. pulling the whole package catalogue (downloader, network) in with it would
  16. cost every launch ~40ms and break the stripped payloads that ship only the
  17. pre-import files.
  18. """
  19. for module in _BUILTIN_MODULES:
  20. if module not in sys.modules:
  21. importlib.import_module(module)
  22. return _packages
  23. def register(cls):
  24. instance = cls()
  25. if not instance.name:
  26. raise ValueError("package has no name")
  27. _packages[instance.name] = instance
  28. return cls
  29. def get_package(name: str) -> Package:
  30. packages = _builtins_loaded()
  31. if name not in packages:
  32. raise KeyError(f"unknown package: {name}")
  33. return packages[name]
  34. def all_packages() -> list[str]:
  35. return sorted(_builtins_loaded())
  36. def source_install_packages(names: list[str]) -> list[str]:
  37. """Select supported runtime roots; internal tools enter only through dependencies."""
  38. from pm.store import current_target
  39. target = current_target()
  40. return [name for name in names
  41. if not get_package(name).internal
  42. and (name == "python" or not get_package(name).optional)
  43. and get_package(name).missing_reason(target) is None]
  44. def tool_roots(names: list[str]) -> list[str]:
  45. """Packages to publish before a venv sync. The venv is not one of them."""
  46. return [name for name in source_install_packages(names) if not isinstance(get_package(name), StatePackage)]
  47. def package_definitions(names: list[str] | None = None) -> list[dict[str, Any]]:
  48. """Declarations for a fresh worker; built-ins already load with pm.
  49. Only import identities and file locations cross the wire, never instances,
  50. source bodies, or the application's import path. ``names`` selects a closure.
  51. """
  52. definitions = []
  53. for package in walk(names) if names is not None else list(_builtins_loaded().values()):
  54. cls = type(package)
  55. if cls.__module__ == "pm.packages":
  56. continue
  57. definition = {"name": package.name, "module": cls.__module__, "qualname": cls.__qualname__}
  58. module = sys.modules.get(cls.__module__)
  59. resolved = module
  60. for part in cls.__qualname__.split("."):
  61. resolved = getattr(resolved, part, None)
  62. if cls.__module__ == "__main__" or "<locals>" in cls.__qualname__ or resolved is not cls:
  63. raise InstallError(
  64. package.name, f"package definition {cls.__module__}.{cls.__qualname__} is not importable",
  65. "define and register a module-level Package subclass in an importable file",
  66. )
  67. source = getattr(module, "__file__", None)
  68. if source:
  69. definition["path"] = str(Path(source).resolve())
  70. namespaces = {}
  71. parent = cls.__module__.rpartition(".")[0]
  72. while parent:
  73. loaded = sys.modules.get(parent)
  74. if loaded is not None and not getattr(loaded, "__file__", None) and hasattr(loaded, "__path__"):
  75. namespaces[parent] = [str(Path(path).resolve()) for path in loaded.__path__]
  76. parent = parent.rpartition(".")[0]
  77. if namespaces:
  78. definition["namespaces"] = namespaces
  79. definitions.append(definition)
  80. return definitions
  81. def load_package_definitions(definitions: list[dict[str, Any]]) -> None:
  82. """Restore explicit registrations after built-ins, without plugin discovery."""
  83. restored = {}
  84. for definition in definitions:
  85. module_name = definition["module"]
  86. try:
  87. # Directory plugins use synthetic package parents. Their explicit
  88. # search paths preserve relative imports without exposing app deps.
  89. for name, locations in definition.get("namespaces", {}).items():
  90. if name not in sys.modules:
  91. namespace = ModuleType(name)
  92. namespace.__path__ = locations
  93. sys.modules[name] = namespace
  94. try:
  95. module = importlib.import_module(module_name)
  96. except ModuleNotFoundError as exc:
  97. # Missing dependencies inside the definition are not evidence
  98. # that its module is absent; never execute such a module twice.
  99. if not exc.name or not (module_name == exc.name or module_name.startswith(exc.name + ".")):
  100. raise
  101. spec = importlib.util.spec_from_file_location(module_name, definition["path"])
  102. if spec is None or spec.loader is None:
  103. raise ImportError(f"cannot load {module_name}")
  104. module = importlib.util.module_from_spec(spec)
  105. sys.modules[module_name] = module
  106. try:
  107. spec.loader.exec_module(module)
  108. except BaseException:
  109. sys.modules.pop(module_name, None)
  110. raise
  111. source = definition.get("path")
  112. if source and Path(getattr(module, "__file__", "") or "").resolve() != Path(source).resolve():
  113. raise ImportError(f"{module_name} resolved to a different source")
  114. cls = module
  115. for part in definition["qualname"].split("."):
  116. cls = getattr(cls, part)
  117. if not isinstance(cls, type) or not issubclass(cls, Package):
  118. raise TypeError("definition is not a Package subclass")
  119. instance = cls()
  120. if instance.name != definition["name"]:
  121. raise ValueError(f"definition now registers {instance.name!r}")
  122. restored[instance.name] = instance
  123. except Exception as exc:
  124. raise InstallError(
  125. definition["name"], f"cannot load package definition {module_name}.{definition['qualname']}: {exc}",
  126. "keep the package definition and its dependencies importable in the isolated PM runtime",
  127. ) from exc
  128. # Import-time decorators must not override the caller's final selection:
  129. # load the built-ins first so a restored definition wins.
  130. _builtins_loaded()
  131. _packages.update(restored)
  132. def walk(names: list[str]) -> list[Package]:
  133. """Deps-first topological order over the requested packages."""
  134. seen: dict[str, Package] = {}
  135. def visit(name: str, chain: tuple[str, ...]) -> None:
  136. if name in chain:
  137. cycle = " -> ".join(chain + (name,))
  138. raise ValueError(f"dependency cycle: {cycle}")
  139. if name in seen:
  140. return
  141. package = get_package(name)
  142. for dep in package.deps:
  143. visit(dep, chain + (name,))
  144. seen[name] = package
  145. for name in names:
  146. visit(name, ())
  147. return list(seen.values())