plugin_loader.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. """Shared directory-plugin loader for ``plugins/<kind>/<name>/`` discovery packages
  2. (cron_providers, context_engine, memory): import ``__init__.py`` by path with siblings
  3. pre-registered so relative imports work, then extract the provider via ``register(ctx)``
  4. or an ABC-subclass fallback."""
  5. from __future__ import annotations
  6. import contextlib
  7. import importlib.machinery
  8. import importlib.util
  9. import logging
  10. import sys
  11. import threading
  12. import time
  13. from contextvars import ContextVar
  14. from pathlib import Path
  15. from typing import Any, Callable, List, Optional, Tuple
  16. _log = logging.getLogger(__name__)
  17. _PLUGINS_ROOT = Path(__file__).parent
  18. _MODULE_LOAD_LOCKS: dict[str, threading.RLock] = {}
  19. _MODULE_LOAD_LOCKS_GUARD = threading.Lock()
  20. # Inside bounded_load_wait(), how long a second caller waits for another thread's in-flight load of
  21. # the same module. Past it the module is marked stalled (a hung import) and later bounded callers
  22. # refuse at once until it finishes. Other callers (agent builds) wait for the load to finish.
  23. _CONCURRENT_LOAD_WAIT_SECS = 10.0
  24. _BOUNDED_WAIT: ContextVar[bool] = ContextVar("plugin_load_bounded_wait", default=False)
  25. _STALLED_LOADS: set[str] = set()
  26. _LOAD_OWNERS: dict[str, int] = {} # module -> thread id running its load
  27. _LOAD_WAITERS: dict[int, str] = {} # thread id -> module whose load it waits for
  28. def _waits_for(tid: int) -> set:
  29. """Threads *tid* is blocked on: a loader lock owner, or an importlib module-lock owner."""
  30. owners = {_LOAD_OWNERS.get(_LOAD_WAITERS.get(tid, ""))}
  31. blocked = getattr(importlib._bootstrap, "_blocking_on", {}).get(tid) # list on 3.12+, lock on 3.11
  32. owners.update(getattr(lk, "owner", None) for lk in (blocked if isinstance(blocked, list) else [blocked]))
  33. return owners - {None}
  34. def _load_would_deadlock(me: int) -> bool:
  35. """Whether the wait graph (loader locks + importlib's module locks) leads back to *me*.
  36. importlib cannot see the loader lock in its own deadlock check, so walk both kinds here."""
  37. seen, todo = set(), list(_waits_for(me))
  38. while todo:
  39. tid = todo.pop()
  40. if tid == me:
  41. return True
  42. if tid not in seen:
  43. seen.add(tid)
  44. todo.extend(_waits_for(tid))
  45. return False
  46. @contextlib.contextmanager
  47. def bounded_load_wait():
  48. """For per-turn callers that must not stall on another thread's hung plugin import: their
  49. loads give up (None) after ``_CONCURRENT_LOAD_WAIT_SECS`` instead of waiting it out."""
  50. token = _BOUNDED_WAIT.set(True)
  51. try:
  52. yield
  53. finally:
  54. _BOUNDED_WAIT.reset(token)
  55. def _module_load_lock(module_name: str) -> threading.RLock:
  56. with _MODULE_LOAD_LOCKS_GUARD:
  57. return _MODULE_LOAD_LOCKS.setdefault(module_name, threading.RLock())
  58. def register_synthetic_package(name: str, search_locations: list[str]) -> None:
  59. """Register an empty package shell so ``<name>.<child>`` relative imports resolve."""
  60. if name in sys.modules:
  61. return
  62. spec = importlib.machinery.ModuleSpec(name, None, is_package=True)
  63. spec.submodule_search_locations = search_locations
  64. sys.modules[name] = importlib.util.module_from_spec(spec)
  65. def user_plugins_dir() -> Optional[Path]:
  66. """Return ``$HERMES_HOME/plugins/`` or None if unavailable."""
  67. try:
  68. from hermes_constants import get_hermes_home
  69. d = get_hermes_home() / "plugins"
  70. return d if d.is_dir() else None
  71. except Exception:
  72. return None
  73. def iter_plugin_dirs(root: Path) -> list[Path]:
  74. """Sorted child dirs of *root* that have an ``__init__.py`` (skips ``_``/``.`` names)."""
  75. if not root.is_dir():
  76. return []
  77. dirs: list[Path] = []
  78. for child in sorted(root.iterdir()):
  79. if child.name.startswith(("_", ".")):
  80. continue
  81. try:
  82. if child.is_dir() and (child / "__init__.py").exists():
  83. dirs.append(child)
  84. except OSError as exc: # one mode-000 / ACL-denied child must not abort the listing
  85. _log.warning("Skipping unreadable plugin directory %s: %s", child, exc)
  86. return dirs
  87. def read_plugin_description(plugin_dir: Path) -> str:
  88. """Return ``description`` from ``plugin.yaml`` (empty string if absent/unreadable)."""
  89. try:
  90. from utils import fast_safe_load
  91. with open(plugin_dir / "plugin.yaml", encoding="utf-8-sig") as f:
  92. meta = fast_safe_load(f) or {}
  93. return meta.get("description", "")
  94. except Exception:
  95. return ""
  96. def _new_module(name: str, file: Path, search_locations: Optional[list[str]] = None) -> Optional[Any]:
  97. """spec -> module -> sys.modules[name] (NOT executed); None if no spec."""
  98. spec = importlib.util.spec_from_file_location(
  99. name, str(file), submodule_search_locations=search_locations)
  100. if not spec:
  101. return None
  102. mod = importlib.util.module_from_spec(spec)
  103. sys.modules[name] = mod
  104. return mod
  105. def _exec(mod: Any, logger: Optional[logging.Logger] = None) -> bool:
  106. """Exec a ``_new_module`` module (None -> False); False + debug-log if it raised. The sys.modules
  107. entry stays on failure; callers needing a clean retry pop it themselves."""
  108. if mod is None:
  109. return False
  110. try:
  111. mod.__spec__.loader.exec_module(mod)
  112. return True
  113. except Exception as e:
  114. if logger:
  115. logger.debug("Failed to exec_module %s: %s", mod.__name__, e)
  116. return False
  117. def load_plugin_module(module_name: str, plugin_dir: Path, *, parents: tuple[str, ...],
  118. logger: logging.Logger, synthetic_namespace: Optional[str] = None) -> Optional[Any]:
  119. """Import ``plugin_dir/__init__.py`` as *module_name* (reusing sys.modules when loaded).
  120. Order matters: parents first (relative imports need them), then siblings as ``module_name.<stem>``
  121. (so ``from ._x import Y`` resolves), then the module. Finally child is bound onto parent and
  122. siblings onto module — the shape normal imports produce, which monkeypatch relies on."""
  123. init_file = plugin_dir / "__init__.py"
  124. if not init_file.exists():
  125. return None
  126. if synthetic_namespace: # user code: never imported in-process under plugins.isolation: host
  127. from hermes_cli.plugin_isolation import in_process_import_refusal
  128. refusal = in_process_import_refusal(f"plugin {plugin_dir.name!r} (loaded as {module_name})")
  129. if refusal:
  130. logger.warning("%s", refusal)
  131. return None
  132. # _new_module publishes a package shell before executing it so sibling relative
  133. # imports work. Serialize the complete load so another thread cannot observe
  134. # that half-built shell as a loaded plugin.
  135. # The wait polls for a cross-thread cycle through importlib's module locks; in a cycle, accept the
  136. # partial module like importlib does. It is bounded only inside bounded_load_wait() (per-turn callers).
  137. lock, me = _module_load_lock(module_name), threading.get_ident()
  138. deadline = float("inf") if not _BOUNDED_WAIT.get() else time.monotonic() + (
  139. 0 if module_name in _STALLED_LOADS else _CONCURRENT_LOAD_WAIT_SECS)
  140. _LOAD_WAITERS[me] = module_name
  141. try:
  142. while not lock.acquire(timeout=0.05):
  143. if _load_would_deadlock(me):
  144. logger.debug("Concurrent circular load of %s; using the partial module", module_name)
  145. return sys.modules.get(module_name)
  146. if time.monotonic() >= deadline:
  147. _STALLED_LOADS.add(module_name)
  148. logger.warning("Skipping plugin %s: another thread's load of it has not finished "
  149. "(import still running after %.0fs)", module_name, _CONCURRENT_LOAD_WAIT_SECS)
  150. return None
  151. finally:
  152. _LOAD_WAITERS.pop(me, None)
  153. outermost = module_name not in _LOAD_OWNERS # the RLock re-enters on this thread's recursive loads
  154. _LOAD_OWNERS.setdefault(module_name, me)
  155. try:
  156. return _load_plugin_module_locked(module_name, plugin_dir, init_file, parents, logger,
  157. synthetic_namespace)
  158. finally:
  159. if outermost:
  160. _LOAD_OWNERS.pop(module_name, None)
  161. _STALLED_LOADS.discard(module_name)
  162. lock.release()
  163. def _load_plugin_module_locked(module_name: str, plugin_dir: Path, init_file: Path,
  164. parents: tuple[str, ...], logger: logging.Logger,
  165. synthetic_namespace: Optional[str]) -> Optional[Any]:
  166. # A synthetic package shell has no __file__; only reuse modules loaded from disk.
  167. cached = sys.modules.get(module_name)
  168. if cached is not None and getattr(cached, "__file__", None):
  169. return cached
  170. for parent in parents:
  171. parent_path = _PLUGINS_ROOT.joinpath(*parent.split(".")[1:])
  172. if parent not in sys.modules and (parent_path / "__init__.py").exists():
  173. _exec(_new_module(parent, parent_path / "__init__.py", [str(parent_path)]))
  174. if synthetic_namespace:
  175. register_synthetic_package(synthetic_namespace, [])
  176. # Reserve the name before siblings exec so their relative imports resolve.
  177. mod = _new_module(module_name, init_file, [str(plugin_dir)])
  178. if mod is None:
  179. return None
  180. loaded_submodules = []
  181. for sub_file in plugin_dir.glob("*.py"):
  182. full_sub_name = f"{module_name}.{sub_file.stem}"
  183. if sub_file.name == "__init__.py" or full_sub_name in sys.modules:
  184. continue
  185. sub_mod = _new_module(full_sub_name, sub_file)
  186. if _exec(sub_mod, logger):
  187. loaded_submodules.append((sub_file.stem, sub_mod))
  188. else:
  189. sys.modules.pop(full_sub_name, None)
  190. if not _exec(mod, logger):
  191. sys.modules.pop(module_name, None)
  192. return None
  193. parent_name, child_name = module_name.rsplit(".", 1)
  194. parent_mod = sys.modules.get(parent_name)
  195. if parent_mod is not None:
  196. setattr(parent_mod, child_name, mod)
  197. for sub_name, sub_mod in loaded_submodules:
  198. setattr(mod, sub_name, sub_mod)
  199. return mod
  200. class NoopPluginContext:
  201. """Base for fake ``register(ctx)`` contexts: no-op registrations except the one a subclass overrides."""
  202. def _noop(self, *args, **kwargs):
  203. pass
  204. register_tool = register_hook = register_cli_command = register_memory_provider = _noop
  205. def instance_from_module(mod: Any, *, collector: Any, collected_attr: str, base_cls: type, name: str,
  206. logger: logging.Logger) -> Optional[Any]:
  207. """Extract the provider instance: ``register(ctx)`` first, then any ``base_cls`` subclass."""
  208. if hasattr(mod, "register"):
  209. try:
  210. mod.register(collector)
  211. instance = getattr(collector, collected_attr)
  212. if instance:
  213. return instance
  214. except Exception as e:
  215. logger.debug("register() failed for %s: %s", name, e)
  216. for attr_name in dir(mod):
  217. attr = getattr(mod, attr_name, None)
  218. if isinstance(attr, type) and issubclass(attr, base_cls) and attr is not base_cls:
  219. with contextlib.suppress(Exception):
  220. return attr()
  221. return None
  222. def load_named(name: str, plugin_dir: Path, load_from_dir: Callable[[Path], Optional[Any]], *, kind: str,
  223. noun: str, logger: logging.Logger) -> Optional[Any]:
  224. """Shared body of ``load_<kind>(name)``: load from *plugin_dir*, warn + None on failure."""
  225. try:
  226. instance = load_from_dir(plugin_dir)
  227. except Exception as e:
  228. logger.warning("Failed to load %s '%s': %s", kind.lower(), name, e)
  229. return None
  230. if not instance:
  231. logger.warning("%s '%s' loaded but no %s instance found", kind, name, noun)
  232. return instance or None
  233. def probe_availability(load: Callable[[], Optional[Any]]) -> bool:
  234. """True iff *load()* returns an instance whose ``is_available()`` (if any) is truthy."""
  235. try:
  236. instance = load()
  237. return instance is not None and (instance.is_available() if hasattr(instance, "is_available") else True)
  238. except Exception:
  239. return False