plugin_isolation_audit.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. """Static answer to "will this plugin run in the plugin host?" (``plugins.isolation: host``).
  2. Reads a plugin directory's manifest and Python source (never imports it) and reports whether the
  3. plugin reaches Hermes only through ``ctx`` surfaces that cross the host boundary. The rules come
  4. from :mod:`hermes_cli.plugin_isolation`, the same tables the host enforces at runtime, so a plugin
  5. the audit calls host-ready loads in the host, and one it flags fails there with the same reason.
  6. What it cannot see in a process boundary it flags:
  7. * ctx methods that hand over live objects (platform adapters, SDK clients, argparse);
  8. * writing attributes of Hermes modules (monkeypatching): the patch lands in the host's copy;
  9. * mutating a global registry directly instead of through ``ctx``: the registration lands in the
  10. host's registry, which Hermes never reads;
  11. * gateway platform adapters (``kind: platform``), model-provider profiles that build their own SDK
  12. client (``create_client``), and dashboard APIs that stream (the host bridge buffers responses).
  13. ``hermes plugins validate`` prints the verdict; ``evals/plugin_isolation/audit_catalog.py`` runs it
  14. over every catalog entry.
  15. """
  16. from __future__ import annotations
  17. import ast
  18. from dataclasses import dataclass, field
  19. from pathlib import Path
  20. from typing import Any, Dict, List, Mapping, Optional, Set
  21. VERDICT_HOST = "host"
  22. VERDICT_IN_PROCESS = "in_process"
  23. VERDICT_PORTABLE = "portable"
  24. # Top-level packages that are Hermes's own process state when imported by a plugin.
  25. HERMES_PACKAGES: frozenset = frozenset({
  26. "agent", "tools", "hermes_cli", "gateway", "cron", "providers", "plugins", "tui_gateway",
  27. "acp_adapter", "run_agent", "cli", "model_tools", "toolsets", "hermes_state", "hermes_constants",
  28. "hermes_logging", "hermes_time", "utils", "registration_lifecycle", "hermes_platform",
  29. })
  30. # ``<module>.<callable>`` that mutate a Hermes-process registry when called from plugin code.
  31. _DIRECT_REGISTRY_CALLS: frozenset = frozenset({
  32. ("tools.registry", "register"), ("tools.registry", "deregister"),
  33. ("gateway.platform_registry", "register"), ("providers", "register_provider"),
  34. })
  35. _MANIFEST_KIND_REASONS: dict[str, str] = {
  36. "platform": "kind 'platform': gateway platform adapters run in the Hermes process",
  37. }
  38. _STREAMING_MARKERS = ("StreamingResponse", "EventSourceResponse", ".websocket(", "WebSocket")
  39. _SKIP_DIRS = frozenset({"tests", "test", ".git", "__pycache__", "node_modules", ".venv", "venv", "docs"})
  40. @dataclass
  41. class IsolationReport:
  42. verdict: str
  43. reasons: list[str] = field(default_factory=list)
  44. notes: list[str] = field(default_factory=list)
  45. ctx_methods: set[str] = field(default_factory=set)
  46. @property
  47. def host_ready(self) -> bool:
  48. return self.verdict in {VERDICT_HOST, VERDICT_PORTABLE}
  49. def summary(self) -> str:
  50. if self.verdict == VERDICT_PORTABLE:
  51. return "portable plugin (no Python): runs anywhere"
  52. if self.verdict == VERDICT_HOST:
  53. return "runs in the plugin host (plugins.isolation: host)"
  54. return "in-process only: " + "; ".join(self.reasons)
  55. def to_dict(self) -> dict[str, Any]:
  56. return {"verdict": self.verdict, "host_ready": self.host_ready, "reasons": list(self.reasons),
  57. "notes": list(self.notes), "ctx_methods": sorted(self.ctx_methods)}
  58. def _python_files(plugin_dir: Path) -> list[Path]:
  59. files = []
  60. for path in sorted(plugin_dir.rglob("*.py")):
  61. rel = path.relative_to(plugin_dir).parts
  62. if any(part in _SKIP_DIRS or part.startswith(".") for part in rel[:-1]):
  63. continue
  64. if rel[-1].startswith("test_") or rel[-1] == "conftest.py":
  65. continue
  66. files.append(path)
  67. return files
  68. class _SourceVisitor(ast.NodeVisitor):
  69. """Collects ctx method calls, Hermes-module aliases, and writes through those aliases."""
  70. def __init__(self, rel: str, *, model_provider: bool = False):
  71. from hermes_cli.plugin_isolation import (
  72. HOST_DEGRADED_HOOKS, HOST_OBJECT_BASES, HOST_SKIPPED_CTX_METHODS, HOST_UNSUPPORTED_CTX_METHODS,
  73. )
  74. self._unsupported, self._skipped = HOST_UNSUPPORTED_CTX_METHODS, HOST_SKIPPED_CTX_METHODS
  75. self._degraded, self._object_methods = HOST_DEGRADED_HOOKS, HOST_OBJECT_BASES
  76. self.rel = rel
  77. # A model-provider plugin's register_provider() call is its contract; the host captures it.
  78. self.model_provider = model_provider
  79. self.aliases: dict[str, str] = {} # local name -> hermes module path (or module.attr)
  80. self.reasons: list[str] = []
  81. self.notes: list[str] = []
  82. self.ctx_methods: set[str] = set()
  83. def _hermes(self, module: str) -> bool:
  84. return module.split(".")[0] in HERMES_PACKAGES
  85. def visit_Import(self, node: ast.Import) -> None:
  86. for alias in node.names:
  87. if self._hermes(alias.name):
  88. self.aliases[alias.asname or alias.name.split(".")[0]] = (
  89. alias.name if alias.asname else alias.name.split(".")[0])
  90. def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
  91. if node.level == 0 and node.module and self._hermes(node.module):
  92. for alias in node.names:
  93. self.aliases[alias.asname or alias.name] = f"{node.module}.{alias.name}"
  94. def _resolve(self, node: ast.AST) -> Optional[str]:
  95. if isinstance(node, ast.Name):
  96. return self.aliases.get(node.id)
  97. if isinstance(node, ast.Attribute):
  98. base = self._resolve(node.value)
  99. return f"{base}.{node.attr}" if base else None
  100. return None
  101. def _flag_write(self, target: ast.AST, line: int) -> None:
  102. if isinstance(target, ast.Attribute):
  103. owner = self._resolve(target.value)
  104. if owner:
  105. self.reasons.append(f"{self.rel}:{line} patches {owner}.{target.attr} (a Hermes module "
  106. f"attribute; the host's copy is not the one Hermes runs)")
  107. def visit_Assign(self, node: ast.Assign) -> None:
  108. for target in node.targets:
  109. self._flag_write(target, node.lineno)
  110. self.generic_visit(node)
  111. def visit_AugAssign(self, node: ast.AugAssign) -> None:
  112. self._flag_write(node.target, node.lineno)
  113. self.generic_visit(node)
  114. def visit_Call(self, node: ast.Call) -> None:
  115. func = node.func
  116. if isinstance(func, ast.Name) and func.id == "setattr" and node.args:
  117. owner = self._resolve(node.args[0])
  118. if owner:
  119. self.reasons.append(f"{self.rel}:{node.lineno} setattr() on {owner} (patches a Hermes module)")
  120. if isinstance(func, ast.Attribute):
  121. name = func.attr
  122. if name in self._unsupported:
  123. self.reasons.append(f"{self.rel}:{node.lineno} ctx.{name}(): {self._unsupported[name]}")
  124. elif name in self._skipped:
  125. self.notes.append(f"ctx.{name}() is skipped in the host ({self._skipped[name]})")
  126. if name.startswith("register_") or name in {"subscribe", "dispatch_tool", "inject_message", "emit"}:
  127. self.ctx_methods.add(name)
  128. if name == "register_hook" and node.args and isinstance(node.args[0], ast.Constant) \
  129. and node.args[0].value in self._degraded:
  130. hook = node.args[0].value
  131. self.notes.append(f"hook {hook} {self._degraded[hook]}; those fields are placeholders in the host")
  132. self._flag_registry_call(self._resolve(func), node.lineno)
  133. elif isinstance(func, ast.Name):
  134. self._flag_registry_call(self.aliases.get(func.id), node.lineno)
  135. self.generic_visit(node)
  136. def _flag_registry_call(self, target: Optional[str], line: int) -> None:
  137. if not target:
  138. return
  139. module, _, attr = target.rpartition(".")
  140. if self.model_provider and (module, attr) == ("providers", "register_provider"):
  141. return
  142. if (module, attr) in _DIRECT_REGISTRY_CALLS or (
  143. module.endswith("_registry") and attr in {"register", "register_provider"}):
  144. self.reasons.append(f"{self.rel}:{line} calls {target}() directly instead of "
  145. f"through ctx (registers in the host, not in Hermes)")
  146. def visit_ClassDef(self, node: ast.ClassDef) -> None:
  147. if self.model_provider and any(isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef))
  148. and item.name == "create_client" for item in node.body):
  149. self.reasons.append(f"{self.rel}:{node.lineno} {node.name}.create_client() returns a live SDK "
  150. f"client (model-provider profiles cross the host as data + methods)")
  151. self.generic_visit(node)
  152. def audit_plugin_dir(plugin_dir: Path, manifest: Optional[Mapping[str, Any]] = None) -> IsolationReport:
  153. """Classify one plugin directory without importing it."""
  154. plugin_dir = Path(plugin_dir)
  155. if manifest is None:
  156. manifest = _read_manifest(plugin_dir)
  157. if not (plugin_dir / "__init__.py").is_file():
  158. return IsolationReport(VERDICT_PORTABLE)
  159. reasons: list[str] = []
  160. notes: list[str] = []
  161. ctx_methods: set[str] = set()
  162. kind = str((manifest or {}).get("kind") or "").strip().lower()
  163. if kind in _MANIFEST_KIND_REASONS:
  164. reasons.append(_MANIFEST_KIND_REASONS[kind])
  165. dashboard = plugin_dir / "dashboard"
  166. if (dashboard / "manifest.json").is_file() and '"api"' in (dashboard / "manifest.json").read_text(
  167. encoding="utf-8", errors="replace"):
  168. api_source = "".join(p.read_text(encoding="utf-8-sig", errors="replace") for p in dashboard.rglob("*.py"))
  169. if any(marker in api_source for marker in _STREAMING_MARKERS):
  170. reasons.append("dashboard backend API streams (SSE/websocket); the host bridge buffers responses")
  171. else:
  172. notes.append("dashboard backend API is served through the plugin host (buffered responses)")
  173. for path in _python_files(plugin_dir):
  174. rel = str(path.relative_to(plugin_dir))
  175. try:
  176. tree = ast.parse(path.read_text(encoding="utf-8-sig", errors="replace"), filename=rel)
  177. except SyntaxError as exc:
  178. notes.append(f"{rel}: not parseable ({exc.msg}); not audited")
  179. continue
  180. visitor = _SourceVisitor(rel, model_provider=kind == "model-provider")
  181. visitor.visit(tree)
  182. reasons.extend(visitor.reasons)
  183. notes.extend(visitor.notes)
  184. ctx_methods |= visitor.ctx_methods
  185. verdict = VERDICT_IN_PROCESS if reasons else VERDICT_HOST
  186. return IsolationReport(verdict, _dedupe(reasons), _dedupe(notes), ctx_methods)
  187. def _dedupe(items: list[str]) -> list[str]:
  188. return list(dict.fromkeys(items))
  189. def _read_manifest(plugin_dir: Path) -> dict[str, Any]:
  190. for name in ("plugin.yaml", "plugin.yml"):
  191. path = plugin_dir / name
  192. if path.is_file():
  193. try:
  194. import hermes_yaml as yaml
  195. data = yaml.safe_load(path.read_text(encoding="utf-8-sig"))
  196. return data if isinstance(data, dict) else {}
  197. except Exception:
  198. return {}
  199. return {}