extras.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. """Extras: the runtime features of the python venv.
  2. Feature names ARE pyproject extra names. This table maps each extra to the
  3. import that proves it (the anchor), so availability is one find_spec — no
  4. package-manager machinery on the hot path. sync_venv([extra]) makes an extra
  5. true; pm owns HOW (uv sync inside the venv package).
  6. """
  7. from __future__ import annotations
  8. import importlib.util
  9. from pathlib import Path
  10. from typing import Callable
  11. # extra name -> module that proves it is installed
  12. ANCHORS: dict[str, str | tuple[str, ...]] = {
  13. "anthropic": "anthropic",
  14. "bedrock": "boto3",
  15. "vertex": "google.auth",
  16. "azure-identity": "azure.identity",
  17. "exa": "exa_py",
  18. "firecrawl": "firecrawl",
  19. "parallel-web": "parallel",
  20. "ddgs": "ddgs",
  21. "otlp": "opentelemetry.sdk",
  22. "mistral": "mistralai",
  23. "edge-tts": "edge_tts",
  24. "neutts": "neutts",
  25. "kittentts": ("kittentts", "soundfile"),
  26. "piper": "piper",
  27. "tts-premium": "elevenlabs",
  28. "voice": "faster_whisper",
  29. "stt-whisper": "faster_whisper",
  30. "audio-io": ("sounddevice", "numpy"),
  31. "silk": "pilk",
  32. "wake": "pyopen_wakeword",
  33. "wake-openwakeword": "pyopen_wakeword",
  34. "wake-sherpa": "sherpa_onnx",
  35. "wake-porcupine": "pvporcupine",
  36. "fal": "fal_client",
  37. "messaging": "telegram",
  38. "telegram": "telegram",
  39. "discord": "discord",
  40. "slack": "slack_bolt",
  41. "matrix": ("mautrix", "asyncpg", "aiosqlite", "markdown", "aiohttp_socks"),
  42. "dingtalk": "dingtalk_stream",
  43. "feishu": "lark_oapi",
  44. "wecom": "defusedxml",
  45. "keychain": "keyring",
  46. "teams": "microsoft_teams.apps",
  47. "modal": "modal",
  48. "daytona": "daytona",
  49. "vercel": "vercel",
  50. "google": ("googleapiclient", "google.auth", "google_auth_oauthlib.flow", "google_auth_httplib2"),
  51. "google-chat": "google.cloud.pubsub_v1",
  52. "google-meet": ("playwright.sync_api", "websockets"),
  53. "youtube": "youtube_transcript_api",
  54. "acp": "acp",
  55. "web": "fastapi",
  56. "doc-extract": "anydoc",
  57. "documents": ("docx", "openpyxl", "pptx", "reportlab", "pypdf"),
  58. "documents-pdf": ("pdfplumber", "pypdfium2"),
  59. "computer-use": "mcp",
  60. "trace-upload": "huggingface_hub",
  61. # Pillow resize recovery for vision tools (the `vision` extra is a no-op
  62. # back-compat alias — Pillow is core — but ensure_import("vision") must
  63. # still resolve an anchor so availability checks work).
  64. "vision": "PIL",
  65. }
  66. def _anchors(extra: str) -> tuple[str, ...]:
  67. got = ANCHORS.get(extra, extra.replace("-", "_"))
  68. return got if isinstance(got, tuple) else (got,)
  69. def _importable(anchor: str) -> bool:
  70. """An anchor already present in sys.modules counts even without a
  71. findable spec — tests fake SDKs by inserting modules there, and the
  72. caller's import right after this check resolves the same way."""
  73. import sys
  74. if anchor in sys.modules:
  75. return True
  76. try:
  77. return importlib.util.find_spec(anchor) is not None
  78. except (ImportError, ValueError):
  79. return False
  80. def available(extra: str) -> bool:
  81. """Fast, side-effect-free: are all of this extra's anchors importable?
  82. A platform-gated extra whose gate excludes this machine reads as
  83. unavailable — the anchors are absent by design, not by accident."""
  84. if not extra_supported(extra):
  85. return False
  86. return all(_importable(a) for a in _anchors(extra))
  87. def _platform_gates() -> dict[str, str]:
  88. """The [tool.hermes.extras-platforms] table from pyproject.toml:
  89. extra -> PEP 508 marker string. Cached per process."""
  90. global _PLATFORM_GATES
  91. if _PLATFORM_GATES is not None:
  92. return _PLATFORM_GATES
  93. import tomllib
  94. from pm.paths import repo_root
  95. gates: dict[str, str] = {}
  96. try:
  97. with (repo_root() / "pyproject.toml").open("rb") as f:
  98. data = tomllib.load(f)
  99. table = data.get("tool", {}).get("hermes", {}).get("extras-platforms", {})
  100. if isinstance(table, dict):
  101. gates = {str(k): str(v) for k, v in table.items()}
  102. except (OSError, ValueError):
  103. pass
  104. _PLATFORM_GATES = gates
  105. return gates
  106. _PLATFORM_GATES: dict[str, str] | None = None
  107. def extra_supported(extra: str, *, environment: dict[str, str] | None = None,
  108. importable: Callable[[str], bool] | None = None) -> bool:
  109. """Is this extra installable on THIS platform? True when the extra
  110. carries no gate, or its marker matches the running platform. An
  111. extra that IS present on this machine (anchors importable) is always
  112. supported — an installed override beats the table (dev machines,
  113. hand-synced venvs)."""
  114. if all((importable or _importable)(a) for a in _anchors(extra)):
  115. return True
  116. marker = _platform_gates().get(extra)
  117. if marker is None:
  118. return True
  119. import os
  120. import platform
  121. import sys
  122. if environment is None:
  123. environment = {
  124. "sys_platform": sys.platform,
  125. "platform_system": platform.system(),
  126. "platform_machine": platform.machine(),
  127. "os_name": os.name,
  128. }
  129. try:
  130. from packaging.markers import Marker
  131. except ImportError:
  132. # The historical update takeover runs PM in the OLD venv's
  133. # interpreter, which need not ship `packaging`. PM's runtime does.
  134. return _evaluate_in_runtime(marker, environment)
  135. try:
  136. return bool(Marker(marker).evaluate(environment=environment))
  137. except Exception:
  138. # A malformed marker must never brick availability — treat as
  139. # ungated and let the resolver be the authority.
  140. return True
  141. def _evaluate_in_runtime(marker: str, environment: dict[str, str]) -> bool:
  142. import json
  143. import subprocess
  144. from pm.runtime import runtime_command, runtime_environment
  145. try:
  146. command = runtime_command(Path(__file__).with_name("_marker_eval.py"),
  147. [marker, json.dumps(environment)])
  148. result = subprocess.run(command, env=runtime_environment(), capture_output=True,
  149. text=True, timeout=60, check=False)
  150. except Exception:
  151. return True # same fallback as a malformed marker: the resolver decides
  152. if result.returncode != 0:
  153. return True
  154. return result.stdout.strip() == "1"
  155. def install_hint(extra: str) -> str:
  156. """The one command users are told to run for a missing extra."""
  157. return f"hermes pm install --extra {extra}"
  158. def ensure_import(extra: str) -> None:
  159. """Make an extra available: no-op when the anchor imports, otherwise
  160. sync the venv with the extra enabled. Raises InstallError on failure
  161. — including when a platform gate excludes this machine."""
  162. if available(extra):
  163. return
  164. if not extra_supported(extra):
  165. from pm.package import InstallError
  166. marker = _platform_gates().get(extra, "")
  167. raise InstallError(
  168. "venv",
  169. f"extra {extra!r} is not supported on this platform "
  170. f"(gate: {marker!r}); the adapter degrades without it",
  171. )
  172. import sys
  173. from pm.package import InstallError
  174. # prompt_toolkit already owns stdin during a CLI turn; never read from it.
  175. app_running = False
  176. if "prompt_toolkit.application.current" in sys.modules:
  177. from prompt_toolkit.application.current import get_app_or_none
  178. app_running = bool(getattr(get_app_or_none(), "is_running", False))
  179. if not app_running and sys.stdin.isatty() and sys.stdout.isatty():
  180. try:
  181. answer = input(f"\nThis needs Hermes' optional {extra!r} feature, which isn't installed yet.\n"
  182. "Install it now? [Y/n] ").strip().lower()
  183. except (EOFError, KeyboardInterrupt):
  184. answer = "n"
  185. if answer and answer not in {"y", "yes"}:
  186. raise InstallError("venv", f"installation of extra {extra!r} declined")
  187. from pm.client import sync_venv
  188. sync_venv([extra])
  189. # The sync published a new generation. Swap this process onto it when nothing
  190. # already imported would change underneath it (adopt_selected); otherwise only
  191. # a restart can load it.
  192. from pm.environments import selected_venv, site_packages
  193. from pm.environments_adopt import adopt_selected, note_restart_pending, restart_needed
  194. from pm.paths import repo_root, runtime_facts_path
  195. import sys
  196. from pathlib import Path
  197. if runtime_facts_path().is_file():
  198. root = repo_root()
  199. adopt_selected(root)
  200. selected = site_packages(selected_venv(root)).resolve()
  201. if selected not in {Path(entry).resolve() for entry in sys.path}:
  202. reason = restart_needed(root) or "this process does not run from the install's dependency environment"
  203. note_restart_pending(f"{extra} installed ({reason})")
  204. raise InstallError("venv", f"{extra} installed; restart Hermes to activate it ({reason})")
  205. def ensure_and_bind(extra, importer, target_globals) -> bool:
  206. """ensure_import + rebind module-level names after a mid-process install.
  207. importer returns {name: value}; bound into target_globals on success."""
  208. try:
  209. ensure_import(extra)
  210. except Exception as exc:
  211. import logging
  212. logging.getLogger(__name__).warning("extra %r unavailable: %s", extra, exc)
  213. return False
  214. try:
  215. bindings = importer()
  216. except ImportError as exc:
  217. import logging
  218. logging.getLogger(__name__).warning(
  219. "import after installing %r failed: %s", extra, exc
  220. )
  221. return False
  222. target_globals.update(bindings)
  223. return True
  224. def missing(extra: str) -> tuple[str, ...]:
  225. return tuple(a for a in _anchors(extra) if not _importable(a))
  226. def _installed_in(site_packages: Path, anchor: str) -> bool:
  227. """Is ``anchor`` installed under a site-packages we must not import from?"""
  228. *parents, leaf = anchor.split(".")
  229. parent = site_packages.joinpath(*parents)
  230. if (parent / leaf).is_dir():
  231. return True
  232. # A top-level module can be a plain file or a compiled extension.
  233. return any(entry.name.rsplit(".", 1)[-1] in {"py", "so", "pyd"}
  234. for entry in parent.glob(f"{leaf}.*"))
  235. DECLINED_EXTRAS_FILENAME = "declined-extras.json"
  236. def declined_extras_path(project_root: Path | None = None) -> Path:
  237. from pm.environments import install_state_dir
  238. from pm.paths import repo_root
  239. return install_state_dir(repo_root() if project_root is None else Path(project_root)) / DECLINED_EXTRAS_FILENAME
  240. def declined_extras(project_root: Path | None = None) -> frozenset[str]:
  241. """Extras the user dropped with ``hermes pm install --without NAME``.
  242. Recorded beside the dependency facts, so a carried selection (legacy
  243. migration, recorded extras, configured-feature auto-installs) never
  244. re-adds one; only an explicit request (``--extra NAME``) opts back in.
  245. """
  246. import json
  247. path = declined_extras_path(project_root)
  248. try:
  249. data = json.loads(path.read_text(encoding="utf-8-sig"))
  250. except FileNotFoundError:
  251. return frozenset()
  252. except (OSError, ValueError) as exc:
  253. # Reading a damaged record as "nothing declined" would re-add what the user removed.
  254. from pm.package import InstallError
  255. raise InstallError("venv", f"cannot read {path}: {exc}", "fix or delete the file, then retry") from exc
  256. names = data.get("declined") if isinstance(data, dict) else None
  257. return frozenset(_canonical(name) for name in names or () if isinstance(name, str))
  258. def record_declined_extras(*, add=(), remove=(), reasons: dict[str, str] | None = None,
  259. project_root: Path | None = None) -> frozenset[str]:
  260. """Update the declined extras; returns the new set. Writes only on change.
  261. *reasons* marks extras an update left out on its own (``pm.carried_extras``) with why,
  262. so a later refusal names the cause instead of a ``--without`` the user never ran.
  263. """
  264. import json
  265. from pm.filesystem import durable_write_bytes
  266. current = declined_extras(project_root)
  267. updated = (current | {_canonical(name) for name in add}) - {_canonical(name) for name in remove}
  268. known = declined_reasons(project_root)
  269. noted = {name: text for name, text in {**known, **{_canonical(name): " ".join(text.split())[:300]
  270. for name, text in (reasons or {}).items()}}.items()
  271. if name in updated}
  272. if updated != current or noted != known:
  273. record: dict = {"schema": 1, "declined": sorted(updated)}
  274. if noted:
  275. record["left_out_by_update"] = dict(sorted(noted.items()))
  276. durable_write_bytes(declined_extras_path(project_root), (json.dumps(record, indent=2) + "\n").encode("utf-8"))
  277. return frozenset(updated)
  278. def declined_reasons(project_root: Path | None = None) -> dict[str, str]:
  279. """Why an update left each automatically declined extra out (absent for a user's ``--without``)."""
  280. import json
  281. try:
  282. data = json.loads(declined_extras_path(project_root).read_text(encoding="utf-8-sig"))
  283. except (OSError, ValueError):
  284. return {} # declined_extras() is the reader that refuses a damaged record
  285. notes = data.get("left_out_by_update") if isinstance(data, dict) else None
  286. return {name: text for name, text in (notes or {}).items() if isinstance(name, str) and isinstance(text, str)}
  287. def _canonical(name: str) -> str:
  288. """The PEP 685 name uv matches an extra by (``foo_bar`` is ``foo-bar``)."""
  289. import re
  290. return re.sub(r"[-_.]+", "-", name).lower()
  291. def droppable_extras(project_root: Path) -> list[str]:
  292. """Declared extras ``--without`` may drop: not ``all`` and nothing ``all`` pulls in.
  293. The default sync selects ``[all]``, so dropping one of its members would be a no-op.
  294. """
  295. import re
  296. import tomllib
  297. with (Path(project_root) / "pyproject.toml").open("rb") as f:
  298. project = tomllib.load(f).get("project", {})
  299. table = project.get("optional-dependencies", {})
  300. self_ref = re.compile(rf"^{re.escape(project.get('name', ''))}\[([^\]]+)\]", re.IGNORECASE)
  301. kept, pending = {"all"}, ["all"]
  302. while pending:
  303. for requirement in table.get(pending.pop(), ()):
  304. match = self_ref.match(requirement.replace(" ", ""))
  305. for member in (match.group(1).split(",") if match else ()):
  306. if member not in kept:
  307. kept.add(member)
  308. pending.append(member)
  309. return sorted(extra for extra in table if extra not in kept)
  310. def legacy_selection(project_root: Path) -> list[str]:
  311. """The extras PM's first generation selects when it replaces a main-era venv.
  312. Main-era installers selected ``[all]`` and then lazily installed opt-in
  313. extras (FAL, messaging SDKs, ...) into the checkout's own venv, with no
  314. ledger. Selecting only ``[all]`` drops those, and the first launch after
  315. the migration asks to reinstall a feature that already worked. The old
  316. venv's site-packages is read, never imported: the migrating process may
  317. not run from it.
  318. """
  319. root = Path(project_root)
  320. trees = [tree for venv in (root / "venv", root / ".venv")
  321. for tree in (*venv.glob("lib/python*/site-packages"), venv / "Lib" / "site-packages")
  322. if tree.is_dir()]
  323. # The old venv is never deleted, so this scan repeats for any data root without facts.
  324. refused = declined_extras(root)
  325. carried = sorted(
  326. # An umbrella extra shares its anchor with one member; carrying it
  327. # would install every sibling the user never chose.
  328. extra for extra in ANCHORS if extra not in {"messaging", "voice", "wake"} and extra not in refused
  329. # PM refuses a gated extra outside its platform even if a hand-synced venv carried it.
  330. # Installed first: judging a gate may cost a PM-runtime subprocess.
  331. if any(all(_installed_in(tree, anchor) for anchor in _anchors(extra)) for tree in trees)
  332. and extra_supported(extra, importable=lambda _anchor: False)
  333. )
  334. return ["all", *carried]