desktop_plugin_halves.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. """Copy each unified agent+desktop package's ``desktop/`` half into the app-level desktop-plugins folder.
  2. A unified package keeps its agent half in ``<home>/plugins/<name>/`` (the default home and every
  3. profile). Its desktop half is loaded from ``<root>/desktop-plugins/<name>/``, with a
  4. ``.hermes-package.json`` marker that pairs it back to the agent row. Native Desktop does this copy
  5. in Electron (``reconcileUnifiedDesktopHalves`` in ``apps/desktop/electron/desktop-plugins-root.ts``),
  6. so a host that Desktop only reaches over SSH never gets it. The Webapp serves that host's folder,
  7. so its routes (``web_routers/desktop_plugins.py``) run this pass at the same points Electron does:
  8. whenever the loader asks for the root, and on demand after an install.
  9. Both writers share the folder on a host that runs native Desktop too, so this keeps Electron's
  10. on-disk contract exactly: the marker's fields, the first-home-wins order, adoption of a byte-equal
  11. marker-less copy, a byte compare for linked dev checkouts, staged publication, and a prune that
  12. only fires when the source is really gone.
  13. """
  14. from __future__ import annotations
  15. import json
  16. import logging
  17. import os
  18. import re
  19. import shutil
  20. import stat
  21. import tempfile
  22. import threading
  23. from pathlib import Path
  24. from pm.filesystem import is_junction
  25. from utils import rmtree_readonly
  26. logger = logging.getLogger(__name__)
  27. PACKAGE_MARKER = ".hermes-package.json"
  28. #: The whole published folder; the loader reads at most a 16 MB ``plugin.js`` from it.
  29. TREE_MAX_BYTES = 64 * 1024 * 1024
  30. #: Folders a root-layout half never publishes (Electron's ``ROOT_LAYOUT_SKIP`` matches).
  31. ROOT_LAYOUT_SKIP = (".git", "__pycache__")
  32. _GIT_ORIGIN = re.compile(r'\[remote "origin"\][^[]*?url\s*=\s*(\S+)')
  33. #: Electron serializes every writer of the root in its main process; the server's threads do here.
  34. #: Public: ``desktop_plugin_install`` takes it too, so a clone-publish never interleaves with a
  35. #: reconcile pass copying the same package's folder.
  36. ROOT_WRITER = threading.Lock()
  37. def _is_link(path: Path) -> bool:
  38. """A symlink or a Windows junction: Node's ``lstat`` reports both as a link, and Electron's
  39. checks of the same folder go by that. ``Path.is_symlink()`` alone misses a junction."""
  40. return path.is_symlink() or is_junction(path)
  41. def _list_dirs(folder: Path, *, follow_links: bool = False) -> list[str]:
  42. """Folder names in *folder*. A package root follows links to folders (``ln -s ~/src/pkg
  43. ~/.hermes/plugins/`` is how a package is developed in place), the app root does not."""
  44. try:
  45. entries = list(os.scandir(folder))
  46. except OSError:
  47. return []
  48. names = []
  49. for entry in entries:
  50. try:
  51. if entry.is_dir(follow_symlinks=False) or (follow_links and entry.is_symlink() and entry.is_dir()):
  52. names.append(entry.name)
  53. except OSError:
  54. continue
  55. return names
  56. def _package_origin(package_dir: Path) -> dict:
  57. """Where the package came from (catalog sidecar first, then the git remote), for "Install here"."""
  58. try:
  59. sidecar = json.loads((package_dir / ".hermes-catalog.json").read_text(encoding="utf-8-sig"))
  60. if sidecar.get("repo"):
  61. origin = {"catalogName": sidecar.get("catalog_name"), "repo": sidecar["repo"], "sha": sidecar.get("sha")}
  62. return {key: value for key, value in origin.items() if value is not None}
  63. except (OSError, ValueError, AttributeError):
  64. pass
  65. try:
  66. match = _GIT_ORIGIN.search((package_dir / ".git" / "config").read_text(encoding="utf-8-sig"))
  67. except OSError:
  68. return {}
  69. return {"repo": match.group(1)} if match else {}
  70. def mtime_ms(st: os.stat_result) -> float:
  71. """``Stats.mtimeMs`` exactly as Node computes it (seconds * 1e3 + nanoseconds / 1e6). Both
  72. writers compare this number against the marker the other may have written, so a value one ulp
  73. off would make each pass see the other's copy as stale and re-copy it."""
  74. seconds, nanoseconds = divmod(st.st_mtime_ns, 1_000_000_000)
  75. return seconds * 1e3 + nanoseconds / 1e6
  76. def read_package_marker(folder: Path) -> dict | None:
  77. try:
  78. marker = json.loads((folder / PACKAGE_MARKER).read_text(encoding="utf-8-sig"))
  79. except (OSError, ValueError):
  80. return None
  81. if not (isinstance(marker, dict) and marker.get("package") and marker.get("source")):
  82. return None
  83. mtime = marker.get("sourceMtimeMs")
  84. if isinstance(mtime, bool) or not isinstance(mtime, (int, float)):
  85. # A hand-edited or truncated timestamp is stale metadata: the next pass re-copies the half
  86. # instead of raising mid-reconcile and taking every other plugin down with it.
  87. marker["sourceMtimeMs"] = -1
  88. return marker
  89. def _write_marker(folder: Path, marker: dict) -> None:
  90. (folder / PACKAGE_MARKER).write_text(json.dumps(marker, indent=2) + "\n", encoding="utf-8")
  91. def _same_file(a: Path, b: Path) -> bool:
  92. """Same bytes on both paths. A comparison that cannot be made is never a match."""
  93. try:
  94. return a.read_bytes() == b.read_bytes()
  95. except OSError:
  96. return False
  97. def desktop_source(package_dir: Path) -> Path | None:
  98. """The folder that is *package_dir*'s desktop plugin, by the rule every installer and the install
  99. probe share (Electron's ``findDesktopEntry``, ``detect_components``): the package root itself when
  100. a ``plugin.js`` sits there, else its ``desktop/`` folder. ``None`` when neither holds one."""
  101. for source in (package_dir, package_dir / "desktop"):
  102. if (source / "plugin.js").is_file():
  103. return source
  104. return None
  105. def root_layout_ignore(source_dir: Path, package_dir: Path):
  106. """What a root-layout half leaves behind when its package root is published: the clone's ``.git``,
  107. as ``desktop_plugin_install`` drops it (nothing reads it, and Git's read-only objects make a
  108. folder Windows cannot delete), and the agent half's ``__pycache__``. A ``desktop/`` half is
  109. copied whole."""
  110. return shutil.ignore_patterns(*ROOT_LAYOUT_SKIP) if source_dir == package_dir else None
  111. def check_desktop_tree(source: Path, base: Path, *, linked_root: bool = False,
  112. skip: tuple[str, ...] = ()) -> str | None:
  113. """Why *source* cannot be published as a desktop plugin, else ``None``. It must be a plain tree:
  114. a link could publish any folder of the host, and the copy is capped at :data:`TREE_MAX_BYTES`.
  115. *linked_root* accepts *source* itself being a link (a dev checkout linked into ``plugins/``);
  116. every entry inside it is still held to the rule. Folders named in *skip* are left out of the
  117. copy (:func:`root_layout_ignore`), so they are not walked."""
  118. if _is_link(source) and not linked_root:
  119. return f"The desktop plugin contains a link ({source.relative_to(base)}); links are not installed."
  120. total = 0
  121. for folder, dirs, files in os.walk(source):
  122. dirs[:] = [n for n in dirs if n not in skip]
  123. for entry in (Path(folder) / n for n in dirs + files):
  124. if _is_link(entry):
  125. return f"The desktop plugin contains a link ({entry.relative_to(base)}); links are not installed."
  126. total += sum((Path(folder) / n).stat().st_size for n in files)
  127. if total > TREE_MAX_BYTES:
  128. return f"The desktop plugin is larger than {TREE_MAX_BYTES // (1024 * 1024)} MB."
  129. return None
  130. def publish_desktop_tree(source_dir: Path, target: Path, marker: dict | None, ignore=None) -> None:
  131. """Copy through a staging sibling and rename the finished tree into place, marker included, so a
  132. failure leaves the old folder or none: never a partial, marker-less one that a later pass would
  133. take for a hand install. A standalone plugin is published with no marker (``None``): one would
  134. claim another package's agent half."""
  135. target.parent.mkdir(parents=True, exist_ok=True)
  136. staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.staging-", dir=target.parent))
  137. staged = staging / target.name
  138. try:
  139. shutil.copytree(source_dir, staged, symlinks=True, ignore=ignore)
  140. if marker is None:
  141. # Only an installer writes the marker: one shipped in the repository would claim
  142. # another package's agent half (Electron's ``installDesktopPluginFromGit`` drops it too).
  143. (staged / PACKAGE_MARKER).unlink(missing_ok=True)
  144. else:
  145. _write_marker(staged, marker)
  146. if _is_link(target):
  147. target.unlink()
  148. elif target.exists():
  149. rmtree_readonly(target)
  150. os.rename(staged, target)
  151. finally:
  152. shutil.rmtree(staging, ignore_errors=True)
  153. def _desktop_entry(package_dir: Path, name: str) -> tuple[Path, os.stat_result] | None:
  154. """``(source_dir, plugin.js stat)`` of *package_dir*'s desktop half under :func:`desktop_source`'s
  155. rule (root ``plugin.js`` first, then ``desktop/plugin.js``), or ``None``. A candidate we may not
  156. stat is logged and ends the search: falling through to the other layout would copy a half the
  157. installer would not have picked."""
  158. for source_dir in (package_dir, package_dir / "desktop"):
  159. try:
  160. entry_stat = (source_dir / "plugin.js").stat()
  161. except (FileNotFoundError, NotADirectoryError):
  162. continue
  163. except OSError as exc:
  164. logger.warning("desktop-plugins: cannot read %s: %s", name, exc)
  165. return None
  166. if stat.S_ISREG(entry_stat.st_mode):
  167. return source_dir, entry_stat
  168. return None
  169. def materialize_desktop_half(package_dir: Path, app_root: Path, name: str) -> Path | None:
  170. """Copy *package_dir*'s desktop half to ``<app_root>/<name>/``: its ``desktop/`` folder, or the
  171. package root (minus :data:`ROOT_LAYOUT_SKIP`) when ``plugin.js`` sits at the root, as the
  172. installers publish a root-layout repository. Returns the folder when it was copied or adopted, ``None`` when it is
  173. already current, absent, or a user's own plugin. Raises :class:`OSError` with the reason when the tree is refused
  174. (a link or an oversized tree), as Electron's ``materializeDesktopHalf`` throws."""
  175. found = _desktop_entry(package_dir, name)
  176. if found is None:
  177. return None
  178. source_dir, entry_stat = found
  179. entry = source_dir / "plugin.js"
  180. target = app_root / name
  181. existing = read_package_marker(target)
  182. linked = _is_link(package_dir)
  183. # A link at ``<app_root>/<name>`` is never a copy this pass may keep or stamp: a marker-less one is
  184. # the user's own linked plugin, and one carrying a marker is an earlier copy of a linked checkout
  185. # that kept the link instead of the files (Electron's ``cp`` did), its marker inside the checkout.
  186. target_is_link = _is_link(target)
  187. if target_is_link and existing is None:
  188. return None
  189. marker = {
  190. "package": name,
  191. "source": str(source_dir),
  192. "sourceMtimeMs": mtime_ms(entry_stat),
  193. **({"linked": True} if linked else {}),
  194. **_package_origin(package_dir),
  195. }
  196. if target.exists():
  197. if existing is None and (target / "plugin.js").exists():
  198. # Our own pre-marker copy still matches byte for byte: adopt it. Anything else is a
  199. # standalone plugin the user put there, and stays exactly as it is (#112450).
  200. if _same_file(target / "plugin.js", entry):
  201. _write_marker(target, marker)
  202. return target
  203. return None
  204. if (
  205. existing is not None
  206. and not target_is_link
  207. and existing["source"] == str(source_dir)
  208. and bool(existing.get("linked")) == linked
  209. and (
  210. _same_file(target / "plugin.js", entry)
  211. if linked
  212. else existing.get("sourceMtimeMs", 0) >= marker["sourceMtimeMs"]
  213. )
  214. ):
  215. return None
  216. ignore = root_layout_ignore(source_dir, package_dir)
  217. if ignore is not None or not linked:
  218. # An installed package's ``desktop/`` folder and any root-layout half (the whole package root,
  219. # agent payload included) are held to the installer's plain-tree rule rather than copy a link to
  220. # anywhere on the host into the served folder. Only a dev checkout linked into ``plugins/``
  221. # publishes its ``desktop/`` folder as it stands, as Electron's reconcile does.
  222. refused = check_desktop_tree(source_dir, package_dir, linked_root=linked,
  223. skip=ROOT_LAYOUT_SKIP if ignore is not None else ())
  224. if refused:
  225. raise OSError(refused)
  226. publish_desktop_tree(source_dir, target, marker, ignore=ignore)
  227. return target
  228. def _source_gone(name: str, entry: Path) -> bool:
  229. """True only when the source ``plugin.js`` is really gone. A source we may not stat is not an
  230. uninstall: pruning its copy would silently drop the pane."""
  231. try:
  232. entry.stat()
  233. except (FileNotFoundError, NotADirectoryError):
  234. return True
  235. except OSError as exc:
  236. logger.warning("desktop-plugins: keeping desktop half of unreadable package %s: %s", name, exc)
  237. return False
  238. def reconcile_desktop_halves(hermes_root: Path, app_root: Path) -> list[str]:
  239. """Materialize every local home's desktop halves into *app_root* (first home wins for a name in
  240. several profiles, the default home first) and drop copies whose package is gone. Returns the
  241. folders it wrote or removed."""
  242. return reconcile_desktop_halves_report(hermes_root, app_root)["touched"]
  243. def reconcile_desktop_halves_report(hermes_root: Path, app_root: Path) -> dict:
  244. """:func:`reconcile_desktop_halves`, also naming each package whose desktop half it refused or could not
  245. read (``refused``: ``{"name", "reason"}``), as Electron's ``reconcileDesktopPluginsReport`` does. A refused
  246. half leaves its name to a later home's half, and is not reported once that half is published."""
  247. with ROOT_WRITER:
  248. touched: list[str] = []
  249. refused: list[dict[str, str]] = []
  250. seen: set[str] = set()
  251. homes = [hermes_root, *(hermes_root / "profiles" / name for name in _list_dirs(hermes_root / "profiles"))]
  252. for home in homes:
  253. plugins_root = home / "plugins"
  254. for name in _list_dirs(plugins_root, follow_links=True):
  255. if name in seen:
  256. continue
  257. try:
  258. result = materialize_desktop_half(plugins_root / name, app_root, name)
  259. except OSError as exc:
  260. # One unreadable or refused package must not stop every other half from loading.
  261. logger.warning("desktop-plugins: skipping package %s: %s", name, exc)
  262. refused.append({"name": name, "reason": str(exc)})
  263. continue
  264. if result is not None or desktop_source(plugins_root / name) is not None:
  265. seen.add(name)
  266. if result is not None:
  267. touched.append(str(result))
  268. # Links count too: a marker read through one is an earlier copy of a linked checkout that kept
  269. # the link. Only the link goes, never the folder it points at.
  270. for name in _list_dirs(app_root, follow_links=True):
  271. folder = app_root / name
  272. marker = read_package_marker(folder)
  273. if marker and _source_gone(name, Path(marker["source"]) / "plugin.js"):
  274. if _is_link(folder):
  275. folder.unlink(missing_ok=True)
  276. else:
  277. shutil.rmtree(folder, ignore_errors=True)
  278. touched.append(str(folder))
  279. return {"touched": touched, "refused": [entry for entry in refused if entry["name"] not in seen]}