memory_provider_migration.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. """Move a user from a memory provider that left core onto its catalog plugin.
  2. A bundled ``plugins/memory/<name>`` that becomes a standalone catalog plugin keeps the same provider
  3. name, the settings it already read, its data directory and tool names, so the migration is only
  4. "the code now lives under ``HERMES_HOME/plugins/<name>``". Two hooks call :func:`migrate_home`:
  5. * ``hermes update`` — for every profile home that shares the venv (primary; runs where the venv was
  6. just rebuilt anyway).
  7. * agent init — when the configured provider cannot be found at all, once per process (Desktop
  8. users update through the app and never run ``hermes update`` by hand). After a failed attempt,
  9. starts skip it for :data:`STARTUP_RETRY_SECONDS`, doubling with each further failure up to
  10. :data:`STARTUP_RETRY_MAX_SECONDS` (``hermes update`` always retries): every ``hermes chat``, cron
  11. run and Desktop restart otherwise paid the network round trip again and showed the same failure.
  12. Both install the catalog entry at its reviewed pin through the normal plugin install path (kill
  13. list, dependency constraints, enable), never a custom source. Every outcome — installed, refused,
  14. failed, absent from the catalog — reaches the user (terminal, Desktop, chat platform), never only
  15. ``agent.log``: a provider that silently stays missing is lost memory.
  16. """
  17. from __future__ import annotations
  18. import logging
  19. import sys
  20. import time
  21. from pathlib import Path
  22. from typing import Callable, Optional
  23. logger = logging.getLogger(__name__)
  24. _attempted: set[tuple[str, str]] = set()
  25. # Providers that shipped bundled in core. Unattended consent covers only these: their users already
  26. # accepted the deps when they picked the built-in. Any other catalog plugin named in memory.provider
  27. # (a synced config, a cloned profile) still needs an explicit install.
  28. _LEFT_CORE = frozenset({"hindsight", "honcho", "mem0", "supermemory", "openviking", "retaindb", "byterover", "holographic"})
  29. # The startup retry window doubles with each consecutive failure, up to a week: a flat window had a
  30. # home whose install can never succeed (a version gate, a root-owned plugins dir, no route to GitHub)
  31. # fail once an hour for good, while an offline laptop still gets its plugin within the hour.
  32. STARTUP_RETRY_SECONDS = 3600.0
  33. STARTUP_RETRY_MAX_SECONDS = 7 * 24 * 3600.0
  34. def _failure_stamp(home: Path, plugin: str) -> Path:
  35. """Written on every failed automatic install: its mtime and failure count gate the next startup attempt."""
  36. return home / "cache" / f"left-core-{plugin}.failed"
  37. def _failure_count(stamp: Path) -> int:
  38. """Consecutive failures recorded in *stamp*; the empty stamp an older Hermes touched counts once."""
  39. try:
  40. text = stamp.read_text(encoding="utf-8-sig").strip()
  41. except OSError:
  42. return 0
  43. return int(text) if text.isdigit() else 1
  44. def _note_failure(home: Path, plugin: str) -> None:
  45. stamp = _failure_stamp(home, plugin)
  46. try:
  47. stamp.parent.mkdir(parents=True, exist_ok=True)
  48. stamp.write_text(str(_failure_count(stamp) + 1), encoding="utf-8")
  49. except OSError as exc:
  50. logger.debug("left-core retry stamp not written for %s: %s", home, exc)
  51. def _failed_recently(home: Path, plugin: str) -> bool:
  52. stamp = _failure_stamp(home, plugin)
  53. try:
  54. age = time.time() - stamp.stat().st_mtime
  55. except OSError:
  56. return False
  57. failures = min(_failure_count(stamp), 32)
  58. return age < min(STARTUP_RETRY_SECONDS * 2 ** (failures - 1), STARTUP_RETRY_MAX_SECONDS)
  59. def _note_success(home: Path, plugin: str) -> None:
  60. """A later failure (the plugin removed, then missed again) starts the backoff from an hour."""
  61. _failure_stamp(home, plugin).unlink(missing_ok=True)
  62. def configured_provider(home: Path) -> str:
  63. """``memory.provider`` of *home*'s effective config, or ``""``."""
  64. from pm.plugins_state import read_home_selection
  65. memory = (read_home_selection(home) or {}).get("memory") or {}
  66. return str(memory.get("provider") or "").strip()
  67. def provider_present(name: str, home: Path) -> bool:
  68. """True when the provider resolves anywhere Hermes looks for *home* (bundled, that home's user
  69. plugins, entry point). The lookup reads the active home, so it is bound explicitly: the update
  70. hook walks several profile homes from one process."""
  71. from plugins.memory import find_provider_dir
  72. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  73. token = set_hermes_home_override(home)
  74. try:
  75. return find_provider_dir(name) is not None
  76. finally:
  77. reset_hermes_home_override(token)
  78. def catalog_source(name: str) -> Optional[str]:
  79. """The catalog entry that ships provider *name*, or None when the catalog has no such plugin."""
  80. from hermes_cli.plugin_catalog import get_live_catalog_entry
  81. entry = get_live_catalog_entry(name)
  82. return entry.name if entry is not None else None
  83. def catalog_install_hint(name: str, *, category: Optional[str] = None) -> Optional[str]:
  84. """``hermes [-p <profile>] plugins install <name>`` when this checkout's catalog ships plugin
  85. *name* (of *category*, when given), else None.
  86. Recovery copy for a provider that left core and is not installed (offline, lazy installs off, a
  87. failed migration): doctor, ``memory status``, ``memory setup <name>`` and the CLI's unknown-command
  88. error point at the one command that fixes it. In-tree catalog only, never the network — these run
  89. offline and on error paths."""
  90. try:
  91. from hermes_cli.plugin_catalog import get_catalog_entry
  92. entry = get_catalog_entry(name) if name else None
  93. except Exception:
  94. return None
  95. if entry is None or (category is not None and entry.category != category):
  96. return None
  97. from hermes_constants import get_hermes_home
  98. return _install_command(name, get_hermes_home())
  99. def _pending_provider(home: Path, *, say: Callable[[str], None]) -> Optional[str]:
  100. """The provider *home* needs from the catalog, or None (nothing to do, or a catalog miss already
  101. reported through *say*). Read-only."""
  102. name = configured_provider(home)
  103. from agent.memory_provider import is_core_memory_provider
  104. if is_core_memory_provider(name) or provider_present(name, home):
  105. return None
  106. if catalog_source(name) is None:
  107. say(f" ⚠ Memory provider '{name}' is configured but not installed and not in the plugin catalog. "
  108. f"Install it with `hermes plugins install <source>` or change memory.provider.")
  109. return None
  110. return name
  111. def _install_command(name: str, home: Path) -> str:
  112. """The exact command that installs *name* into *home*. ``-p`` is dropped only for the default
  113. home while no sticky profile is set: a bare command run from a shell targets the sticky
  114. profile, never the home of the agent (Desktop, gateway, ``hermes -p``) that printed it."""
  115. from hermes_cli.profiles import get_active_profile
  116. from hermes_constants import profile_name_for_home
  117. profile = profile_name_for_home(home)
  118. if profile is None or (profile == "default" and get_active_profile() == "default"):
  119. return f"hermes plugins install {name}"
  120. return f"hermes -p {profile} plugins install {name}"
  121. def _install_pending(home: Path, name: str, *, install: Callable[[str], dict],
  122. say: Callable[[str], None]) -> bool:
  123. try:
  124. result = install(name)
  125. except Exception as exc: # network, uv, kill list — report, do not raise
  126. result = {"ok": False, "error": str(exc)}
  127. if result.get("ok"):
  128. say(f" ✓ Memory provider '{name}' moved out of core — installed its plugin from the catalog "
  129. f"(memory.provider and your stored memories are unchanged; check its settings with "
  130. f"`hermes memory status`).")
  131. return True
  132. error = str(result.get("error") or "unknown error").rstrip(". ")
  133. say(f" ⚠ Memory provider '{name}' moved out of core and could not be installed automatically: "
  134. f"{error}. Run `{_install_command(name, home)}`.")
  135. return False
  136. def migrate_home(home: Path, *, install: Callable[[str], dict], say: Callable[[str], None] = print) -> Optional[str]:
  137. """Install the configured provider's catalog plugin into *home* when the provider is gone.
  138. Returns the installed plugin name, or None when nothing needed doing or the install could not
  139. happen (already reported through *say*). Never raises: memory being down must not take the
  140. update or the agent down with it.
  141. """
  142. name = _pending_provider(home, say=say)
  143. if name is None:
  144. return None
  145. return name if _install_pending(home, name, install=install, say=say) else None
  146. def _interactive() -> bool:
  147. return sys.stdin is not None and sys.stdout is not None and sys.stdin.isatty() and sys.stdout.isatty()
  148. def _unattended_consent() -> bool:
  149. """Without a terminal (Desktop, gateway, ``hermes update`` from a script) nobody can answer the
  150. dependency prompt, so every provider that declares Python deps would fail to migrate. The
  151. configured ``memory.provider`` plus ``security.allow_lazy_installs`` (read for the active home,
  152. i.e. the one being migrated) is the same consent that let the bundled provider install its deps
  153. on demand; with a terminal the user is still asked."""
  154. from pm.install import lazy_installs_allowed
  155. return not _interactive() and lazy_installs_allowed()
  156. def _home_consent(home: Path) -> bool:
  157. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  158. token = set_hermes_home_override(home)
  159. try:
  160. return _unattended_consent()
  161. finally:
  162. reset_hermes_home_override(token)
  163. def _install_into(home: Path, *, consent: Optional[bool] = None) -> Callable[[str], dict]:
  164. def _install(name: str) -> dict:
  165. from hermes_cli.plugins_cmd import dashboard_install_plugin
  166. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  167. token = set_hermes_home_override(home)
  168. try:
  169. return dashboard_install_plugin("", force=False, enable=True, catalog_name=name,
  170. assume_deps_consent=(name in _LEFT_CORE and _unattended_consent())
  171. if consent is None else consent)
  172. finally:
  173. reset_hermes_home_override(token)
  174. return _install
  175. def _home_label(home: Path) -> str:
  176. from hermes_constants import profile_name_for_home
  177. profile = profile_name_for_home(home)
  178. return f"profile '{profile}'" if profile else str(home)
  179. def migrate_all_homes(*, say: Callable[[str], None] = print) -> list[str]:
  180. """``hermes update`` hook: every profile home sharing this venv. Returns installed plugin names.
  181. Each line names the profile it is about. Homes missing the same provider share one environment,
  182. so they share one set of dependency answers (#125794): the first install asks, the rest reuse
  183. its answers. Homes are grouped by provider AND by their unattended consent (each home's own
  184. ``security.allow_lazy_installs``), so a refusal only speaks for homes that would be refused the
  185. same way: when an install in a group fails, the rest of that group are named in one line
  186. instead of failing (or re-prompting) one by one, and the other groups still migrate. Ctrl-C
  187. ends the migration with a message, not a traceback into the updater.
  188. """
  189. from hermes_cli.plugins_cmd_install import shared_dependency_answers
  190. from pm.plugins_state import dependency_homes
  191. def labelled(home: Path) -> Callable[[str], None]:
  192. return lambda message: say(f" [{_home_label(home)}] {message.lstrip()}")
  193. pending: dict[tuple[str, bool], list[Path]] = {}
  194. for home in dependency_homes():
  195. try:
  196. name = _pending_provider(home, say=labelled(home))
  197. consent = bool(name) and _home_consent(home)
  198. except Exception as exc:
  199. logger.debug("memory provider migration skipped for %s: %s", home, exc)
  200. continue
  201. if name:
  202. pending.setdefault((name, consent), []).append(home)
  203. installed: list[str] = []
  204. try:
  205. for (name, _consent), homes in pending.items():
  206. if len(homes) > 1 and _interactive():
  207. say(f" Memory provider '{name}' is configured in {len(homes)} profiles "
  208. f"({', '.join(_home_label(h) for h in homes)}); your answers to its dependency "
  209. f"questions apply to all of them.")
  210. with shared_dependency_answers():
  211. for index, home in enumerate(homes):
  212. if _install_pending(home, name, install=_install_into(home), say=labelled(home)):
  213. installed.append(name)
  214. continue
  215. rest = homes[index + 1:]
  216. if rest:
  217. say(f" ⚠ Memory provider '{name}' was not installed for "
  218. f"{', '.join(_home_label(h) for h in rest)} either. Run "
  219. + ", ".join(f"`{_install_command(name, h)}`" for h in rest) + ".")
  220. break
  221. except KeyboardInterrupt:
  222. say(" ⚠ Memory provider migration cancelled. Profiles already migrated keep their plugin; "
  223. "run `hermes plugins install <name>` (with `-p <profile>`) for the rest.")
  224. return installed
  225. def recover_at_startup(name: str, *, say: Optional[Callable[[str], None]] = None) -> bool:
  226. """Agent-init hook for a configured provider that resolved nowhere. One attempt per process per
  227. home and name; honours ``security.allow_lazy_installs`` because it installs code. True when installed."""
  228. from hermes_constants import get_hermes_home, hermes_home_key
  229. home = get_hermes_home()
  230. key = (hermes_home_key(home), name)
  231. if key in _attempted:
  232. return False
  233. _attempted.add(key)
  234. def report(message: str) -> None:
  235. logger.warning(message)
  236. if say is not None:
  237. try:
  238. say(message)
  239. except Exception:
  240. logger.debug("Memory migration notification failed", exc_info=True)
  241. from pm.install import lazy_installs_allowed
  242. if not lazy_installs_allowed():
  243. report(f"⚠ Memory provider '{name}' is not installed, so external memory is off for this session. "
  244. f"security.allow_lazy_installs is off, so Hermes did not fetch it: "
  245. f"run `{_install_command(name, home)}`.")
  246. return False
  247. # Agent init cannot answer a dependency prompt: under the CLI the prompt_toolkit input owns the
  248. # terminal (the question hangs the turn), elsewhere there is no terminal (the install is refused,
  249. # every process). So it installs with consent or not at all: a provider that shipped in core
  250. # carries the consent its built-in had; any other one needs the user's own install.
  251. if name not in _LEFT_CORE:
  252. if _pending_provider(home, say=report) == name:
  253. report(f"⚠ Memory provider '{name}' is not installed, so external memory is off for this session. "
  254. f"It never shipped with Hermes, so Hermes installs it only when you ask: "
  255. f"run `{_install_command(name, home)}`.")
  256. return False
  257. if _failed_recently(home, name):
  258. report(f"⚠ Memory provider '{name}' is not installed, so external memory is off for this session. "
  259. f"Its automatic install failed recently: run `{_install_command(name, home)}`.")
  260. return False
  261. installed = migrate_home(home, install=_install_into(home, consent=True), say=report) == name
  262. if installed:
  263. _note_success(home, name)
  264. else:
  265. _note_failure(home, name)
  266. return installed