web_server_memory.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. """Memory-provider dashboard helpers: manifest/schema loading, setup-env and dependency probes, configured-status discovery.
  2. """
  3. import logging
  4. import json
  5. import os
  6. import shlex
  7. import subprocess
  8. from fastapi import HTTPException
  9. from pathlib import Path
  10. from typing import Any, Dict, List, Optional
  11. # Same logger the code used before extraction (record parity).
  12. _log = logging.getLogger("hermes_cli.web_server")
  13. def _normalize_memory_provider_name(name: Any) -> str:
  14. from agent.memory_provider import is_core_memory_provider
  15. provider = str(name or "").strip()
  16. return "" if is_core_memory_provider(provider) else provider
  17. def _load_memory_provider(name: str, *, register_skills: Optional[bool] = None):
  18. try:
  19. from plugins.memory import load_memory_provider
  20. return load_memory_provider(name, register_skills=register_skills)
  21. except Exception:
  22. _log.debug("Failed to load memory provider %s", name, exc_info=True)
  23. return None
  24. def _memory_provider_manifest(name: str) -> dict[str, Any]:
  25. try:
  26. from hermes_cli.memory_setup import memory_provider_dependency_inputs
  27. manifest, _inputs = memory_provider_dependency_inputs(name)
  28. return manifest
  29. except Exception:
  30. _log.debug("Failed to read memory provider manifest for %s", name, exc_info=True)
  31. return {}
  32. def _string_list(value: Any) -> list[str]:
  33. if not isinstance(value, list):
  34. return []
  35. return [str(item).strip() for item in value if str(item).strip()]
  36. def _memory_provider_setup_manifest(name: str) -> tuple[dict, dict]:
  37. from hermes_cli.memory_setup import memory_provider_dependency_inputs
  38. from plugins.memory import find_provider_dir
  39. from pm.plugin_declarations import read_python_declaration
  40. manifest, inputs = memory_provider_dependency_inputs(name)
  41. plugin_dir = find_provider_dir(name)
  42. external_dependencies: list[dict[str, str]] = []
  43. for raw in manifest.get("external_dependencies") or []:
  44. if not isinstance(raw, dict):
  45. continue
  46. dep = {k: str(raw.get(k) or "").strip() for k in ("name", "install", "check")}
  47. if any(dep.values()):
  48. external_dependencies.append(dep)
  49. return {
  50. # Display only; neither import names nor these labels determine readiness. Shown from the
  51. # same declaration PM installs (an authored pyproject.toml wins over the manifest lists).
  52. "pip_dependencies": list(read_python_declaration(plugin_dir).install_requirements) if plugin_dir else [],
  53. "python_dependencies_declared": bool(inputs),
  54. "external_dependencies": external_dependencies,
  55. "required_env": _string_list(manifest.get("requires_env")),
  56. }, inputs
  57. def _memory_provider_setup_info(name: str) -> dict[str, Any]:
  58. import pm
  59. try:
  60. setup, inputs = _memory_provider_setup_manifest(name)
  61. except Exception:
  62. _log.debug("Invalid dependency declaration for %s", name, exc_info=True)
  63. return {"pip_dependencies": [], "python_dependencies_declared": True,
  64. "external_dependencies": [], "required_env": [], "dependencies_installed": False}
  65. try:
  66. python_ready = not inputs or pm.venv_is_current(**inputs)
  67. except Exception:
  68. _log.debug("Could not read dependency state for %s", name, exc_info=True)
  69. python_ready = False
  70. setup["dependencies_installed"] = python_ready and _memory_provider_external_dependencies_installed(setup)
  71. return setup
  72. def _memory_provider_setup_env() -> dict[str, str]:
  73. # External package-manager child (npm/uv/pip): exact env preservation —
  74. # scrubbing or HOME rewriting could break user tool auth/config.
  75. from tools.environments.local import build_subprocess_env
  76. env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False)
  77. home = Path.home()
  78. extra_bins = [home / ".brv-cli" / "bin", home / ".local" / "bin", home / ".npm-global" / "bin", Path("/usr/local/bin")]
  79. prefix = os.pathsep.join(str(path) for path in extra_bins if path.exists())
  80. if prefix:
  81. env["PATH"] = prefix + os.pathsep + env.get("PATH", "")
  82. return env
  83. def _run_setup_command(
  84. command: Any, *, display: str, shell: bool = False, timeout: int = 180
  85. ) -> subprocess.CompletedProcess:
  86. return subprocess.run(
  87. command,
  88. shell=shell,
  89. executable="/bin/bash" if shell else None,
  90. env=_memory_provider_setup_env(),
  91. capture_output=True,
  92. text=True,
  93. # Lossy UTF-8 decode — setup tools emit UTF-8; a locale-mismatched byte must never raise.
  94. # Force UTF-8 with lossy decoding so child output containing bytes that are invalid in the system
  95. # locale (e.g. GBK on Chinese Windows) can't raise UnicodeDecodeError inside the drain threads and
  96. # crash the gateway. See #53137.
  97. encoding="utf-8",
  98. errors="replace",
  99. timeout=timeout,
  100. check=False,
  101. )
  102. def _memory_provider_external_dependencies_installed(setup: dict[str, Any]) -> bool:
  103. external_ok = True
  104. for dep in setup.get("external_dependencies") or []:
  105. if not isinstance(dep, dict):
  106. continue
  107. check_cmd = str(dep.get("check") or "").strip()
  108. if not check_cmd:
  109. if str(dep.get("install") or "").strip():
  110. external_ok = False
  111. continue
  112. try:
  113. completed = _run_setup_command(shlex.split(check_cmd), display=check_cmd, timeout=20)
  114. except Exception:
  115. external_ok = False
  116. continue
  117. if completed.returncode != 0:
  118. external_ok = False
  119. return external_ok
  120. def _schema_field_kind(raw: dict[str, Any], choices: list) -> str:
  121. """Field kind from explicit ``kind``/``type`` hints, else inferred from ``default``."""
  122. explicit_kind = str(raw.get("kind") or raw.get("type") or "").strip().lower()
  123. default = raw.get("default")
  124. if raw.get("secret"):
  125. return "secret"
  126. if choices:
  127. return "select"
  128. if explicit_kind in {"bool", "boolean"} or isinstance(default, bool):
  129. return "boolean"
  130. if explicit_kind in {"int", "integer"} or (isinstance(default, int) and not isinstance(default, bool)):
  131. return "integer"
  132. if explicit_kind in {"float", "number"} or isinstance(default, float):
  133. return "number"
  134. return "text"
  135. def _normalize_memory_provider_schema(name: str, provider: Any) -> list[dict[str, Any]]:
  136. raw_schema: list[dict[str, Any]] = []
  137. if provider is not None and hasattr(provider, "get_config_schema"):
  138. try:
  139. raw = provider.get_config_schema()
  140. if isinstance(raw, list):
  141. raw_schema = [field for field in raw if isinstance(field, dict)]
  142. except Exception:
  143. _log.warning("Failed to read memory provider schema for %s", name, exc_info=True)
  144. fields: list[dict[str, Any]] = []
  145. for raw in raw_schema:
  146. key = str(raw.get("key") or "").strip()
  147. if not key:
  148. continue
  149. choices = raw.get("choices") or raw.get("options") or []
  150. if not isinstance(choices, list):
  151. choices = []
  152. fields.append({
  153. "key": key,
  154. "label": str(raw.get("label") or key.replace("_", " ").title()),
  155. "kind": _schema_field_kind(raw, choices),
  156. "description": str(raw.get("description") or ""),
  157. "placeholder": str(raw.get("placeholder") or ""),
  158. "required": bool(raw.get("required", False)),
  159. "default": raw.get("default", ""),
  160. "options": [{"value": str(c), "label": str(c), "description": ""} for c in choices],
  161. "url": str(raw.get("url") or ""),
  162. "when": raw.get("when") if isinstance(raw.get("when"), dict) else None,
  163. "minimum": raw.get("minimum"),
  164. "maximum": raw.get("maximum"),
  165. "step": raw.get("step"),
  166. "_env_key": str(raw.get("env_var") or "") or None,
  167. })
  168. return fields
  169. def _read_json_file(path: Path) -> dict[str, Any]:
  170. try:
  171. data = json.loads(path.read_text(encoding="utf-8-sig")) if path.exists() else {}
  172. except Exception:
  173. _log.debug("Failed to read JSON config from %s", path, exc_info=True)
  174. return {}
  175. return data if isinstance(data, dict) else {}
  176. def _read_memory_provider_existing_values(name: str) -> dict[str, Any]:
  177. """Best-effort read of existing provider config across legacy/native stores."""
  178. from hermes_cli.config import get_hermes_home, load_config
  179. hermes_home = get_hermes_home()
  180. values: dict[str, Any] = {}
  181. for path in (hermes_home / f"{name}.json", hermes_home / name / "config.json"):
  182. values.update(_read_json_file(path))
  183. try:
  184. cfg = load_config()
  185. except Exception:
  186. cfg = {}
  187. if not isinstance(cfg, dict):
  188. cfg = {}
  189. memory_cfg = cfg.get("memory")
  190. if isinstance(memory_cfg, dict):
  191. provider_cfg = memory_cfg.get(name)
  192. if isinstance(provider_cfg, dict):
  193. values.update(provider_cfg)
  194. legacy_cfg = memory_cfg.get("provider_config")
  195. if isinstance(legacy_cfg, dict):
  196. values = {**legacy_cfg, **values}
  197. # Holographic stores under plugins.hermes-memory-store.
  198. plugins_cfg = cfg.get("plugins")
  199. if name == "holographic" and isinstance(plugins_cfg, dict):
  200. holographic_cfg = plugins_cfg.get("hermes-memory-store")
  201. if isinstance(holographic_cfg, dict):
  202. values.update(holographic_cfg)
  203. return values
  204. def _env_lookup(env_key: Optional[str]) -> str:
  205. from hermes_cli.config import load_env
  206. if not env_key:
  207. return ""
  208. return str(load_env().get(env_key) or os.environ.get(env_key) or "")
  209. def _coerce_bool(value: Any, *, default: bool = False) -> bool:
  210. if isinstance(value, bool):
  211. return value
  212. if value is None or value == "":
  213. return default
  214. if isinstance(value, (int, float)):
  215. return bool(value)
  216. text = str(value).strip().lower()
  217. if text in {"1", "true", "yes", "on"}:
  218. return True
  219. if text in {"0", "false", "no", "off"}:
  220. return False
  221. raise ValueError(f"Invalid boolean value: {value}")
  222. def _field_default(field: dict[str, Any]) -> Any:
  223. default = field.get("default", "")
  224. if field["kind"] == "boolean":
  225. return _coerce_bool(default, default=False)
  226. return default
  227. def _field_value(field: dict[str, Any], data: dict[str, Any]) -> Any:
  228. if field["kind"] == "secret":
  229. return ""
  230. value = data.get(field["key"])
  231. if value in (None, ""):
  232. value = _env_lookup(field.get("_env_key"))
  233. if value in (None, ""):
  234. value = _field_default(field)
  235. if field["kind"] == "select":
  236. allowed = {opt["value"] for opt in field.get("options", [])}
  237. value = str(value)
  238. return value if value in allowed else str(_field_default(field))
  239. if field["kind"] == "boolean":
  240. return _coerce_bool(value, default=_coerce_bool(_field_default(field), default=False))
  241. return str(value)
  242. def _field_is_set(field: dict[str, Any], data: dict[str, Any]) -> bool:
  243. if field["kind"] == "secret":
  244. return bool(_env_lookup(field.get("_env_key")) or data.get(field["key"]))
  245. return _field_value(field, data) not in (None, "")
  246. def _field_visible(
  247. field: dict[str, Any], data: dict[str, Any], fields_by_key: Optional[dict[str, dict[str, Any]]] = None
  248. ) -> bool:
  249. when = field.get("when")
  250. if not isinstance(when, dict) or not when:
  251. return True
  252. for dep_key, expected in when.items():
  253. dep_field = (fields_by_key or {}).get(str(dep_key)) or {
  254. "key": str(dep_key), "kind": "text", "default": "", "_env_key": None
  255. }
  256. if str(_field_value(dep_field, data)) != str(expected):
  257. return False
  258. return True
  259. def _memory_provider_is_configured(name: str, provider: Any) -> bool:
  260. data = _read_memory_provider_existing_values(name)
  261. fields = _normalize_memory_provider_schema(name, provider)
  262. fields_by_key = {field["key"]: field for field in fields}
  263. return all(
  264. _field_is_set(field, data)
  265. for field in fields
  266. if field.get("required") and _field_visible(field, data, fields_by_key)
  267. )
  268. def _memory_provider_status(row: dict[str, Any], setup: dict[str, Any], configured: bool, schema_fields: list) -> str:
  269. from plugins.memory import parked_provider_notice
  270. if row["missing"]:
  271. return "missing"
  272. if parked_provider_notice(row["name"]): # plugins.disabled parks it: it will not load
  273. return "unavailable"
  274. if not setup.get("dependencies_installed", True):
  275. return "unavailable"
  276. if not configured or (not row["available"] and schema_fields):
  277. return "needs_config"
  278. return "ready" if row["available"] else "unavailable"
  279. def _discover_memory_provider_statuses() -> list[dict[str, Any]]:
  280. from hermes_cli.config import load_config
  281. discovered: dict[str, dict[str, Any]] = {}
  282. try:
  283. from plugins.memory import discover_memory_providers
  284. for name, description, available in discover_memory_providers():
  285. discovered[str(name)] = {
  286. "name": str(name),
  287. "description": str(description or ""),
  288. "available": bool(available),
  289. "missing": False,
  290. }
  291. except Exception:
  292. _log.exception("discover_memory_providers failed")
  293. mem = load_config().get("memory")
  294. active = _normalize_memory_provider_name(mem.get("provider")) if isinstance(mem, dict) else ""
  295. if active and active not in discovered:
  296. discovered[active] = {
  297. "name": active,
  298. "description": "Configured provider was not found.",
  299. "available": False,
  300. "missing": True,
  301. }
  302. providers: list[dict[str, Any]] = []
  303. for name in sorted(discovered):
  304. row = discovered[name]
  305. missing = row["missing"]
  306. provider = None if missing else _load_memory_provider(name)
  307. setup = _memory_provider_setup_info(name)
  308. configured = False if missing else _memory_provider_is_configured(name, provider)
  309. schema_fields = [] if missing else _normalize_memory_provider_schema(name, provider)
  310. providers.append({
  311. "name": name,
  312. "description": row["description"],
  313. "available": row["available"],
  314. "configured": configured,
  315. "status": _memory_provider_status(row, setup, configured, schema_fields),
  316. "setup": setup,
  317. })
  318. return providers
  319. def _require_memory_provider_ready(name: str) -> None:
  320. if not name:
  321. return
  322. row = next((r for r in _discover_memory_provider_statuses() if r["name"] == name), None)
  323. if row is None:
  324. raise HTTPException(status_code=400, detail=f"Unknown memory provider '{name}'.")
  325. if row["status"] != "ready":
  326. raise HTTPException(
  327. status_code=400,
  328. detail=(
  329. f"Memory provider '{name}' is not ready "
  330. f"({row['status'].replace('_', ' ')}). Configure it in the dashboard first."
  331. ),
  332. )