doctor_state.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715
  1. """HERMES_HOME state checks for hermes doctor: directories, memory files, state.db health, skills hub, memory provider, profiles.
  2. Split out of ``hermes_cli/doctor.py``, which re-exports every name so ``hermes_cli.doctor.<name>`` keeps resolving (and monkeypatching)."""
  3. from __future__ import annotations
  4. import os
  5. import subprocess
  6. from pathlib import Path
  7. from hermes_cli.doctor_report import (
  8. Finding, _fail_and_issue, _section, check_bool, check_info, check_ok, check_warn, doctor_check, ensure_dir,
  9. warn_on_error,
  10. )
  11. from hermes_cli.sizefmt import format_bytes as _human_bytes
  12. from hermes_state_holders import read_only_db_uri
  13. def _doctor_memory_config(hermes_home: Path | None = None) -> dict:
  14. """Return the effective memory section used by doctor diagnostics."""
  15. from hermes_cli.config_backend import config_exists
  16. from hermes_cli.doctor import HERMES_HOME
  17. try:
  18. from hermes_cli.config_effective import load_user_config_effective
  19. config_path = (hermes_home if hermes_home is not None else HERMES_HOME) / "config.yaml"
  20. if not config_exists(config_path):
  21. return {}
  22. section = load_user_config_effective(config_path).get("memory")
  23. return section if isinstance(section, dict) else {}
  24. except Exception:
  25. return {}
  26. # state.db size threshold — advisory only; deliberately a module constant, not config (doctor warnings are guidance, not policy).
  27. STATE_DB_SIZE_WARN_BYTES = 1 * 1024 * 1024 * 1024 # 1 GiB logical size
  28. def _bits(*pairs) -> list:
  29. """``[fmt() for value, fmt in pairs if value is not None]`` — present-only stat fragments."""
  30. return [fmt() for value, fmt in pairs if value is not None]
  31. def host_gateway_note() -> str:
  32. """``" (the host gateway (PID 42) serving profiles default, coder)"`` when one gateway process
  33. owns this host, else ``""``. Multiplex-only: the state.db holder and WAL lines used to imply a
  34. gateway per profile; the truth is one shared process serving N profiles, and stopping it stops
  35. every one of them."""
  36. try:
  37. from gateway.host_topology import host_gateway_topology
  38. topology = host_gateway_topology()
  39. except Exception:
  40. return ""
  41. return f" ({topology.describe()})" if topology is not None else ""
  42. def _render_state_db_stats(stats: dict, holders=None, host_note: str = "") -> list:
  43. """Turn a collect_state_db_stats() dict into ``(kind, text, detail)`` rows, kind 'info' / 'warn'.
  44. Pure formatting — no I/O — so it is unit-testable without the doctor CLI. Tolerates None in every field.
  45. ``host_note`` names the shared host gateway among the holders (see :func:`host_gateway_note`).
  46. """
  47. lines: list = []
  48. stats = stats or {}
  49. logical, wal, freelist = (stats.get(k) for k in ("logical_size_bytes", "wal_size_bytes", "freelist_count"))
  50. size_bits = _bits(
  51. (logical, lambda: f"logical size {_human_bytes(logical)}"),
  52. (stats.get("page_count"), lambda: f"{stats['page_count']:,} pages"),
  53. (freelist, lambda: f"{freelist:,} free"),
  54. (wal, lambda: f"WAL {_human_bytes(wal)}"),
  55. )
  56. if size_bits:
  57. lines.append(("info", "state.db " + ", ".join(size_bits), ""))
  58. row_bits = _bits(
  59. (stats.get("messages"), lambda: f"{stats['messages']:,} messages"),
  60. (stats.get("sessions"), lambda: f"{stats['sessions']:,} sessions"),
  61. (stats.get("journal_mode") or None, lambda: f"journal_mode={stats['journal_mode']}"),
  62. (holders, lambda: f"{holders} process(es) holding the DB open{host_note}"),
  63. )
  64. if row_bits:
  65. lines.append(("info", ", ".join(row_bits), ""))
  66. fts = stats.get("fts_tables")
  67. if fts:
  68. present = [t for t, ok in fts.items() if ok]
  69. lines.append(("info", "FTS tables: " + (", ".join(present) if present else "none"), ""))
  70. deferral = stats.get("fts_rebuild_deferral")
  71. if isinstance(deferral, dict):
  72. pids = deferral.get("holder_pids") or "unknown"
  73. if deferral.get("futile"):
  74. lines.append(("warn", f"state.db FTS repair is blocked by the same holder(s) PID(s) {pids} for "
  75. f"{deferral.get('holders_attempts') or '?'} consecutive deferral(s); waiting is futile",
  76. "(stop ONLY the listed process(es) — the host gateway keeps running and its own "
  77. "retry rebuilds within a minute of the holder leaving)"))
  78. else:
  79. lines.append(("warn", f"state.db FTS repair is blocked after {deferral.get('attempts') or '?'} deferral(s) "
  80. f"by PID(s) {pids}",
  81. "(stop the listed processes; the host gateway's own retry then rebuilds, or run "
  82. "'hermes sessions optimize-storage' with every holder stopped)"))
  83. # Oversized DB: suggest auto_prune, plus the offline optimize-storage pass when the FTS rebuild is
  84. # pending, an earlier run stopped before tearing down the old index, or the stored FTS layout is an
  85. # older release's (not just a lagging marker).
  86. if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES:
  87. detail = "consider enabling sessions.auto_prune in config.yaml to bound growth"
  88. structural_fts_upgrade = bool(stats.get("fts_storage_upgrade_needed"))
  89. if stats.get("fts_rebuild_pending") or stats.get("fts_teardown_pending") or structural_fts_upgrade:
  90. detail += "; run 'hermes sessions optimize-storage' offline (with the host gateway stopped) to compact FTS storage"
  91. lines.append(("warn", f"state.db is large ({_human_bytes(logical)})", f"({detail})"))
  92. # WAL runaway is deliberately NOT warned here: _state_db_wal already warns above 50 MB and offers --fix.
  93. return lines
  94. def _memory_store_flags(hermes_home: Path) -> tuple:
  95. from tools.memory_tool import get_builtin_memory_store_flags
  96. return get_builtin_memory_store_flags({"memory": _doctor_memory_config(hermes_home)})
  97. def check_legacy_desktop_checkout() -> None:
  98. """Report the unused legacy checkout under an embedded desktop install.
  99. Before the embedded runtime existed, the desktop app installed a git
  100. checkout at $HERMES_HOME/hermes-agent. An embedded app never uses it,
  101. so it sits on disk (1-2 GB of tree + venv). Doctor only REPORTS the
  102. checkout and its size — it never suggests a deletion command, and
  103. never deletes anything itself: a pristineness probe cannot prove no
  104. other client uses the tree or that every local commit is published,
  105. so the review-and-decide step belongs to the user.
  106. """
  107. from hermes_cli.steward import STEWARD_DESKTOP, sealed_steward
  108. try:
  109. from hermes_cli.main import PROJECT_ROOT
  110. except Exception:
  111. return
  112. if sealed_steward(Path(PROJECT_ROOT)) != STEWARD_DESKTOP:
  113. return
  114. from hermes_cli.doctor import HERMES_HOME, _DHH
  115. checkout = HERMES_HOME / "hermes-agent"
  116. if not (checkout / ".git").exists():
  117. return
  118. _section("Legacy Desktop Checkout")
  119. def _git(*args: str):
  120. try:
  121. return subprocess.run(
  122. ["git", "-C", str(checkout), *args],
  123. capture_output=True, text=True, encoding="utf-8", timeout=10,
  124. check=False,
  125. )
  126. except (OSError, subprocess.SubprocessError):
  127. return None
  128. status = _git("status", "--porcelain")
  129. branch = _git("rev-parse", "--abbrev-ref", "HEAD")
  130. stashes = _git("stash", "list")
  131. # Conservative pristineness: every probe must succeed AND come back
  132. # clean. Any probe failure counts as "has local work".
  133. pristine = (
  134. status is not None and status.returncode == 0 and status.stdout.strip() == ""
  135. and branch is not None and branch.returncode == 0 and branch.stdout.strip() == "main"
  136. and stashes is not None and stashes.returncode == 0 and stashes.stdout.strip() == ""
  137. )
  138. size_note = ""
  139. try:
  140. total = sum(f.stat().st_size for f in checkout.rglob("*") if f.is_file())
  141. size_note = f" (~{total / 1_000_000_000:.1f} GB)"
  142. except OSError:
  143. pass
  144. if pristine:
  145. check_warn(
  146. f"Unused checkout at {_DHH}/hermes-agent{size_note}",
  147. "(the desktop app runs embedded and does not use it; the tree is clean)",
  148. )
  149. print(" Review it and decide whether to keep or remove it — doctor does not delete anything.")
  150. else:
  151. check_info(
  152. f"A checkout exists at {_DHH}/hermes-agent but holds local work "
  153. "(changes, a branch, or stashes). The desktop app does not use "
  154. "it; review it before you remove anything."
  155. )
  156. @doctor_check()
  157. def _check_directory_structure(should_fix: bool, f: Finding) -> None:
  158. """HERMES_HOME, expected subdirs, SOUL.md, and the enabled built-in memory files."""
  159. try:
  160. check_legacy_desktop_checkout()
  161. except Exception:
  162. pass # best-effort report; must never break the directory check
  163. from hermes_cli.doctor import HERMES_HOME, _DHH
  164. hermes_home = HERMES_HOME
  165. ensure_dir(f, should_fix, hermes_home, f"{_DHH} directory exists", f"Created {_DHH} directory", f"{_DHH} not found")
  166. _memory_enabled, _user_profile_enabled = _memory_store_flags(hermes_home)
  167. memory_on = bool(_memory_enabled or _user_profile_enabled)
  168. # The built-in file store neither creates nor consumes memories/ when both targets are disabled.
  169. for subdir_name in ["cron", "sessions", "logs", "skills"] + (["memories"] if memory_on else []):
  170. ensure_dir(f, should_fix, hermes_home / subdir_name, f"{_DHH}/{subdir_name}/ exists",
  171. f"Created {_DHH}/{subdir_name}/", f"{_DHH}/{subdir_name}/ not found")
  172. _check_scratch_dir(hermes_home, _DHH)
  173. # SOUL.md persona file
  174. soul_path = hermes_home / "SOUL.md"
  175. if soul_path.exists():
  176. lines = soul_path.read_text(encoding="utf-8-sig").strip().splitlines()
  177. if any(l.strip() and not l.strip().startswith(("<!--", "-->", "#")) for l in lines):
  178. check_ok(f"{_DHH}/SOUL.md exists (persona configured)")
  179. else: # template comments only (no real content)
  180. check_info(f"{_DHH}/SOUL.md exists but is empty — edit it to customize personality")
  181. else:
  182. check_warn(f"{_DHH}/SOUL.md not found", "(create it to give Hermes a custom personality)")
  183. if should_fix:
  184. soul_path.parent.mkdir(parents=True, exist_ok=True)
  185. soul_path.write_text("# Hermes Agent Persona\n\n<!-- Edit this file to customize how Hermes communicates. -->\n\n"
  186. "You are Hermes, a helpful AI assistant.\n", encoding="utf-8")
  187. check_ok(f"Created {_DHH}/SOUL.md with basic template")
  188. f.fixed += 1
  189. # Only enabled built-in stores: users can disable either legacy file target, and stale migration files
  190. # must not read as active memory usage.
  191. memories_dir = hermes_home / "memories"
  192. if not memory_on:
  193. return check_info("Built-in memory files disabled by config")
  194. existed = memories_dir.exists()
  195. ensure_dir(f, should_fix, memories_dir, f"{_DHH}/memories/ directory exists", f"Created {_DHH}/memories/",
  196. f"{_DHH}/memories/ not found")
  197. for fname in [n for on, n in ((_memory_enabled, "MEMORY.md"), (_user_profile_enabled, "USER.md")) if on and existed]:
  198. if (memories_dir / fname).exists():
  199. check_ok(f"{fname} exists ({len((memories_dir / fname).read_text(encoding='utf-8-sig').strip())} chars)")
  200. else:
  201. check_info(f"{fname} not created yet (will be created when the agent first writes a memory)")
  202. # Cache-root entries at least this big that no pruner covers get a doctor warning.
  203. _UNPRUNED_CACHE_WARN_BYTES = 1 << 30
  204. _PRUNED_CACHE_DIRS = frozenset({"scratch", "terminal"})
  205. def unpruned_cache_hogs(hermes_home: Path, min_bytes: int = _UNPRUNED_CACHE_WARN_BYTES) -> list[tuple[str, int]]:
  206. """``(name, bytes)`` for ``cache/`` entries outside the pruned dirs that exceed *min_bytes*.
  207. Finished campaign trees parked at the cache root sat for weeks (95 GB on one host)
  208. because only ``scratch/`` and ``terminal/`` are reaped; doctor is where that shows."""
  209. from hermes_constants import scratch_dir_usage_bytes
  210. cache = hermes_home / "cache"
  211. hogs: list[tuple[str, int]] = []
  212. try:
  213. entries = [e for e in cache.iterdir() if e.is_dir() and not e.is_symlink() and e.name not in _PRUNED_CACHE_DIRS]
  214. except OSError:
  215. return hogs
  216. for entry in entries:
  217. size = scratch_dir_usage_bytes(entry)
  218. if size >= min_bytes:
  219. hogs.append((entry.name, size))
  220. return sorted(hogs, key=lambda item: -item[1])
  221. def _check_scratch_dir(hermes_home: Path, _DHH: str) -> None:
  222. """Report the scratch dir (TMPDIR target) and its size; a user-set TMPDIR elsewhere is shown, not judged."""
  223. from hermes_constants import (
  224. SCRATCH_DIR_MARKER_ENV, SCRATCH_MAX_IDLE_HOURS, get_scratch_dir, scratch_dir_usage_bytes)
  225. scratch = get_scratch_dir(hermes_home, prune=False)
  226. size = _human_bytes(scratch_dir_usage_bytes(scratch))
  227. check_ok(f"{_DHH}/cache/scratch/ is the scratch dir (TMPDIR; {size}, entries pruned after {SCRATCH_MAX_IDLE_HOURS}h idle)")
  228. from hermes_constants_scratch_stuck import removal_hint, stuck_entries
  229. for entry in stuck_entries(scratch):
  230. check_warn(f"{_DHH}/cache/scratch/{entry.name}/ holds files you may not delete, so the prune skips it — {removal_hint(entry)}")
  231. for name, nbytes in unpruned_cache_hogs(hermes_home):
  232. check_warn(
  233. f"{_DHH}/cache/{name}/ is {_human_bytes(nbytes)} and outside every pruner "
  234. f"(only cache/scratch/ and cache/terminal/ are reaped) — move task files under "
  235. f"cache/scratch/<task>/ or delete it"
  236. )
  237. tmpdir = os.environ.get("TMPDIR", "")
  238. if tmpdir and tmpdir != os.environ.get(SCRATCH_DIR_MARKER_ENV, ""):
  239. check_info(f"TMPDIR={tmpdir} is set by you or the OS, so Hermes leaves it alone")
  240. def _session_count(state_db_path: Path):
  241. import sqlite3
  242. # mode=ro: doctor is a reader; a writable open of a gateway-held WAL DB is the second-writer class (#103339).
  243. conn = sqlite3.connect(read_only_db_uri(state_db_path), uri=True)
  244. try:
  245. return conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0]
  246. finally:
  247. conn.close()
  248. # Above this the snapshot copy a held store needs costs more than the probe is worth; --fix still probes.
  249. _WRITE_PROBE_SNAPSHOT_MAX_BYTES = 1 << 30
  250. def _write_health_reason(state_db_path: Path, *, should_fix: bool):
  251. """FTS/write-health probe (a rolled-back BEGIN IMMEDIATE). Against a store a live writer holds,
  252. that probe is the second-writer class (#103339), so probe a read-only snapshot instead; a quiet
  253. store is probed in place. Returns the failure reason, or None when healthy or skipped."""
  254. from hermes_state_repair import _db_opens_cleanly, _live_writer_holds_db
  255. if not _live_writer_holds_db(state_db_path):
  256. return _db_opens_cleanly(state_db_path)
  257. if not should_fix and state_db_path.stat().st_size > _WRITE_PROBE_SNAPSHOT_MAX_BYTES:
  258. check_info("state.db write-health probe skipped: store is held by a live writer and larger than 1 GB "
  259. "(run 'hermes doctor --fix' to probe it)")
  260. return None
  261. import sqlite3
  262. import tempfile
  263. with tempfile.TemporaryDirectory() as tmp:
  264. snapshot = Path(tmp) / "state.db"
  265. src = sqlite3.connect(read_only_db_uri(state_db_path), uri=True, timeout=1.0)
  266. try:
  267. dest = sqlite3.connect(str(snapshot))
  268. try:
  269. src.backup(dest)
  270. finally:
  271. dest.close()
  272. finally:
  273. src.close()
  274. return _db_opens_cleanly(snapshot)
  275. # Corruption class -> (ok label, not-fixed label, failed issue, fix hint). ``{count}`` = recovered sessions.
  276. # ``structural`` has no in-place repair: an FTS rebuild cannot fix a canonical b-tree, and the
  277. # ``.malformed-backup`` the repair path would leave beside state.db is a copy of the same damage (#88587).
  278. _STATE_DB_REPAIRS = {
  279. "fts": ("Repaired state.db FTS write health",
  280. "state.db FTS write-health repair did not recover automatically",
  281. "state.db FTS write corruption and auto-repair failed — restore from the backup copy beside state.db",
  282. "state.db FTS write corruption — run 'hermes doctor --fix' (or 'hermes sessions repair') to rebuild the FTS index"),
  283. "schema": ("Repaired state.db schema ({count} sessions recovered)",
  284. "state.db schema repair did not recover automatically",
  285. "state.db schema malformed and auto-repair failed — restore from the backup copy beside state.db",
  286. "state.db schema malformed — run 'hermes doctor --fix' (or 'hermes sessions repair') to recover hidden sessions"),
  287. }
  288. _STATE_DB_STRUCTURAL_ISSUE = (
  289. "state.db structural corruption (canonical tables/indexes damaged, not the FTS index) — an FTS rebuild "
  290. "cannot repair it. Stop the gateway, then run 'hermes {profile_arg}sessions recover --source {db_path} "
  291. "--inspect-only' and, if it reports recoverable, 'hermes {profile_arg}sessions recover --source {db_path} "
  292. "--output recovered-state.db'. Do NOT restore a .malformed-backup copy beside state.db: it is a snapshot "
  293. "of the same corrupt file."
  294. )
  295. def _repair_state_db(f: Finding, should_fix: bool, state_db_path: Path, kind: str) -> None:
  296. """Shared --fix path for both state.db corruption classes (FTS write health, malformed schema)."""
  297. if kind == "structural":
  298. from hermes_constants import profile_cli_selector
  299. return f.manual_issues.append(_STATE_DB_STRUCTURAL_ISSUE.format(
  300. profile_arg=profile_cli_selector(), db_path=state_db_path))
  301. ok_label, not_fixed_label, failed_issue, fix_hint = _STATE_DB_REPAIRS[kind]
  302. if not should_fix:
  303. return f.issues.append(fix_hint)
  304. from hermes_state_repair import repair_state_db_schema
  305. report = repair_state_db_schema(state_db_path)
  306. if not report.get("repaired"):
  307. check_warn(not_fixed_label, f"({report.get('error')}; backup: {report.get('backup_path')})")
  308. return f.issues.append(failed_issue)
  309. if "{count}" in ok_label:
  310. try:
  311. ok_label = ok_label.format(count=_session_count(state_db_path))
  312. except Exception:
  313. ok_label = ok_label.format(count="?")
  314. backup_name = Path(report["backup_path"]).name if report.get("backup_path") else "n/a"
  315. check_ok(ok_label, f"(strategy: {report.get('strategy')}; backup: {backup_name})")
  316. f.fixed += 1
  317. def _report_structural_damage(f: Finding, should_fix: bool, state_db_path: Path, _DHH: str, reason) -> bool:
  318. """True (and reported/repaired) when the canonical b-tree, not just the FTS index, is damaged."""
  319. from hermes_state_repair import state_db_has_structural_damage
  320. if not state_db_has_structural_damage(state_db_path):
  321. return False
  322. check_warn(f"{_DHH}/state.db has structural corruption (canonical tables/indexes damaged, "
  323. "not the FTS index)", f"({reason})")
  324. _repair_state_db(f, should_fix, state_db_path, "structural")
  325. return True
  326. def _classify_unreadable_state_db(f: Finding, should_fix: bool, state_db_path: Path, _DHH: str, exc: Exception) -> None:
  327. """Structural damage first; only then schema repair. Avoids SessionDB auto-repair side effects."""
  328. from hermes_state import is_malformed_db_error
  329. if _report_structural_damage(f, should_fix, state_db_path, _DHH, exc):
  330. return
  331. if not is_malformed_db_error(exc):
  332. return check_warn(f"{_DHH}/state.db exists but has issues: {exc}")
  333. # sqlite_master itself is malformed (e.g. duplicate messages_fts): every statement fails before it runs,
  334. # so this is NOT a plain FTS rebuild — repair sqlite_master in place (backup first).
  335. check_warn(f"{_DHH}/state.db schema is malformed (sessions hidden until repaired)", f"({exc})")
  336. _repair_state_db(f, should_fix, state_db_path, "schema")
  337. def _state_db_health(f: Finding, should_fix: bool, state_db_path: Path, _DHH: str) -> None:
  338. """Session count + FTS write-health probe; malformed-schema path when even COUNT(*) fails."""
  339. try:
  340. check_ok(f"{_DHH}/state.db exists ({_session_count(state_db_path)} sessions)")
  341. # COUNT(*) succeeds even when the FTS index is corrupt and every write fails through the triggers.
  342. _write_reason = _write_health_reason(state_db_path, should_fix=should_fix)
  343. except Exception as e:
  344. return _classify_unreadable_state_db(f, should_fix, state_db_path, _DHH, e)
  345. if _write_reason is not None:
  346. if _report_structural_damage(f, should_fix, state_db_path, _DHH, _write_reason):
  347. return
  348. check_warn(f"{_DHH}/state.db fails a write-health probe (FTS index may be corrupt)", f"({_write_reason})")
  349. _repair_state_db(f, should_fix, state_db_path, "fts")
  350. def _state_db_stats(issues: list, state_db_path: Path) -> None:
  351. """Health/stats snapshot: strictly read-only (mode=ro) so it is safe against a live DB held by
  352. the gateway; any failure degrades to one info line rather than failing doctor."""
  353. with warn_on_error("state.db stats unavailable ({e})", "", report=lambda t, _d: check_info(t)):
  354. from hermes_state_dbfile import collect_state_db_stats, count_db_holders
  355. rows = _render_state_db_stats(collect_state_db_stats(state_db_path), holders=count_db_holders(state_db_path),
  356. host_note=host_gateway_note())
  357. for _kind, _text, _detail in rows:
  358. if _kind != "warn":
  359. check_info(_text + (f" {_detail}" if _detail else ""))
  360. continue
  361. check_warn(_text, _detail)
  362. if "auto_prune" in _detail:
  363. issues.append("state.db is large — enable sessions.auto_prune in config.yaml"
  364. + (" and run 'hermes sessions optimize-storage' offline (gateway stopped)" if "optimize-storage" in _detail else ""))
  365. def _state_db_wal(f: Finding, should_fix: bool, state_db_path: Path) -> None:
  366. """WAL file size (unbounded growth indicates missed checkpoints)."""
  367. wal_path = state_db_path.parent / "state.db-wal"
  368. wal_size = lambda: wal_path.stat().st_size if wal_path.exists() else 0
  369. with warn_on_error(""):
  370. size = wal_size()
  371. if size > 50 * 1024 * 1024: # 50 MB
  372. # Checkpoint-lock premise (#40177, #103339): a bare connect runs WAL recovery and the checkpoint
  373. # joins the live WAL — under a running gateway that second-writer handling corrupts state.db.
  374. # Holder scan first (any other process holding the DB, or an unknown, fails closed), then run the
  375. # checkpoint on the exclusive repair guard so an opener arriving in between is refused, not joined.
  376. from hermes_state_repair import _exclusive_repair_db_guard, _live_writer_holds_db
  377. title = f"WAL file is large ({size // (1024*1024)} MB)"
  378. _SKIP = ("Large WAL file — cannot prove state.db is quiet (stop the profile's gateway first, then "
  379. "run 'hermes doctor --fix' to checkpoint)")
  380. # Honest disjunction (gate C1): a True here means "held OR unprovable" — never assert a live
  381. # writer as fact.
  382. if _live_writer_holds_db(state_db_path):
  383. # A large WAL is normal while Desktop or the gateway is running; a bare "run --fix" here sent
  384. # users straight into the second-writer trap (#110054).
  385. check_warn(title, "(normal while Desktop or the gateway is running, or state.db cannot be "
  386. "inspected — checkpoint only with them stopped)")
  387. return f.issues.append(_SKIP)
  388. check_warn(title, "(may indicate missed checkpoints)")
  389. if not should_fix:
  390. return f.issues.append(
  391. "Large WAL file — stop the profile's gateway, then run 'hermes doctor --fix' to checkpoint")
  392. with _exclusive_repair_db_guard(state_db_path) as (guard, guard_error):
  393. if guard is None:
  394. check_warn("WAL checkpoint skipped: could not take exclusive ownership of state.db",
  395. f"({guard_error}; stop the profile's gateway and re-run 'hermes doctor --fix')")
  396. return f.issues.append(_SKIP)
  397. guard.execute("PRAGMA wal_checkpoint(PASSIVE)")
  398. check_ok(f"WAL checkpoint performed ({size // 1024}K → {wal_size() // 1024}K)")
  399. f.fixed += 1
  400. elif size > 10 * 1024 * 1024: # 10 MB
  401. check_info(f"WAL file is {size // (1024*1024)} MB (normal for active sessions)")
  402. def _retired_wal_holders(f: Finding, state_db_path: Path, _DHH: str) -> bool:
  403. """Name the processes holding a retired -wal/-shm generation (#110054). Every SessionDB open is
  404. refused while they live, and the current inode has no holders, so the plain holder count says
  405. "0 holding the DB open" beside a green state.db line — the opposite of the truth."""
  406. from hermes_constants import profile_cli_selector
  407. from hermes_state_dbfile import iter_deleted_sqlite_sidecar_holders
  408. from hermes_state_holders import describe_holder_pid
  409. pids = list(dict.fromkeys(pid for pid, _ in iter_deleted_sqlite_sidecar_holders(state_db_path)))
  410. if not pids:
  411. return False
  412. rendered = ", ".join(describe_holder_pid(pid) for pid in pids)
  413. check_warn(f"{_DHH}/state.db: {len(pids)} process(es) still hold a retired WAL generation ({rendered})",
  414. "(every new session refuses to open until they exit; health/stats probes skipped)")
  415. f.issues.append(f"state.db retired WAL generation held by {rendered}{host_gateway_note()} — stop the host "
  416. f"gateway, dashboard and cron writers among them ('hermes {profile_cli_selector()}gateway "
  417. "stop' stops the ONE host process serving every profile, quit the Desktop app), do not "
  418. "delete the WAL yourself, then rerun 'hermes doctor'")
  419. return True
  420. @doctor_check()
  421. def _check_state_db(should_fix: bool, f: Finding) -> None:
  422. """state.db session count, FTS write health, schema repair, stats snapshot, WAL size."""
  423. from hermes_cli.doctor import HERMES_HOME, _DHH
  424. state_db_path = HERMES_HOME / "state.db"
  425. # A read-only connect on the new generation is itself another opener, so nothing below may run.
  426. if _retired_wal_holders(f, state_db_path, _DHH):
  427. return
  428. if state_db_path.exists():
  429. _state_db_health(f, should_fix, state_db_path, _DHH)
  430. _state_db_stats(f.issues, state_db_path)
  431. else:
  432. check_info(f"{_DHH}/state.db not created yet (will be created on first session)")
  433. _state_db_wal(f, should_fix, state_db_path)
  434. @doctor_check()
  435. def _check_cron_store(should_fix: bool, f: Finding) -> None:
  436. """The cron store accepts writes and its filesystem is not nearly full (jobs skip otherwise)."""
  437. from hermes_cli.doctor import HERMES_HOME, _DHH
  438. from cron.store_health import LOW_FREE_BYTES, free_bytes, probe_report
  439. cron_dir = HERMES_HOME / "cron"
  440. if (report := probe_report(cron_dir)) is not None:
  441. check_warn(f"{_DHH}/cron/ is NOT writable ({report['error']}) — scheduled cron jobs are being skipped")
  442. return f.issues.append(f"Cron store {report['store']} is not writable: {report['fix']}")
  443. free = free_bytes(cron_dir) if cron_dir.is_dir() else None
  444. if free is not None and free < LOW_FREE_BYTES:
  445. check_warn(f"{_DHH}/cron/ filesystem has only {_human_bytes(free)} free — cron jobs stop when it fills")
  446. return f.issues.append(f"Free disk space on the filesystem holding {cron_dir}")
  447. check_ok(f"{_DHH}/cron/ store is writable")
  448. @doctor_check()
  449. def _check_checkpoint_store(should_fix: bool, f: Finding) -> None:
  450. """/rollback store footprint: warn when checkpoints are on and the store sits above its cap."""
  451. from tools.checkpoint_maintenance import checkpoint_footprint_notice
  452. notice = checkpoint_footprint_notice()
  453. if notice:
  454. check_warn(notice)
  455. def _plugin_provenance_rows(plugins_dir) -> list:
  456. """Report local provenance using the update checker's admission rules."""
  457. from hermes_cli.plugins_provenance import ProvenanceClass, plugins_provenance
  458. from hermes_cli.plugins_updates import check_local_provenance
  459. if plugins_dir is None or not Path(plugins_dir).is_dir():
  460. return [("info", "No plugins directory yet (nothing to check provenance for)", "")]
  461. try:
  462. provenances = plugins_provenance(Path(plugins_dir))
  463. except (OSError, ValueError, RuntimeError) as exc:
  464. return [("warn", "Plugin provenance could not be read", str(exc))]
  465. rows = []
  466. for provenance in provenances:
  467. result = check_local_provenance(provenance)
  468. detail = result.needs_fixing or result.reason
  469. if result.needs_fixing or provenance.klass is ProvenanceClass.DRIFT:
  470. rows.append(("warn", f"Plugin '{provenance.name}': {detail}", ""))
  471. elif provenance.klass is ProvenanceClass.MANUAL:
  472. rows.append(("info", f"Plugin '{provenance.name}' installed manually", detail))
  473. elif provenance.klass is ProvenanceClass.SELF_CLONED:
  474. rows.append(("info", f"Plugin '{provenance.name}': self-cloned", detail))
  475. else:
  476. rows.append(("ok", f"Plugin '{provenance.name}': provenance in good standing", detail))
  477. return rows or [("info", "No provenanced plugins found", "")]
  478. @doctor_check("")
  479. def _check_update_provenance(should_fix: bool, f: Finding) -> None:
  480. """Inspect local plugin update provenance without network or writes."""
  481. from hermes_constants import get_hermes_home
  482. printers = {"warn": check_warn, "ok": check_ok, "info": lambda text, detail: check_info(f"{text} {detail}".rstrip())}
  483. for kind, text, detail in _plugin_provenance_rows(get_hermes_home() / "plugins"):
  484. printers[kind](text, detail)
  485. def _gh_authenticated() -> bool:
  486. """Check if gh CLI is authenticated via token file or device flow.
  487. Plain ``gh auth status`` (exit code only): gh 2.98+ dropped the ``authenticated`` JSON
  488. field, so ``--json authenticated`` exits 1 even when logged in. Availability is resolved
  489. through shutil.which (the same probe every other doctor tool check uses); an OS-level
  490. launch failure (a Store/MSIX shim, a deleted binary) reads as "not authenticated" rather
  491. than crashing the Skills Hub check.
  492. """
  493. from hermes_cli.doctor_tools import _safe_which
  494. if not _safe_which("gh"):
  495. return False
  496. try:
  497. result = subprocess.run(["gh", "auth", "status"], capture_output=True, timeout=10, check=False)
  498. return result.returncode == 0
  499. except (OSError, subprocess.TimeoutExpired):
  500. return False
  501. @doctor_check()
  502. def _check_skills_hub(should_fix: bool, f: Finding) -> None:
  503. from hermes_cli.doctor import HERMES_HOME, _DHH
  504. hub_dir = HERMES_HOME / "skills" / ".hub"
  505. if check_bool(hub_dir.exists(), "Skills Hub directory exists", ("Skills Hub directory not initialized", "(run: hermes skills list)")):
  506. lock_file = hub_dir / "lock.json"
  507. if lock_file.exists():
  508. with warn_on_error("Lock file", "(corrupted or unreadable)"):
  509. import json
  510. count = len(json.loads(lock_file.read_text(encoding="utf-8-sig")).get("installed", {}))
  511. check_ok(f"Lock file OK ({count} hub-installed skill(s))")
  512. quarantine = hub_dir / "quarantine"
  513. q_count = sum(1 for d in quarantine.iterdir() if d.is_dir()) if quarantine.exists() else 0
  514. if q_count > 0:
  515. check_warn(f"{q_count} skill(s) in quarantine", "(pending review)")
  516. from hermes_cli.config import get_env_value
  517. if get_env_value("GITHUB_TOKEN") or get_env_value("GH_TOKEN"):
  518. check_ok("GitHub token configured", "(validity checked under API Connectivity)")
  519. else:
  520. check_bool(_gh_authenticated(), ("GitHub authenticated via gh CLI", "(full API access — no GITHUB_TOKEN needed)"),
  521. ("No GITHUB_TOKEN", f"(60 req/hr rate limit — set in {_DHH}/.env for better rates)"))
  522. # provider -> (checker, ImportError row, ImportError issue, label for "check failed"); every provider
  523. # still bundled or catalog-installed uses the generic check.
  524. _MEMORY_PROVIDER_CHECKS: dict = {}
  525. def _memory_provider_generic(name: str) -> None:
  526. """Generic check for every memory provider (honcho, hindsight, mem0, openviking, ...)."""
  527. from plugins.memory import load_memory_provider
  528. _provider = load_memory_provider(name)
  529. if _provider and _provider.is_available():
  530. check_ok(f"{name} provider active")
  531. elif _provider:
  532. check_warn(f"{name} configured but not available", "run: hermes memory status")
  533. else:
  534. from plugins.memory import find_provider_dir
  535. from hermes_cli.memory_provider_migration import catalog_install_hint
  536. hint = catalog_install_hint(name, category="memory") if find_provider_dir(name) is None else None
  537. check_warn(f"{name} plugin not found", f"run: {hint or 'hermes memory setup'}")
  538. @doctor_check()
  539. def _check_memory_provider(should_fix: bool, f: Finding) -> None:
  540. from hermes_cli.doctor import HERMES_HOME
  541. from agent.memory_provider import is_core_memory_provider
  542. name = _doctor_memory_config(HERMES_HOME).get("provider", "")
  543. if is_core_memory_provider(name):
  544. check_ok("Built-in memory active", "(no external provider configured — this is fine)")
  545. return
  546. from plugins.memory import parked_provider_notice
  547. if parked_provider_notice(name):
  548. check_warn(f"{name} is turned off", "(plugins.disabled lists it, so built-in memory is used)")
  549. f.manual_issues.append(f"Memory provider '{name}' is turned off — run: hermes plugins enable {name}")
  550. return
  551. checker, missing_row, missing_issue, label = _MEMORY_PROVIDER_CHECKS.get(name, (None, None, None, name))
  552. try:
  553. checker(f.issues) if checker else _memory_provider_generic(name)
  554. except ImportError as _e:
  555. if missing_row is None:
  556. check_warn(f"{label} check failed", str(_e))
  557. else:
  558. _fail_and_issue(*missing_row, missing_issue, f.issues)
  559. except Exception as _e:
  560. check_warn(f"{label} check failed", str(_e))
  561. @doctor_check("") # best-effort: an unreadable config is the config check's finding
  562. def _check_plugins_disabled_by_update(should_fix: bool, f: Finding) -> None:
  563. """Plugins an update turned off (pm.plugin_eviction): every later update retries them."""
  564. from hermes_cli.doctor import HERMES_HOME
  565. from pm.plugins_state import RECORD, read_home_selection
  566. plugins = (read_home_selection(HERMES_HOME) or {}).get("plugins") or {}
  567. record, disabled = plugins.get(RECORD), set(plugins.get("disabled") or ())
  568. entries = {name: entry for name, entry in record.items() if name in disabled} if isinstance(record, dict) else {}
  569. if not entries:
  570. return
  571. _section("Plugins Turned Off by an Update")
  572. for name, entry in entries.items():
  573. reason = entry.get("reason", "") if isinstance(entry, dict) else ""
  574. check_warn(f"{name} is off", f"({reason})" if reason else "")
  575. f.manual_issues.append(f"Plugin '{name}' was turned off by an update; a later update retries it, "
  576. f"or run: hermes plugins enable {name}")
  577. @doctor_check("") # best-effort: an unreadable config is the config check's finding
  578. def _check_plugins_in_both_lists(should_fix: bool, f: Finding) -> None:
  579. """A plugin in both plugins.enabled and plugins.disabled loads nowhere (the disable wins). Older
  580. updates left plugins in that state when they had to turn them off. Judged by the loader's gate."""
  581. from hermes_cli.plugins_discovery import plugins_in_both_lists
  582. both = plugins_in_both_lists()
  583. if not both:
  584. return
  585. _section("Plugins Listed as Both Enabled and Disabled")
  586. for name in both:
  587. check_warn(f"{name} will not load", "(plugins.disabled wins)")
  588. f.manual_issues.append(f"Plugin '{name}' is in both plugins.enabled and plugins.disabled, so it is off. "
  589. f"Turn it on with `hermes plugins enable {name}` (a memory provider also needs "
  590. f"memory.provider set), or keep it off with `hermes plugins disable {name}`")
  591. @doctor_check("") # best-effort: profile enumeration must never break doctor
  592. def _check_profiles(should_fix: bool, f: Finding) -> None:
  593. from hermes_constants import PROFILE_ID_RE
  594. from hermes_cli.config_backend import config_exists
  595. from hermes_cli.profiles import _wrapper_path, list_profiles, _get_wrapper_dir, profile_exists
  596. import re as _re
  597. named_profiles = [p for p in list_profiles() if not p.is_default]
  598. if not named_profiles:
  599. return
  600. _section("Profiles")
  601. check_ok(f"{len(named_profiles)} profile(s) found")
  602. wrapper_dir = _get_wrapper_dir()
  603. for p in named_profiles:
  604. parts = [text for cond, text in (
  605. (p.gateway_running, "gateway running"), (p.model, (p.model or "")[:30]),
  606. (not config_exists(p.path / "config.yaml"), "⚠ missing config"), (not (p.path / ".env").exists(), "no .env"),
  607. (p.alias_path is None, "no alias")) if cond]
  608. check_ok(f" {p.name}: {', '.join(parts) if parts else 'configured'}")
  609. # Orphan wrappers
  610. if wrapper_dir.is_dir():
  611. for wrapper in wrapper_dir.iterdir():
  612. if not wrapper.is_file():
  613. continue
  614. with warn_on_error(""):
  615. _m = _re.search(r"hermes -p (\S+)", wrapper.read_text(encoding="utf-8-sig"))
  616. if _m and not profile_exists(_m.group(1)):
  617. alias = wrapper.stem if _wrapper_path(wrapper.stem) == wrapper else wrapper.name
  618. fix = f"hermes profile alias {alias} --remove" if PROFILE_ID_RE.match(alias) else f"delete {wrapper}"
  619. check_warn(f"Orphan alias: {alias} → profile '{_m.group(1)}' no longer exists", f"(remove with: {fix})")
  620. # Same helper as the multiplex migration preflight, so doctor names the duplicates that make
  621. # `hermes gateway migrate --multiplex` refuse (and made pre-multiplex standalone gateways race).
  622. from hermes_cli.gateway_migrate import duplicate_credential_findings
  623. for line in duplicate_credential_findings():
  624. check_warn("Duplicate platform credential across profiles", f"({line})")
  625. f.manual_issues.append(line)