hermes_state_holders.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. """Process and descriptor authority for state.db structural maintenance.
  2. This module owns the proof that no foreign process still holds the active or
  3. an unlinked SQLite DB/WAL/SHM generation. ``hermes_state`` supplies only the
  4. SQLite connection factory needed by the final lock probe.
  5. """
  6. from __future__ import annotations
  7. import errno
  8. import functools
  9. import logging
  10. import os
  11. import sqlite3
  12. import sys
  13. from pathlib import Path
  14. from typing import Callable, List, Optional, Sequence, Set, Tuple
  15. from hermes_state_errors import is_sqlite_lock_error
  16. try: # Hard dependency, but tolerate scaffold-phase imports before pip install.
  17. import psutil
  18. except ImportError: # pragma: no cover - stripped/scaffold installs only
  19. psutil = None # type: ignore[assignment]
  20. def read_only_db_uri(db_path) -> str:
  21. """``file:`` URI for a ``mode=ro`` open. ``as_uri()`` percent-encodes ``?``/``#`` in the home
  22. path; a raw ``f"file:{path}?mode=ro"`` truncates there and opens the wrong (empty) database."""
  23. return Path(db_path).resolve().as_uri() + "?mode=ro"
  24. logger = logging.getLogger(__name__)
  25. _IS_WINDOWS = sys.platform == "win32"
  26. _HERMES_EXECUTABLES = frozenset({"hermes", "hermes-agent", "hermes-acp"})
  27. _HERMES_PYTHON_MODULES = frozenset({"acp_adapter", "hermes_cli.main"})
  28. _HERMES_PYTHON_SCRIPTS = frozenset({"hermes_cli/main.py", "run_agent.py"})
  29. _PYTHON_SHORT_OPTIONS_WITH_OPERANDS = frozenset({"Q", "W", "X"})
  30. _PYTHON_LONG_OPTIONS_WITH_OPERANDS = frozenset(
  31. {"--check-hash-based-pycs", "--jit"}
  32. )
  33. def _read_proc_argv(pid: int) -> Optional[list[str]]:
  34. """Read /proc/<pid>/cmdline without losing argv boundaries."""
  35. try:
  36. with open(f"/proc/{pid}/cmdline", "rb") as handle:
  37. raw = handle.read()
  38. if not raw:
  39. return None
  40. argv = raw.decode("utf-8", "replace").split("\x00")
  41. if argv[-1] == "":
  42. argv.pop()
  43. return argv or None
  44. except OSError:
  45. return None
  46. def describe_holder_pid(pid: int) -> str:
  47. """``PID 123 (hermes gateway run)`` for operator-facing holder lists; /proc argv first, psutil elsewhere."""
  48. argv = _read_proc_argv(pid)
  49. if argv is None and psutil is not None:
  50. try:
  51. argv = psutil.Process(pid).cmdline() or None
  52. except Exception:
  53. argv = None
  54. who = " ".join(" ".join([os.path.basename(argv[0]), *argv[1:]]).split())[:80] if argv else "command line unavailable"
  55. return f"PID {pid} ({who})"
  56. def _looks_like_python_executable(program: str) -> bool:
  57. name = os.path.basename(program).lower().removesuffix(".exe")
  58. for prefix in ("python", "pypy"):
  59. if name.startswith(prefix):
  60. suffix = name[len(prefix) :]
  61. return not suffix or all(char.isdigit() or char == "." for char in suffix)
  62. return False
  63. def _python_execution_target(argv: Sequence[str]) -> Optional[tuple[str, str]]:
  64. """Return the Python module or script selected by interpreter options."""
  65. index = 1
  66. while index < len(argv):
  67. arg = argv[index]
  68. if arg == "--":
  69. index += 1
  70. return ("script", argv[index]) if index < len(argv) else None
  71. if arg in _PYTHON_LONG_OPTIONS_WITH_OPERANDS:
  72. index += 2
  73. continue
  74. if arg.startswith("--check-hash-based-pycs=") or arg.startswith("--jit="):
  75. index += 1
  76. continue
  77. if arg.startswith("--"):
  78. index += 1
  79. continue
  80. if arg.startswith("-") and arg != "-":
  81. options = arg[1:]
  82. option_index = 0
  83. consumed_next = False
  84. while option_index < len(options):
  85. option = options[option_index]
  86. attached = options[option_index + 1 :]
  87. if option == "c":
  88. return None
  89. if option == "m":
  90. if attached:
  91. return "module", attached
  92. index += 1
  93. return ("module", argv[index]) if index < len(argv) else None
  94. if option in _PYTHON_SHORT_OPTIONS_WITH_OPERANDS:
  95. consumed_next = not attached
  96. break
  97. option_index += 1
  98. index += 2 if consumed_next else 1
  99. continue
  100. return "script", arg
  101. return None
  102. def _looks_like_hermes(argv: Sequence[str]) -> bool:
  103. """Return whether argv identifies a supported Hermes execution target."""
  104. if not argv:
  105. return False
  106. program = os.path.basename(argv[0]).lower().removesuffix(".exe")
  107. if program in _HERMES_EXECUTABLES:
  108. return True
  109. if not _looks_like_python_executable(program):
  110. return False
  111. target = _python_execution_target(argv)
  112. if target is None:
  113. return False
  114. kind, value = target
  115. normalized = value.lower().replace("\\", "/")
  116. if kind == "module":
  117. return normalized in _HERMES_PYTHON_MODULES
  118. return any(
  119. normalized == script or normalized.endswith(f"/{script}")
  120. for script in _HERMES_PYTHON_SCRIPTS
  121. )
  122. def canonical_sqlite_path(path: str) -> str:
  123. """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix."""
  124. return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)")))
  125. _HOME_FLAGS = ("--hermes-home",)
  126. _PROFILE_FLAGS = ("--profile", "-p")
  127. _STATE_DB_NAMES = ("state.db", "state.db-wal", "state.db-shm")
  128. def _norm_path(value: str) -> str:
  129. return os.path.normcase(os.path.normpath(value))
  130. def _argv_flag_value(argv: Sequence[str], flags: Sequence[str]) -> Optional[str]:
  131. """Last ``--flag X`` / ``--flag=X`` value, token-exact (``--profile timothy`` is not ``tim``)."""
  132. value: Optional[str] = None
  133. index, count = 0, len(argv)
  134. while index < count:
  135. token = argv[index]
  136. if isinstance(token, str):
  137. if token in flags and index + 1 < count and isinstance(argv[index + 1], str):
  138. value = argv[index + 1]
  139. index += 2
  140. continue
  141. for flag in flags:
  142. if token.startswith(flag + "="):
  143. value = token[len(flag) + 1:]
  144. break
  145. index += 1
  146. return value
  147. def _argv_env_home(argv: Sequence[str]) -> Optional[str]:
  148. """``HERMES_HOME=<path>`` env-style assignment on the argv (``env HERMES_HOME=… hermes …``)."""
  149. for token in reversed(list(argv)):
  150. if isinstance(token, str) and token.startswith("HERMES_HOME="):
  151. return token[len("HERMES_HOME="):]
  152. return None
  153. def _store_install_layout(this_home: str) -> tuple[Optional[str], Optional[str]]:
  154. """``(<install root>, <our profile name>)`` for the home holding the store, else ``(None, None)``.
  155. Derived with the canonical ``named_profile_home`` predicate, never a ``basename == "profiles"``
  156. string test: an arbitrary ``<X>/profiles/<n>/`` tree is not a Hermes install, and promoting
  157. ``<X>`` to "ours" swallows an unrelated instance living under it — the literal two-instance
  158. shape of #92401. The root store (``~/.hermes/state.db``) is its own root with no profile name,
  159. so ANY named-profile selection contradicts it.
  160. """
  161. try:
  162. from hermes_constants import named_profile_home
  163. profile_home = named_profile_home(this_home)
  164. if profile_home is not None:
  165. return os.path.abspath(str(profile_home.parent.parent)), profile_home.name
  166. if os.path.basename(this_home) == ".hermes":
  167. return os.path.abspath(this_home), None
  168. except Exception: # constants import/resolution must never break a holder scan
  169. logger.debug("Could not classify the install layout of %s", this_home, exc_info=True)
  170. return None, None
  171. def _names_other_profile(normalized: str, install_root: Optional[str], our_profile: Optional[str]) -> bool:
  172. """True when the token is under ``<install root>/profiles/<name>`` for a name that is not ours."""
  173. if install_root is None:
  174. return False
  175. prefix = _norm_path(os.path.join(install_root, "profiles")) + os.sep
  176. if not normalized.startswith(prefix):
  177. return False
  178. name = normalized[len(prefix):].split(os.sep, 1)[0]
  179. return bool(name) and name != (os.path.normcase(our_profile) if our_profile else None)
  180. def _argv_home_selection(
  181. argv: Sequence[str], this_home: str, install_root: Optional[str], our_profile: Optional[str]
  182. ) -> Optional[str]:
  183. """``"ours"``/``"other"``/``None`` from the process's OWN profile/home selection.
  184. Under one process per host the shared binary path proves nothing about which home a process
  185. serves; its ``--hermes-home``/``HERMES_HOME=``/``--profile``/``-p`` selection does. Same
  186. token-exact parsers ``gateway/run.py::_argv_contradicts_home`` uses.
  187. """
  188. home_value = _argv_flag_value(argv, _HOME_FLAGS) or _argv_env_home(argv)
  189. if home_value:
  190. return "ours" if _norm_path(home_value) == _norm_path(this_home) else "other"
  191. profile_value = _argv_flag_value(argv, _PROFILE_FLAGS)
  192. if profile_value:
  193. if our_profile is not None:
  194. return "ours" if profile_value == our_profile else "other"
  195. # Root/custom home: any explicit named profile selects a different home.
  196. return "ours" if (install_root is not None and profile_value == "default") else "other"
  197. return None
  198. def _argv_path_tokens(argv: Sequence[str]) -> list[tuple[int, str]]:
  199. """``(argv index, normalized absolute path)`` for every path-bearing token."""
  200. tokens: list[tuple[int, str]] = []
  201. for index, token in enumerate(argv):
  202. if not isinstance(token, str):
  203. continue
  204. if token.startswith("/"):
  205. path_token = token
  206. elif token.startswith("-") and "=" in token:
  207. # ``--db=/abs/path``-style options carry a path value; anchor on
  208. # the text after '=' so normpath does not prepend the option.
  209. value = token.split("=", 1)[1]
  210. path_token = value if value.startswith("/") else None
  211. else:
  212. path_token = None
  213. if path_token is not None:
  214. tokens.append((index, _norm_path(path_token)))
  215. return tokens
  216. def _argv_scoped_to_other_home(argv: Sequence[str], db_path: Path) -> bool:
  217. """Return whether argv proves the process belongs to a DIFFERENT instance.
  218. ``state.db`` lives at the HERMES_HOME root, so an absolute-path token
  219. containing a ``/.hermes`` segment (or naming a ``state.db``/WAL/SHM under
  220. some other parent) identifies that token's own Hermes home. When at least
  221. one such token exists AND no token references this instance's state.db,
  222. its sidecars, or its home directory, the process provably works on a
  223. different generation and must not be counted as an uninspectable holder
  224. of ours (issue #92401: a second gateway under /home/demo/.hermes deferred
  225. this instance's stale-FTS rebuild forever despite lsof proving zero open
  226. handles). Ambiguous argv without absolute-path tokens returns False and
  227. keeps the fail-closed suspicion.
  228. Evidence is ranked, because one host now runs ONE process for every profile:
  229. 1. A token naming our state.db or a sidecar exactly — definitive, ours.
  230. 2. The process's own ``--hermes-home``/``HERMES_HOME=``/``--profile``/``-p``
  231. selection — that is what decides which home a multiplexer serves.
  232. 3. Path tokens. A token under ``<install root>/profiles/<other>`` is another
  233. profile's store even though it sits under our root; a token that names only
  234. the SHARED install root is NEUTRAL (it is the same binary for every profile,
  235. so it can neither prove nor disprove a hold); ``argv[0]`` locates the INSTALL,
  236. not the home, so it is not other-home evidence for a store whose home is not
  237. part of an install layout (a custom ``HERMES_HOME`` is served BY the binary
  238. under ``~/.hermes`` — dismissing on it admits maintenance under a live writer).
  239. """
  240. db_path_str = os.path.abspath(os.fspath(db_path))
  241. this_home = os.path.dirname(db_path_str)
  242. install_root, our_profile = _store_install_layout(this_home)
  243. sidecars = {os.path.normcase(candidate) for candidate in _sqlite_family(db_path_str)}
  244. this_home_norm = os.path.normcase(this_home)
  245. root_norm = os.path.normcase(install_root) if install_root else None
  246. path_tokens = _argv_path_tokens(argv)
  247. if any(normalized in sidecars for _, normalized in path_tokens):
  248. return False
  249. selection = _argv_home_selection(argv, this_home, install_root, our_profile)
  250. if selection == "ours":
  251. return False
  252. # A store whose home is not itself part of an install layout cannot be identified from the
  253. # install location, so argv[0] alone never dismisses a holder of it.
  254. argv0_locates_home = install_root is not None
  255. other_home_seen = selection == "other"
  256. for index, normalized in path_tokens:
  257. if _names_other_profile(normalized, install_root, our_profile):
  258. other_home_seen = True
  259. continue
  260. if normalized == this_home_norm or normalized.startswith(this_home_norm + os.sep):
  261. return False
  262. if root_norm is not None and (
  263. normalized == root_norm or normalized.startswith(root_norm + os.sep)
  264. ):
  265. continue # shared install root: neutral, every served profile lives under it
  266. if index == 0 and not argv0_locates_home:
  267. continue
  268. if "/.hermes" in normalized or normalized.endswith("/.hermes") or os.path.basename(normalized) in _STATE_DB_NAMES:
  269. other_home_seen = True
  270. return other_home_seen
  271. _RM_ERROR_MORE_DATA = 234
  272. _RM_SESSION_KEY_LEN = 33 # CCH_RM_SESSION_KEY + 1
  273. @functools.cache
  274. def _rm_ctypes():
  275. """Restart Manager ctypes metadata, built once on first scan.
  276. Once, because ``ctypes.POINTER()`` on a freshly defined Structure pins it in
  277. ``ctypes._pointer_type_cache`` forever (a per-scan rebuild leaked a class set per call);
  278. lazily, because POSIX processes import this module but never scan with Restart Manager.
  279. """
  280. import ctypes
  281. from ctypes import wintypes
  282. class _RmUniqueProcess(ctypes.Structure):
  283. _fields_ = [("pid", wintypes.DWORD), ("started", wintypes.FILETIME)]
  284. class _RmProcessInfo(ctypes.Structure):
  285. _fields_ = [
  286. ("process", _RmUniqueProcess),
  287. ("app_name", wintypes.WCHAR * 256), # CCH_RM_MAX_APP_NAME + 1
  288. ("service_name", wintypes.WCHAR * 64), # CCH_RM_MAX_SVC_NAME + 1
  289. ("app_type", wintypes.DWORD),
  290. ("app_status", wintypes.ULONG),
  291. ("ts_session_id", wintypes.DWORD),
  292. ("restartable", wintypes.BOOL),
  293. ]
  294. argtypes = {
  295. "RmStartSession": [ctypes.POINTER(wintypes.DWORD), wintypes.DWORD, wintypes.LPWSTR],
  296. "RmRegisterResources": [
  297. wintypes.DWORD, wintypes.UINT, ctypes.POINTER(wintypes.LPCWSTR),
  298. wintypes.UINT, ctypes.c_void_p, wintypes.UINT, ctypes.c_void_p,
  299. ],
  300. "RmGetList": [
  301. wintypes.DWORD, ctypes.POINTER(wintypes.UINT), ctypes.POINTER(wintypes.UINT),
  302. ctypes.POINTER(_RmProcessInfo), ctypes.POINTER(wintypes.DWORD),
  303. ],
  304. "RmEndSession": [wintypes.DWORD],
  305. }
  306. return _RmProcessInfo, argtypes
  307. def _sqlite_family(base: str) -> tuple[str, str, str]:
  308. """The main database file plus the ``-wal``/``-shm`` sidecars SQLite may hold alongside it."""
  309. return (base, base + "-wal", base + "-shm")
  310. def _windows_restart_manager_holders(db_path: Path) -> list[tuple[int, str]]:
  311. """Return foreign processes using state.db or a WAL sidecar via Windows Restart Manager."""
  312. import ctypes
  313. from ctypes import wintypes
  314. process_info, argtypes = _rm_ctypes()
  315. # WinDLL per call (not memoised) so the unit test can inject a fake rstrtmgr through ctypes.WinDLL.
  316. api = ctypes.WinDLL("rstrtmgr", use_last_error=True)
  317. for name, types in argtypes.items():
  318. fn = getattr(api, name)
  319. fn.argtypes = types
  320. fn.restype = wintypes.DWORD
  321. start, register, get_list, end = (
  322. api.RmStartSession, api.RmRegisterResources, api.RmGetList, api.RmEndSession
  323. )
  324. db_abspath = os.path.abspath(os.fspath(db_path))
  325. resources = [path for path in _sqlite_family(db_abspath) if os.path.exists(path)]
  326. if not resources:
  327. return []
  328. session = wintypes.DWORD()
  329. key = ctypes.create_unicode_buffer(_RM_SESSION_KEY_LEN)
  330. rc = start(ctypes.byref(session), 0, key)
  331. if rc:
  332. raise OSError(rc, "RmStartSession failed")
  333. try:
  334. filenames = (wintypes.LPCWSTR * len(resources))(*resources)
  335. rc = register(session, len(resources), filenames, 0, None, 0, None)
  336. if rc:
  337. raise OSError(rc, "RmRegisterResources failed")
  338. # Pass 1 sizes (ERROR_MORE_DATA); the process set can change before the data pass, so retry the
  339. # bounded race with a re-sized buffer.
  340. needed, reasons = wintypes.UINT(), wintypes.DWORD()
  341. for _ in range(4):
  342. apps = (process_info * needed.value)()
  343. count = wintypes.UINT(needed.value)
  344. rc = get_list(
  345. session, ctypes.byref(needed), ctypes.byref(count),
  346. apps if needed.value else None, ctypes.byref(reasons),
  347. )
  348. if rc == 0:
  349. own_pid = os.getpid()
  350. return [
  351. (int(apps[index].process.pid), db_abspath)
  352. for index in range(count.value)
  353. if int(apps[index].process.pid) != own_pid
  354. ]
  355. if rc != _RM_ERROR_MORE_DATA:
  356. raise OSError(rc, "RmGetList failed")
  357. raise RuntimeError("Restart Manager holder set kept changing")
  358. finally:
  359. end(session)
  360. def foreign_state_db_holders(db_path: Path) -> list[tuple[int, str]]:
  361. """Return foreign holders of the DB or one of its WAL sidecars.
  362. A scan failure is represented as an unknown holder. Structural maintenance
  363. must not assume quiescence when an old, unlinked SQLite generation may
  364. still be open by another process.
  365. """
  366. if _IS_WINDOWS:
  367. try:
  368. return _windows_restart_manager_holders(db_path)
  369. except Exception as exc:
  370. logger.warning(
  371. "Could not prove state.db has no Windows holders; deferring structural maintenance: %s",
  372. exc,
  373. )
  374. return [(-1, f"Windows Restart Manager scan failed: {exc}")]
  375. # realpath, not abspath: psutil/libproc report the kernel-resolved pathname, so a symlinked
  376. # HERMES_HOME would otherwise make every holder invisible and let maintenance proceed.
  377. db_path_str = os.path.realpath(os.fspath(db_path))
  378. watched = {canonical_sqlite_path(candidate) for candidate in _sqlite_family(db_path_str)}
  379. holders: list[tuple[int, str]] = []
  380. watched_ids: set[tuple[int, int]] = set()
  381. db_dev: Optional[int] = None
  382. for candidate in _sqlite_family(db_path_str):
  383. try:
  384. stat_result = os.stat(candidate)
  385. except OSError as exc:
  386. if exc.errno not in (errno.ENOENT, errno.ESRCH):
  387. holders.append(
  388. (-1, f"watched-file stat failed: {candidate}: {exc}")
  389. )
  390. continue
  391. watched_ids.add((stat_result.st_dev, stat_result.st_ino))
  392. if candidate == db_path_str:
  393. db_dev = stat_result.st_dev
  394. if sys.platform.startswith("linux"):
  395. try:
  396. own_pid = os.getpid()
  397. for pid_str in os.listdir("/proc"):
  398. if not pid_str.isdigit():
  399. continue
  400. pid = int(pid_str)
  401. if pid == own_pid:
  402. continue
  403. fd_dir = f"/proc/{pid}/fd"
  404. try:
  405. fds = os.listdir(fd_dir)
  406. except OSError:
  407. argv = _read_proc_argv(pid)
  408. if (
  409. argv is not None
  410. and _looks_like_hermes(argv)
  411. and not _argv_scoped_to_other_home(argv, db_path)
  412. ):
  413. cmdline = " ".join(argv)
  414. holders.append((pid, f"uninspectable holder: {cmdline[:80]}"))
  415. continue
  416. for fd in fds:
  417. fd_path = f"{fd_dir}/{fd}"
  418. try:
  419. target = os.readlink(fd_path)
  420. except OSError as exc:
  421. if exc.errno in (errno.ENOENT, errno.ESRCH):
  422. continue
  423. argv = _read_proc_argv(pid)
  424. if (
  425. argv is not None
  426. and _looks_like_hermes(argv)
  427. and not _argv_scoped_to_other_home(argv, db_path)
  428. ):
  429. holders.append(
  430. (
  431. pid,
  432. f"uninspectable descriptor: {fd_path}: {exc}",
  433. )
  434. )
  435. continue
  436. target_is_watched = canonical_sqlite_path(target) in watched
  437. try:
  438. fd_stat = os.stat(fd_path)
  439. except OSError as exc:
  440. if exc.errno in (errno.ENOENT, errno.ESRCH):
  441. continue
  442. if target_is_watched:
  443. holders.append(
  444. (pid, f"uninspectable descriptor: {target}: {exc}")
  445. )
  446. else:
  447. argv = _read_proc_argv(pid)
  448. if (
  449. argv is not None
  450. and _looks_like_hermes(argv)
  451. and not _argv_scoped_to_other_home(argv, db_path)
  452. ):
  453. holders.append(
  454. (
  455. pid,
  456. "uninspectable descriptor: "
  457. f"{target}: {exc}",
  458. )
  459. )
  460. continue
  461. if (fd_stat.st_dev, fd_stat.st_ino) in watched_ids or (
  462. target_is_watched
  463. and target.endswith(" (deleted)")
  464. and db_dev is not None
  465. and fd_stat.st_dev == db_dev
  466. ):
  467. holders.append((pid, target))
  468. except Exception as exc:
  469. logger.warning(
  470. "Could not prove state.db has no foreign holders; "
  471. "deferring structural maintenance: %s",
  472. exc,
  473. )
  474. holders.append((-1, f"open-file scan failed: {exc}"))
  475. return holders
  476. if psutil is None:
  477. return [(-1, "open-file scan unavailable")]
  478. try:
  479. for process in psutil.process_iter(["pid", "open_files"]):
  480. info = process.info
  481. pid = int(info["pid"])
  482. if pid == os.getpid():
  483. continue
  484. for opened in info.get("open_files") or ():
  485. path = getattr(opened, "path", "")
  486. if path and canonical_sqlite_path(os.path.realpath(path)) in watched:
  487. holders.append((pid, path))
  488. except Exception as exc:
  489. logger.warning(
  490. "Could not prove state.db has no foreign holders; "
  491. "deferring structural maintenance: %s",
  492. exc,
  493. )
  494. holders.append((-1, f"open-file scan failed: {exc}"))
  495. return holders
  496. def in_process_state_db_holders(
  497. db_path: Path, *, exclude=None
  498. ) -> list[tuple[int, str]]:
  499. """Return holders of ``db_path`` inside THIS process, other than *exclude*.
  500. :func:`foreign_state_db_holders` skips ``os.getpid()`` by design, so it answers a
  501. cross-PROCESS question only. Consumers that read "no holders" as "the store is quiet"
  502. (auto-VACUUM admission) need this arm too: a VACUUM plus its TRUNCATE checkpoint retires
  503. the generation a sibling SessionDB in this very process still holds.
  504. """
  505. from hermes_state_registry import other_generations_for_path
  506. return [
  507. (os.getpid(), description)
  508. for description in other_generations_for_path(db_path, exclude=exclude)
  509. ]
  510. def held_store_refusal(db_path: Path, *, command: str, force_hint: Optional[str] = "--force") -> Optional[str]:
  511. """Operator-facing refusal for structural maintenance (VACUUM, index rebuild, bulk delete) while another
  512. process holds ``db_path`` or a WAL sidecar; ``None`` when the store is provably quiet.
  513. Running ``hermes sessions optimize-storage`` underneath a fleet of live gateways put every agent into
  514. the retired-WAL refusal until all writers were stopped (#110054). Same fail-closed scan doctor and
  515. repair use: an incomplete scan refuses too, it never reads as an all-clear.
  516. """
  517. holders = foreign_state_db_holders(db_path)
  518. if not holders:
  519. return None
  520. from hermes_constants import profile_cli_selector
  521. from hermes_state_errors import STORAGE_RECOVERY_DOCS_URL
  522. by_pid: dict[int, set[str]] = {}
  523. unknown: list[str] = []
  524. for pid, target in holders:
  525. if pid <= 0 or target.startswith("uninspectable"):
  526. unknown.append(target)
  527. else:
  528. by_pid.setdefault(pid, set()).add(Path(target.removesuffix(" (deleted)")).name)
  529. lines = [f"Refusing `hermes sessions {command}`: another process is using {db_path}."]
  530. lines += [f" {describe_holder_pid(pid)}: {', '.join(sorted(by_pid[pid]))}" for pid in sorted(by_pid)]
  531. if unknown:
  532. lines.append(f" cannot prove the database is quiet (holder scan incomplete: {unknown[0][:120]})")
  533. profile_arg = profile_cli_selector()
  534. lines += [
  535. "Rewriting the database under a live writer is how every agent ends up refusing turns with the "
  536. "retired state.db-wal error. Nothing is lost.",
  537. f"Stop them first (`hermes {profile_arg}gateway stop`, quit the Desktop app, pause cron), then re-run.",
  538. ]
  539. if force_hint:
  540. lines.append(f"Override with {force_hint} if you accept the risk.")
  541. lines.append(f"Recovery guide: {STORAGE_RECOVERY_DOCS_URL}")
  542. return "\n".join(lines)
  543. def live_writer_holds_db(
  544. db_path: Path,
  545. *,
  546. connect_repair_durable: Callable[..., sqlite3.Connection],
  547. ) -> bool:
  548. """Return whether repair lacks proven exclusive ownership of ``db_path``.
  549. ANY foreign process holding the DB or a sidecar is a live holder (#103339): the lock probe below
  550. cannot see a DELETE-mode reader (SHARED only) and cannot run at all on a malformed file, and those
  551. are exactly the states repair/VACUUM/checkpoint get invoked in. The holder scan is the authority and
  552. fails closed on its own failures (unknown/uninspectable sentinels); the probe only adds a positive
  553. lock signal on top."""
  554. if foreign_state_db_holders(db_path):
  555. return True
  556. probe = None
  557. try:
  558. probe = connect_repair_durable(db_path, timeout=0.0)
  559. probe.execute("PRAGMA locking_mode=EXCLUSIVE")
  560. probe.execute("BEGIN IMMEDIATE")
  561. probe.execute("ROLLBACK")
  562. return False
  563. except sqlite3.OperationalError as exc:
  564. return is_sqlite_lock_error(exc)
  565. except sqlite3.DatabaseError:
  566. # Malformed/unreadable with no holder on the scan: nobody else has it open, so repair may run.
  567. return False
  568. finally:
  569. if probe is not None:
  570. try:
  571. probe.execute("PRAGMA locking_mode=NORMAL")
  572. except Exception:
  573. pass
  574. try:
  575. probe.close()
  576. except Exception:
  577. pass