data_cleanup_holders.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. """Home-scoped runtime coordination for an explicitly confirmed data wipe."""
  2. from __future__ import annotations
  3. from contextlib import ExitStack, contextmanager
  4. from pathlib import Path
  5. from agent.deadline import poll_until
  6. _OWNER_STOP_HINTS = {
  7. "systemd": "stop its systemd unit with systemctl before retrying",
  8. "launchd": "stop its LaunchAgent with launchctl before retrying",
  9. "desktop": "quit the owning desktop app before retrying",
  10. "external": "stop the gateway through its external supervisor before retrying",
  11. }
  12. def manual_gateway_identity(home: Path) -> dict | None:
  13. """The ``identify`` answer of the manually launched gateway owning ``home``, None when none runs.
  14. Raises RuntimeError for a gateway this client may not drain: a supervisor (systemd, launchd,
  15. Desktop, ...) relaunches it right after a pause, so it would take the profile back.
  16. """
  17. from gateway.control_socket import identify_gateway, launch_supervisor
  18. from gateway.status import get_running_pid_identity_strict
  19. identity = identify_gateway(home)
  20. if identity is None:
  21. if get_running_pid_identity_strict(home / "gateway.pid") is not None:
  22. raise RuntimeError(f"gateway has no usable control socket for {home}; stop it through its owner and retry")
  23. return None
  24. if not identity.get("hermes_home") or Path(identity["hermes_home"]).resolve() != home:
  25. raise RuntimeError(f"gateway control identity does not match {home}")
  26. owner = launch_supervisor(identity)
  27. if owner != "manual":
  28. hint = _OWNER_STOP_HINTS.get(owner or "", "stop it through its supervisor before retrying")
  29. raise RuntimeError(f"{owner or 'unknown'} gateway still owns {home}; {hint}")
  30. return identity
  31. def pause_acked(response: dict | None, identity: dict | None) -> bool:
  32. """True when the identified gateway itself accepted (or is already running) the drain."""
  33. return bool(response and identity is not None and response.get("pid") == identity.get("pid")
  34. and (response.get("pausing") or response.get("already_stopping")))
  35. def _drain_manual_gateway(home: Path) -> None:
  36. from gateway.control_socket import identify_gateway, pause_gateway_for_update
  37. from gateway.status import get_process_start_time, get_running_pid_identity_strict, _pid_exists
  38. identity = manual_gateway_identity(home)
  39. if identity is None:
  40. return
  41. if identity.get("served_profiles"):
  42. raise RuntimeError("gateway serves multiple profiles; stop it explicitly before deleting one profile's data")
  43. pid = identity.get("pid")
  44. start = identity.get("start_time")
  45. if type(pid) is not int or pid <= 0 or start is None or get_process_start_time(pid) != start:
  46. raise RuntimeError(f"gateway process identity is unverified for {home}")
  47. response = pause_gateway_for_update(home)
  48. if not pause_acked(response, identity):
  49. raise RuntimeError(f"gateway refused to drain for {home}")
  50. def _gone() -> bool:
  51. current = get_process_start_time(pid)
  52. original_gone = not _pid_exists(pid) or (current is not None and current != start)
  53. return original_gone and identify_gateway(home) is None and get_running_pid_identity_strict(home / "gateway.pid") is None
  54. if not poll_until(_gone, min(120.0, max(5.0, float(response.get("drain_timeout", 30.0)) + 5.0)), 0.05):
  55. raise RuntimeError(f"gateway did not exit for {home}; no data removed")
  56. def _refuse_backend_writers(home: Path) -> None:
  57. from hermes_constants import get_default_hermes_root
  58. from hermes_cli.process_identity import LEDGER_FILENAME, _read_ledger, entry_still_running
  59. root = get_default_hermes_root(home=home).resolve()
  60. rows = _read_ledger(root / LEDGER_FILENAME)
  61. if rows is None:
  62. raise RuntimeError(f"cannot read backend ownership: {root / LEDGER_FILENAME}")
  63. for row in rows:
  64. if row.get("purpose") not in {"serve", "dashboard"}:
  65. continue
  66. # The recorded profile is the initial UI selection, not the backend's
  67. # write scope: it can serve any profile under this machine root.
  68. pid = row.get("pid")
  69. if type(pid) is not int or pid <= 0:
  70. raise RuntimeError("backend ownership contains an invalid PID")
  71. if entry_still_running(row):
  72. raise RuntimeError(f"backend PID {pid} still owns {home}; close its desktop/dashboard before retrying")
  73. def _refuse_cron_writers(home: Path) -> None:
  74. from contextlib import closing
  75. import sqlite3
  76. from gateway.status import get_process_start_time, _pid_exists
  77. path = home / "cron" / "executions.db"
  78. try:
  79. path.stat()
  80. except FileNotFoundError:
  81. return
  82. # Importing cron initializes scheduler/config defaults. A deletion probe
  83. # must only read the existing ledger, not recreate the data being removed.
  84. try:
  85. with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.25)) as conn:
  86. conn.row_factory = sqlite3.Row
  87. rows = conn.execute("SELECT * FROM executions WHERE status IN ('claimed','running')").fetchall()
  88. except sqlite3.Error as exc:
  89. raise RuntimeError(f"cannot inspect cron owners in {path}: {exc}") from exc
  90. for raw in rows:
  91. row = dict(raw)
  92. pid = row["pid"]
  93. if row.get("handoff_pending"):
  94. raise RuntimeError(f"cron execution {row['id']} is being handed to a worker; wait for it to finish")
  95. if not _pid_exists(pid):
  96. continue
  97. current = get_process_start_time(pid)
  98. recorded = row.get("process_started_at")
  99. if current is None or recorded is None or current == recorded:
  100. raise RuntimeError(f"cron execution {row['id']} still owns {home} (PID {pid}); stop it before retrying")
  101. def _refuse_multiplexer(home: Path) -> None:
  102. from hermes_constants import get_default_hermes_root
  103. from gateway.control_socket import identify_gateway
  104. from gateway.status import get_running_pid_identity_strict
  105. root = get_default_hermes_root(home=home).resolve()
  106. if home == root:
  107. return
  108. identity = identify_gateway(root)
  109. if identity is not None:
  110. if home.name in identity.get("served_profiles", []):
  111. raise RuntimeError(f"the default gateway serves {home}; stop the multiplexer explicitly before retrying")
  112. elif get_running_pid_identity_strict(root / "gateway.pid") is not None:
  113. raise RuntimeError("the default gateway's profile scope is unverified; stop it explicitly before retrying")
  114. @contextmanager
  115. def quiescent_home(home: Path):
  116. """Hold existing writer locks through deletion; do not remove their inodes."""
  117. from gateway.status import _release_file_lock, _try_acquire_file_lock
  118. from hermes_cli.active_sessions import _FileLock, _prune_dead, _read_entries
  119. from hermes_cli.runtime_state import _lock
  120. from tools.checkpoint_pruning import store_lock
  121. home = home.resolve()
  122. _refuse_backend_writers(home)
  123. _refuse_multiplexer(home)
  124. _drain_manual_gateway(home)
  125. with ExitStack() as stack:
  126. stack.enter_context(store_lock(home / "checkpoints"))
  127. gateway_lock = stack.enter_context((home / "gateway.lock").open("a+", encoding="utf-8"))
  128. if not _try_acquire_file_lock(gateway_lock):
  129. raise RuntimeError(f"gateway is still active for {home}")
  130. stack.callback(_release_file_lock, gateway_lock)
  131. stack.enter_context(_FileLock(home / "runtime" / "active_sessions.lock"))
  132. sessions = _prune_dead(_read_entries(home / "runtime" / "active_sessions.json", strict=True), strict=True)
  133. if sessions:
  134. pids = sorted({entry["pid"] for entry in sessions})
  135. raise RuntimeError(f"live chat sessions still own {home} (PIDs {pids}); close them before retrying")
  136. for path in (home / ".backup.lock", home / "cron" / ".tick.lock"):
  137. path.parent.mkdir(parents=True, exist_ok=True)
  138. handle = stack.enter_context(path.open("a+b"))
  139. if not _lock(handle.fileno(), wait=False):
  140. raise RuntimeError(f"an active writer still owns {path}")
  141. _refuse_cron_writers(home)
  142. yield