cli_snapshot.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172
  1. """Classic CLI snapshot restore: hand off the profile offline, then rejoin its authority."""
  2. from __future__ import annotations
  3. from contextlib import suppress
  4. import logging
  5. import os
  6. from hermes_cli.cli_commands_session_tools import _pr, _t
  7. from hermes_cli.gateway_client import GatewayClientError
  8. from hermes_cli.gateway_profile_lifecycle import ack_confirmed, ack_failure
  9. logger = logging.getLogger(__name__)
  10. def restore_snapshot_command(cli, parts):
  11. from hermes_cli.backup import list_quick_snapshots, quick_snapshot_dir, restore_quick_snapshot
  12. from hermes_cli.gateway_tools_notify import gateway_store
  13. if len(parts) < 3:
  14. print(f" {_t('snapshot.usage_restore')}")
  15. snaps = list_quick_snapshots(limit=1)
  16. if snaps:
  17. print(f" {_t('snapshot.most_recent', snapshot_id=snaps[0]['id'])}")
  18. return
  19. snap_id = parts[2]
  20. try:
  21. idx = int(snap_id)
  22. except ValueError:
  23. idx = None
  24. if idx is not None:
  25. snaps = list_quick_snapshots()
  26. if not 1 <= idx <= len(snaps):
  27. return print(f" {_t('snapshot.invalid_number', max=len(snaps))}")
  28. snap_id = snaps[idx - 1]["id"]
  29. store = gateway_store(cli)
  30. chat = store is not None and getattr(cli, "_gateway_session_open", False)
  31. # A remote gateway's state is not on this host: its chat restores this host's own snapshot offline,
  32. # as before, and keeps the remote connection open.
  33. gateway = chat and not os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip()
  34. try:
  35. # An invalid/missing target must not take a working gateway offline.
  36. if gateway and quick_snapshot_dir(snap_id) is not None:
  37. restored = _restore_gateway_snapshot(cli, store, snap_id)
  38. else:
  39. local_db = getattr(cli, "_session_db", None)
  40. if local_db is not None and not chat:
  41. with suppress(Exception):
  42. local_db.close()
  43. cli._session_db = None
  44. restored = restore_quick_snapshot(snap_id)
  45. except (GatewayClientError, OSError, ValueError) as exc: # ValueError: a corrupt manifest.json
  46. logger.warning("Snapshot restore failed: %s", exc, exc_info=True)
  47. return _pr(f" {_t('snapshot.restore_incomplete', snapshot_id=snap_id)}", f" {exc}")
  48. if restored:
  49. print(f" {_t('snapshot.restored', snapshot_id=snap_id)}")
  50. if not gateway:
  51. print(f" {_t('snapshot.restart_recommended')}")
  52. elif snap_id in {s.get("id") for s in list_quick_snapshots(limit=10**6)}:
  53. print(f" {_t('snapshot.restore_incomplete', snapshot_id=snap_id)}")
  54. else:
  55. print(f" {_t('snapshot.not_found', snapshot_id=snap_id)}")
  56. def _pause_owner(home, control_home, identity):
  57. """Hand the profile off; True when this restore parked a satellite and must serve it again."""
  58. from gateway.control_socket import pause_gateway_for_update, request_unserve_profile
  59. from hermes_cli.data_cleanup_holders import pause_acked
  60. from hermes_cli.profiles import parked_marker_path
  61. satellite = control_home != home
  62. ours = False
  63. if satellite:
  64. marker = parked_marker_path(home)
  65. # Persist intent BEFORE teardown: the periodic rescan must not re-serve it mid-restore.
  66. try:
  67. marker.touch(exist_ok=False)
  68. ours = True
  69. except FileExistsError:
  70. pass # A profile the user parked stays parked and unserved.
  71. accepted = False
  72. try:
  73. ack = request_unserve_profile(control_home, home.name)
  74. accepted = ack_confirmed(ack, "unserved", home.name) or (
  75. isinstance(ack, dict) and ack.get("pending") and not ack.get("error"))
  76. finally:
  77. if ours and not accepted:
  78. marker.unlink(missing_ok=True)
  79. else:
  80. accepted = pause_acked(pause_gateway_for_update(control_home, timeout=8), identity)
  81. if not accepted:
  82. # Stopping the host gateway would take down every profile it multiplexes.
  83. retry = (f"Park the profile with `hermes -p {home.name} gateway stop` and retry."
  84. if satellite else "Stop the gateway and retry.")
  85. raise GatewayClientError(f"Gateway refused the snapshot maintenance handoff; nothing was restored. {retry}")
  86. return ours
  87. def _restore_gateway_snapshot(cli, store, snap_id):
  88. from gateway.control_socket import request_serve_profile_hot
  89. from hermes_cli.backup import restore_quick_snapshot
  90. from hermes_cli.data_cleanup_holders import manual_gateway_identity
  91. from hermes_cli.gateway_connection import GatewayConnection
  92. from hermes_cli.gateway_runtime import control_home_for, discover_gateway_endpoint
  93. from hermes_cli.profiles import parked_marker_path
  94. from hermes_cli.profiles_remove import wait_profile_unowned
  95. from hermes_constants import get_hermes_home
  96. from hermes_state_registry import close_all_under
  97. home = get_hermes_home().resolve()
  98. observed = discover_gateway_endpoint(home)
  99. if observed.state != "ready" or observed.endpoint is None:
  100. raise GatewayClientError("Local gateway is not ready for snapshot maintenance. Reconnect and retry.")
  101. control_home = control_home_for(home, observed.endpoint).resolve()
  102. identity = None
  103. if control_home == home:
  104. # A supervised owner is relaunched right after the pause and takes the profile back.
  105. try:
  106. identity = manual_gateway_identity(home)
  107. except RuntimeError as exc:
  108. raise GatewayClientError(f"Nothing was restored: {exc}.") from exc
  109. store.close_reader()
  110. close_all_under(home)
  111. # A final viewer cannot redial from an idle poll while maintenance is replacing its database.
  112. store.connection.close()
  113. unserved = False
  114. try:
  115. unserved = _pause_owner(home, control_home, identity)
  116. if not wait_profile_unowned(home):
  117. raise GatewayClientError("Gateway still owns the profile; nothing was restored. "
  118. "Wait for its running work to stop and retry.")
  119. return restore_quick_snapshot(snap_id)
  120. finally:
  121. if unserved:
  122. parked_marker_path(home).unlink(missing_ok=True) # serve-profile refuses a parked profile
  123. ack = request_serve_profile_hot(control_home, home.name)
  124. if not ack_confirmed(ack, "served", home.name):
  125. logger.warning("Profile re-serve after snapshot restore: %s", ack)
  126. print(f" Profile '{home.name}' is not served again yet: {ack_failure(ack)}. "
  127. "The host retries on its next rescan (within 30s).")
  128. store.snapshots.clear()
  129. store.current = None
  130. # The restore's own result or error decides the reply; a failed rejoin must not replace it.
  131. try:
  132. drained, store.connection = store.connection, GatewayConnection.shared()
  133. # The relaunched owner revoked this CLI's launch-only key: every resume re-supplies it.
  134. store.connection.launch_api_key = drained.launch_api_key
  135. _rejoin_cli(cli, store)
  136. except Exception as exc:
  137. logger.warning("Rejoin after snapshot restore failed: %s", exc, exc_info=True)
  138. print(f" Reconnecting to the gateway failed: {exc}. Run /new or restart the CLI.")
  139. def _rejoin_cli(cli, store):
  140. from hermes_cli.cli_init_mixin import _ROUTE_FLAGS, _gateway_launch, lease_handed_over
  141. from hermes_cli.gateway_tools import bind_session
  142. # Same-id restores still replace the connection and invalidate the cached proxies.
  143. cli.agent = None
  144. for attr in ("_goal_manager", "_loop_manager", "_heartbeat_manager"):
  145. setattr(cli, attr, None)
  146. # A snapshot older than the gateway's import of a legacy row makes this attach import it again.
  147. with lease_handed_over(cli, getattr(cli, "_active_session_lease", None) is not None):
  148. try:
  149. snapshot = store.attach(cli.session_id)
  150. except GatewayClientError as exc:
  151. if str(exc) != "not_found":
  152. raise
  153. # The snapshot predates this session: only the owner may mint its replacement.
  154. launch = {key: value for key, value in _gateway_launch(cli).items() if key not in _ROUTE_FLAGS}
  155. store.create(launch, record_cwd=False)
  156. snapshot = store.current
  157. cli.session_id = snapshot["stored_session_id"]
  158. cli.conversation_history = [{key: value for key, value in message.items() if key != "row_id"}
  159. for message in snapshot.get("messages") or [] if not message.get("archived")]
  160. bind_session(cli, store.connection)
  161. getattr(cli, "_write_terminal_breadcrumb", lambda: None)()