hosted_rooms_gateway_import.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184
  1. """Hand the standalone backend's Group Chats to the gateway that owns the profile (WA-19).
  2. A native Desktop on a standalone ``hermes serve``/``hermes dashboard`` (no session authority) keeps
  3. its rooms in the root ``shared-state.db`` and drives them in that process (``tui_gateway``
  4. route), while the gateway keeps its own in the profile's ``state.db`` (owner claim and epoch fence
  5. in the room insert's transaction). Once a gateway serves the profile, the standalone backend
  6. forwards ``groups.*`` to it (``tui_gateway/methods_groups_forward.py``); this module moves the rooms
  7. that backend already hosts into the gateway's store so the one list shows them.
  8. Each room moves in two crash-safe steps, so it is driven by exactly one process:
  9. 1. **Fence the source.** The room's authority in ``shared-state.db`` moves to
  10. :func:`handoff_gateway_id` at ``epoch + 1`` (``authority.lost``, the RoomLink demotion). The
  11. standalone driver binds only rooms of its own authority and every room write checks it, so that
  12. copy is frozen from here on. The source rows are never deleted.
  13. 2. **Copy and claim.** A fenced room the gateway store does not hold is copied with its child rows,
  14. then claimed for this install at ``epoch + 1`` (``authority.claimed``) and owned by the host
  15. owner, the subject native Desktop and a signed-in Webapp tab both act as.
  16. Re-running is a no-op: step 1 only takes rooms still on the local authority, step 2 only copies
  17. fenced rooms the target has never known (neither held nor reserved in ``hosted_room_retired_ids``),
  18. so a room the user disbanded in the gateway never comes back once its tombstone is pruned. After
  19. the copy commits, each handed room is recorded in the source's :data:`HANDOFF_TABLE`, which step 2
  20. never reads again. A crash between the steps is finished by the next start.
  21. ``shared-state.db`` is one root file with no profile column, so only the root home's gateway (the
  22. default profile, whose ``state.db`` sits beside it) imports. A standalone room that was not imported
  23. stays listed and driven by the standalone backend from every profile's view, and a named profile's
  24. view reaches the imported ones on the root gateway (``tui_gateway/methods_groups_forward.py``), so
  25. no view loses a room it showed before.
  26. The import runs when the gateway builds its room service (before ``ready``) and once more right
  27. after ``ready`` (:func:`import_after_ready`), covering a room written in between.
  28. """
  29. from __future__ import annotations
  30. import logging
  31. from pathlib import Path
  32. logger = logging.getLogger(__name__)
  33. SOURCE_NAME = "shared-state.db"
  34. #: Source-side record of every room the gateway store took over: never offered again.
  35. HANDOFF_TABLE = "hosted_room_gateway_handoffs"
  36. _OWNER = "gateway.hosted.owner.v1:"
  37. def handoff_gateway_id() -> str:
  38. """The authority a standalone room is fenced to once the gateway store holds it."""
  39. from gateway.hosted_rooms import local_authority_gateway_id
  40. return f"{local_authority_gateway_id()}:gateway-store"
  41. def source_for(target_db: Path) -> Path | None:
  42. """The standalone store this gateway store imports from, or ``None`` (named profile, tests)."""
  43. target_db = Path(target_db)
  44. source = target_db.with_name(SOURCE_NAME)
  45. return source if target_db.name == "state.db" and source.is_file() else None
  46. def _fence_source(source: Path, local_id: str, handoff_id: str) -> int:
  47. from gateway.hosted_room_replicas import demote_room
  48. from gateway.hosted_rooms import list_rooms, MAX_ROOM_LIST_LIMIT
  49. fenced, offset = 0, 0
  50. while True:
  51. page = list_rooms(source, limit=MAX_ROOM_LIST_LIMIT, offset=offset)
  52. for room in page:
  53. if room["authority_gateway_id"] == local_id:
  54. demote_room(source, room_id=room["room_id"], observed_gateway_id=handoff_id,
  55. observed_epoch=int(room["authority_epoch"]) + 1)
  56. fenced += 1
  57. if len(page) < MAX_ROOM_LIST_LIMIT:
  58. return fenced
  59. offset += len(page)
  60. def _fenced_ids(source: Path, handoff_id: str) -> set[str]:
  61. """Fenced live rooms not yet recorded as handed over."""
  62. from gateway.hosted_rooms import _transaction
  63. with _transaction(source, immediate=True) as conn:
  64. _ensure_handoff_table(conn)
  65. return {str(row[0]) for row in conn.execute(
  66. f"SELECT room_id FROM hosted_rooms WHERE authority_gateway_id=? AND disbanded_at IS NULL"
  67. f" AND room_id NOT IN (SELECT room_id FROM {HANDOFF_TABLE})", (handoff_id,))}
  68. def _ensure_handoff_table(conn) -> None:
  69. conn.execute(f"CREATE TABLE IF NOT EXISTS {HANDOFF_TABLE} (room_id TEXT PRIMARY KEY, handed_at REAL NOT NULL)")
  70. def _record_handoff(source: Path, room_ids: set[str], now: float) -> None:
  71. from gateway.hosted_rooms import _transaction
  72. with _transaction(source, immediate=True) as conn:
  73. _ensure_handoff_table(conn)
  74. conn.executemany(f"INSERT OR IGNORE INTO {HANDOFF_TABLE} (room_id, handed_at) VALUES (?, ?)",
  75. [(room_id, now) for room_id in sorted(room_ids)])
  76. def handed_room_ids(source: Path) -> set[str]:
  77. """Every room ``source`` handed to the gateway store (recorded or still only fenced)."""
  78. from contextlib import closing
  79. from gateway.hosted_rooms import _read_connection
  80. with closing(_read_connection(source)) as conn:
  81. return {str(row[0]) for row in conn.execute(
  82. "SELECT room_id FROM hosted_rooms WHERE authority_gateway_id=?", (handoff_gateway_id(),))}
  83. def _claim(conn, room_id: str, local_id: str, handoff_id: str, owner: str, now: float) -> None:
  84. from gateway.hosted_room_replicas import _append_control_event, _control_event
  85. row = conn.execute("SELECT authority_epoch, next_seq FROM hosted_rooms WHERE room_id=?", (room_id,)).fetchone()
  86. epoch = int(row["authority_epoch"]) + 1
  87. _append_control_event(conn, room_id, int(row["next_seq"]), epoch, _control_event("claimed", epoch, {
  88. "previous_gateway_id": handoff_id, "authority_gateway_id": local_id, "authority_epoch": epoch}), now)
  89. conn.execute("""UPDATE hosted_rooms SET authority_gateway_id=?, authority_epoch=?, next_seq=next_seq+1,
  90. revision=revision+1, updated_at=? WHERE room_id=?""", (local_id, epoch, now, room_id))
  91. conn.execute("INSERT OR IGNORE INTO state_meta(key, value) VALUES (?, ?)", (_OWNER + room_id, owner))
  92. def import_standalone_rooms(target_db: Path, *, runtime_epoch: int | None = None) -> int:
  93. """Move the standalone backend's live rooms into ``target_db`` once; returns rooms copied.
  94. ``runtime_epoch``: the authority's writer epoch, checked inside the copy's transaction so a
  95. superseded gateway process never writes the store. Never raises: a locked or unreadable source
  96. leaves both stores as they were and is retried on the next start.
  97. """
  98. import sqlite3
  99. from gateway.hosted_rooms import HostedRoomError, _transaction, local_authority_gateway_id
  100. from gateway.hosted_rooms_common import clock
  101. from gateway.hosted_rooms_legacy_import import _copy_rows
  102. from gateway.owner_identity import host_owner_subject
  103. from hermes_state_runtime import RuntimeStoreError, _epoch
  104. source = source_for(target_db)
  105. if source is None:
  106. return 0
  107. try:
  108. local_id, handoff_id = local_authority_gateway_id(), handoff_gateway_id()
  109. _fence_source(source, local_id, handoff_id)
  110. fenced = _fenced_ids(source, handoff_id)
  111. if not fenced:
  112. return 0
  113. owner, now = host_owner_subject(), clock(None)
  114. with _transaction(target_db, immediate=True) as conn:
  115. if runtime_epoch is not None:
  116. _epoch(conn, runtime_epoch)
  117. # Every id this store has ever known: a disbanded room's row is pruned later, its id stays.
  118. known = {str(row[0]) for row in conn.execute(
  119. "SELECT room_id FROM hosted_rooms UNION SELECT room_id FROM hosted_room_retired_ids")}
  120. missing = frozenset(fenced - known)
  121. copied = _copy_rows(conn, source, only=missing) if missing else 0
  122. for room_id in sorted(missing):
  123. _claim(conn, room_id, local_id, handoff_id, owner, now)
  124. _record_handoff(source, fenced, now)
  125. except (OSError, sqlite3.Error, HostedRoomError, RuntimeStoreError) as exc:
  126. logger.warning("hosted rooms: could not import the standalone Group Chats from %s (%s); "
  127. "will retry on the next start", source, exc)
  128. return 0
  129. if copied:
  130. logger.info("hosted rooms: imported %d standalone Group Chat(s) from %s", copied, source)
  131. return copied
  132. def import_after_ready(authorities) -> None:
  133. """Import once more after the gateway published ``ready``, off the event loop.
  134. The start-up import runs before readiness; a standalone backend that saw no gateway yet may
  135. still write a room in between. Once ready, that backend forwards every call (or waits while
  136. the gateway is starting), so this second pass leaves nothing stranded until the next restart.
  137. The service's driver re-reads its bindings each tick and is woken for the claimed rooms."""
  138. import threading
  139. for authority in authorities:
  140. service = getattr(authority, "hosted_room_service", None)
  141. if service is None or getattr(service, "_ready_import_started", False):
  142. continue
  143. service._ready_import_started = True
  144. target, epoch = Path(authority.db.db_path), authority.epoch
  145. def run(target=target, epoch=epoch, service=service, authority=authority):
  146. from gateway.session_authorities import owner_scope
  147. with owner_scope(authority): # a new thread holds the launch profile's scope otherwise
  148. if import_standalone_rooms(target, runtime_epoch=epoch):
  149. service.wakeup()
  150. threading.Thread(target=run, name="hosted-rooms-ready-import", daemon=True).start()