hosted_rooms_legacy_import.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. """One-shot import of the pre-isolation ``hosted_room*`` rows into ``shared-state.db``.
  2. ``0e422e0ece`` repointed the room store from the root ``state.db`` to ``shared-state.db`` without
  3. moving the rows it already held, so an install that had rooms started with an empty coordination
  4. set and every pre-existing room resolved to "hosted room not found" (#109775). The first open after
  5. the upgrade copies the rows across once; the marker row keeps that a one-shot step, because a purge
  6. in THIS store must never be undone by re-importing rows the legacy store still holds.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import sqlite3
  11. from contextlib import closing
  12. from pathlib import Path
  13. from gateway.hosted_rooms_common import clock, table_columns, table_exists
  14. logger = logging.getLogger(__name__)
  15. SOURCE_NAME = "state.db"
  16. MARKER_TABLE = "hosted_room_legacy_imports"
  17. # Liveness state, never copied: a lease is a ~15s heartbeat plus a process generation, so a copied
  18. # lease names a process that is gone. The driver claims a fresh one instead.
  19. _SKIP_TABLES = frozenset({"hosted_room_driver_leases", "hosted_room_router_claims"})
  20. # Sources this process could not import (unreadable file, rows the target refused). Every store open
  21. # re-checks readiness, so without this a broken legacy file would re-run the copy and re-warn on
  22. # every poll; the retry happens on the next process start instead.
  23. _failed_sources: set[Path] = set()
  24. def source_path(db_path: Path) -> Path | None:
  25. """The pre-isolation store for ``db_path``, or ``None`` when this database has no predecessor.
  26. Only the shared coordination database has one: callers that pass any other path (older
  27. layouts, tests) own that file directly.
  28. """
  29. return db_path.with_name(SOURCE_NAME) if db_path.name == "shared-state.db" else None
  30. def settled(conn: sqlite3.Connection, db_path: Path) -> bool:
  31. """True once the import for this database has been recorded, given up on for this process, or never applies."""
  32. source = source_path(db_path)
  33. if source is None or source in _failed_sources:
  34. return True
  35. return table_exists(conn, MARKER_TABLE) and conn.execute(
  36. f"SELECT 1 FROM {MARKER_TABLE} WHERE source=?", (SOURCE_NAME,)).fetchone() is not None
  37. def _select_expressions(legacy: sqlite3.Connection, target: sqlite3.Connection, name: str) -> list[tuple[str, str]]:
  38. """``(target column, source expression)`` pairs for copying ``name``.
  39. Columns only the target has take the same default the in-place column migration applies, so a
  40. legacy layout from before the actor/authority columns imports instead of tripping NOT NULL.
  41. Columns only the source has are dropped; columns the target added without a default are left
  42. to its DDL default.
  43. """
  44. from gateway.hosted_rooms import _LEGACY_COLUMNS
  45. defaults = {(table, column): default for table, column, _, default in _LEGACY_COLUMNS if default is not None}
  46. source_columns = [str(row[1]) for row in legacy.execute(f"PRAGMA table_info({name})")]
  47. target_columns = table_columns(target, name)
  48. pairs = [(column, column) for column in source_columns if column in target_columns]
  49. pairs.extend((column, default) for (table, column), default in defaults.items()
  50. if table == name and column in target_columns and column not in source_columns)
  51. return pairs
  52. def _copy_rows(target: sqlite3.Connection, source: Path, *, only: frozenset[str] | None = None) -> int:
  53. """Copy every room ``target`` does not have yet, with all of its child rows; returns rooms copied.
  54. ``only``: copy just these room ids and no install-wide (room-less) table rows.
  55. A room id present in both stores is skipped as a unit: grafting the legacy events under this
  56. store's room would leave ``next_seq`` behind ``MAX(seq)`` and every later append colliding.
  57. Room-scoped tables therefore use a plain INSERT — a row the target refuses raises and aborts
  58. the whole import rather than being dropped in silence. A table this store has not created yet
  59. (the driver, policy and replica schemas are initialized by their own modules) is created from
  60. the source's own DDL so its rows survive the upgrade too.
  61. """
  62. from gateway.hosted_rooms import _EVENT_BYTES_BACKFILL
  63. copied_rooms: list[str] = []
  64. existing = {str(row[0]) for row in target.execute("SELECT room_id FROM hosted_rooms")}
  65. with closing(sqlite3.connect(f"file:{source}?mode=ro", uri=True, timeout=10)) as legacy:
  66. names = [str(row[0]) for row in legacy.execute(
  67. "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'hosted_room*'")]
  68. # Parents first: hosted_room_events carries a foreign key into hosted_rooms.
  69. for name in sorted(names, key=lambda name: (name != "hosted_rooms", name)):
  70. if name in _SKIP_TABLES or name == MARKER_TABLE or name.endswith(("_next", "_migrating")):
  71. continue
  72. if not table_exists(target, name):
  73. target.execute(str(legacy.execute(
  74. "SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone()[0]))
  75. pairs = _select_expressions(legacy, target, name)
  76. if not pairs:
  77. continue
  78. columns = [column for column, _ in pairs]
  79. rows = legacy.execute(f"SELECT {', '.join(expr for _, expr in pairs)} FROM {name}")
  80. if "room_id" in columns:
  81. room_index = columns.index("room_id")
  82. rows = (row for row in rows if str(row[room_index]) not in existing
  83. and (only is None or str(row[room_index]) in only))
  84. verb = "INSERT"
  85. elif only is not None:
  86. continue
  87. else:
  88. verb = "INSERT OR IGNORE"
  89. if name == "hosted_rooms":
  90. rows = list(rows)
  91. copied_rooms = [str(row[room_index]) for row in rows]
  92. target.executemany(
  93. f"{verb} INTO {name} ({', '.join(columns)}) VALUES ({', '.join('?' * len(columns))})", rows)
  94. if copied_rooms and "event_bytes" not in table_columns(legacy, "hosted_rooms"):
  95. target.execute(
  96. _EVENT_BYTES_BACKFILL.format(where=f"room_id IN ({', '.join('?' * len(copied_rooms))})"), copied_rooms)
  97. return len(copied_rooms)
  98. def import_legacy_rooms(conn: sqlite3.Connection, db_path: Path) -> None:
  99. """Copy the pre-isolation rows in once, then record the marker; never fails the open.
  100. The copy runs inside the caller's schema transaction under its own savepoint, so a crash or a
  101. refused row leaves either both the copied rows and the marker or neither.
  102. """
  103. source = source_path(db_path)
  104. if source is None or settled(conn, db_path):
  105. return
  106. conn.execute("SAVEPOINT legacy_import")
  107. try:
  108. copied = _copy_rows(conn, source) if source.is_file() else 0
  109. except (OSError, sqlite3.Error) as exc:
  110. # A locked, unreadable or incompatible legacy store must not take hosted rooms down with
  111. # it: drop the partial copy, leave the marker unset, retry on the next process start.
  112. conn.execute("ROLLBACK TO legacy_import")
  113. conn.execute("RELEASE legacy_import")
  114. _failed_sources.add(source)
  115. logger.warning("hosted rooms: could not import the pre-isolation %s (%s); will retry on the next start",
  116. source, exc)
  117. return
  118. conn.execute(
  119. f"CREATE TABLE IF NOT EXISTS {MARKER_TABLE} (source TEXT PRIMARY KEY, imported_at REAL NOT NULL, rooms INTEGER NOT NULL)")
  120. conn.execute(
  121. f"INSERT OR IGNORE INTO {MARKER_TABLE} (source, imported_at, rooms) VALUES (?, ?, ?)",
  122. (SOURCE_NAME, clock(None), copied))
  123. conn.execute("RELEASE legacy_import")
  124. if copied:
  125. logger.info("hosted rooms: imported %d pre-isolation Group Chat(s) from %s", copied, source)