hosted_room_replicas.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. """Replica store and takeover primitives for hosted Group Chat rooms.
  2. Non-authority gateways keep a durable local copy of the room log (``ingest_page()``: idempotent,
  3. gap- and epoch-regression-safe) plus fenced primitives to continue the room when the authority host
  4. dies: ``promote_replica()`` resumes locally at ``epoch + 1`` with a lineage-proving ``authority.claimed``
  5. event; ``demote_room()`` records ``authority.lost`` when a returning stale authority is shown a newer
  6. epoch. Storage primitives only: the caller decides *when* takeover is safe.
  7. """
  8. from __future__ import annotations
  9. import json
  10. import sqlite3
  11. from contextlib import closing, contextmanager
  12. from functools import partial
  13. from typing import Any, Iterator
  14. from gateway.hosted_rooms import (
  15. MAX_ACTOR_ID_CHARS, HostedRoomError, RoomConflictError, _actor_json, _connect, _payload_json, _room_id,
  16. _transaction, _validate_identifier, _validate_members, _validate_room_name, local_authority_gateway_id)
  17. from gateway.hosted_rooms_common import DbPath, bounded_int, clock, utf8_len
  18. MAX_REPLICA_ROOMS = 256
  19. MAX_REPLICA_EVENT_BYTES = 256 * 1024 * 1024
  20. _SYSTEM_ACTOR = {"kind": "system", "id": "authority-control"}
  21. _EVENT_COLUMNS = "(room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at)"
  22. _INSERT_ROOM_EVENT = f"INSERT INTO hosted_room_events {_EVENT_COLUMNS} VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
  23. _INSERT_REPLICA_EVENT = f"INSERT INTO hosted_room_replica_events {_EVENT_COLUMNS} VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
  24. _SELECT_REPLICA = "SELECT * FROM hosted_room_replicas WHERE room_id=?"
  25. class ReplicaError(HostedRoomError): """Base class for invalid or conflicting replica operations."""
  26. class ReplicaGapError(ReplicaError): """A page does not start at the replica's next expected sequence."""
  27. class ReplicaEpochRegressionError(ReplicaError): """A page or demotion carries an older authority epoch than stored."""
  28. def _initialize_replica_schema(conn: sqlite3.Connection) -> None:
  29. conn.execute("""CREATE TABLE IF NOT EXISTS hosted_room_replicas (
  30. room_id TEXT PRIMARY KEY, name TEXT NOT NULL, members_json TEXT NOT NULL,
  31. authority_gateway_id TEXT NOT NULL,
  32. authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 1),
  33. last_seq INTEGER NOT NULL DEFAULT 0 CHECK (last_seq >= 0),
  34. latest_seq INTEGER NOT NULL DEFAULT 0, event_bytes INTEGER NOT NULL DEFAULT 0,
  35. created_at REAL NOT NULL, updated_at REAL NOT NULL
  36. )""")
  37. conn.execute("""CREATE TABLE IF NOT EXISTS hosted_room_replica_events (
  38. room_id TEXT NOT NULL, seq INTEGER NOT NULL CHECK (seq >= 1), event_id TEXT NOT NULL,
  39. kind TEXT NOT NULL, actor_json TEXT NOT NULL, authority_epoch INTEGER,
  40. payload_json TEXT NOT NULL, created_at REAL NOT NULL,
  41. PRIMARY KEY (room_id, seq)
  42. )""")
  43. @contextmanager
  44. def _replica_transaction(db_path: DbPath) -> Iterator[sqlite3.Connection]:
  45. """Ensure the replica schema (own autocommit connection), then open an IMMEDIATE transaction. The DDL is
  46. deliberately re-run inside the transaction: that double init is the established statement order."""
  47. with closing(_connect(db_path)) as conn, conn:
  48. _initialize_replica_schema(conn)
  49. with _transaction(db_path, immediate=True) as conn:
  50. _initialize_replica_schema(conn)
  51. yield conn
  52. _positive_int = partial(bounded_int, error=ReplicaError, low=1)
  53. def _control_event(kind: str, epoch: int, payload: dict[str, Any]) -> tuple[str, str, str, str]:
  54. """(event_id, kind, actor_json, payload_json) of the system ``authority.<kind>`` control event for ``epoch``."""
  55. return f"system:authority-{kind}:{epoch}", f"authority.{kind}", _actor_json(_SYSTEM_ACTOR), _payload_json(payload)
  56. def _append_control_event(
  57. conn: sqlite3.Connection, room_id: str, seq: int, epoch: int, event: tuple[str, str, str, str], now: float
  58. ) -> None:
  59. event_id, kind, actor_json, payload_json = event
  60. conn.execute(_INSERT_ROOM_EVENT, (room_id, seq, event_id, kind, actor_json, epoch, payload_json, now))
  61. def _event_bytes(event: dict[str, Any]) -> int:
  62. return utf8_len(
  63. str(event["event_id"]), str(event["kind"]),
  64. json.dumps(event["actor"], ensure_ascii=False, separators=(",", ":")),
  65. json.dumps(event["payload"], ensure_ascii=False, separators=(",", ":")))
  66. def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]:
  67. if not isinstance(page, dict):
  68. raise ReplicaError("page must be an object")
  69. events, authority = page.get("events"), page.get("authority")
  70. if not isinstance(events, list):
  71. raise ReplicaError("page.events must be a list")
  72. if not isinstance(authority, dict):
  73. raise ReplicaError("page.authority is required for replication")
  74. gateway_id = _validate_identifier(
  75. authority.get("gateway_id"), label="page.authority.gateway_id", max_chars=MAX_ACTOR_ID_CHARS)
  76. epoch = _positive_int(authority.get("epoch"), message="page.authority.epoch must be a positive integer")
  77. previous_seq: int | None = None
  78. for event in events:
  79. if not isinstance(event, dict):
  80. raise ReplicaError("page events must be objects")
  81. seq = _positive_int(event.get("seq"), message="event.seq must be a positive integer")
  82. if previous_seq is not None and seq != previous_seq + 1:
  83. raise ReplicaGapError("page events must be contiguous")
  84. previous_seq = seq
  85. for field in ("event_id", "kind"):
  86. if not isinstance(event.get(field), str) or not event[field]:
  87. raise ReplicaError(f"event.{field} must be a non-empty string")
  88. if not isinstance(event.get("actor"), dict):
  89. raise ReplicaError("event.actor must be an object")
  90. if "payload" not in event:
  91. raise ReplicaError("event.payload is required")
  92. return events, {"gateway_id": gateway_id, "epoch": epoch}
  93. def _replica_row_state(conn: sqlite3.Connection, room_id: str) -> tuple[sqlite3.Row | None, int, int, int]:
  94. """Return (row, stored_epoch, last_seq, stored_bytes); a new room is admitted only under the room cap."""
  95. row = conn.execute("""SELECT authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes
  96. FROM hosted_room_replicas WHERE room_id=?""", (room_id,)).fetchone()
  97. if row is None:
  98. count = conn.execute("SELECT COUNT(*) FROM hosted_room_replicas").fetchone()[0]
  99. if int(count) >= MAX_REPLICA_ROOMS:
  100. raise ReplicaError("replica room capacity exhausted")
  101. return None, 0, 0, 0
  102. return row, int(row["authority_epoch"]), int(row["last_seq"]), int(row["event_bytes"])
  103. def _store_replica(
  104. conn: sqlite3.Connection, *, is_new: bool, room_id: str, room_name: str, members_json: str,
  105. authority: dict[str, Any], new_last: int, latest_seq: int, added_bytes: int, now: float) -> None:
  106. """INSERT the replica row for a new room, else UPDATE it (event_bytes accumulates)."""
  107. values = (room_name, members_json, authority["gateway_id"], authority["epoch"], new_last, max(latest_seq, new_last))
  108. if is_new:
  109. conn.execute("""INSERT INTO hosted_room_replicas (room_id, name, members_json,
  110. authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes,
  111. created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
  112. (room_id, *values, added_bytes, now, now))
  113. else:
  114. conn.execute("""UPDATE hosted_room_replicas SET name=?, members_json=?, authority_gateway_id=?,
  115. authority_epoch=?, last_seq=?, latest_seq=?, event_bytes=event_bytes+?,
  116. updated_at=? WHERE room_id=?""", (*values, added_bytes, now, room_id))
  117. def ingest_page(
  118. db_path: DbPath, *, room_id: Any, room_name: Any, members: Any, page: Any, now: float | None = None
  119. ) -> dict[str, Any]:
  120. """Persist one verbatim ``read_events()`` page idempotently; refuses seq gaps and epoch regressions."""
  121. room_id = _room_id(room_id)
  122. room_name = _validate_room_name(room_name)
  123. _, members_json = _validate_members(members)
  124. events, authority = _validate_page(page)
  125. now = clock(now)
  126. with _replica_transaction(db_path) as conn:
  127. row, stored_epoch, last_seq, stored_bytes = _replica_row_state(conn, room_id)
  128. if authority["epoch"] < stored_epoch:
  129. raise ReplicaEpochRegressionError("page authority epoch is older than the stored replica epoch")
  130. new_events = [e for e in events if int(e["seq"]) > last_seq]
  131. if new_events and int(new_events[0]["seq"]) != last_seq + 1:
  132. raise ReplicaGapError("page skips sequences the replica has not stored")
  133. added_bytes = 0
  134. for event in new_events:
  135. size = _event_bytes(event)
  136. if stored_bytes + added_bytes + size > MAX_REPLICA_EVENT_BYTES:
  137. raise ReplicaError("replica event storage exhausted")
  138. conn.execute(
  139. _INSERT_REPLICA_EVENT,
  140. (
  141. room_id, int(event["seq"]), event["event_id"], event["kind"], _actor_json(event["actor"]),
  142. event.get("authority_epoch"), _payload_json(event["payload"]),
  143. float(event.get("created_at") or now)))
  144. added_bytes += size
  145. new_last = int(new_events[-1]["seq"]) if new_events else last_seq
  146. latest_seq = page.get("latest_seq")
  147. if isinstance(latest_seq, bool) or not isinstance(latest_seq, int):
  148. latest_seq = new_last
  149. _store_replica(
  150. conn, is_new=row is None, room_id=room_id, room_name=room_name, members_json=members_json,
  151. authority=authority, new_last=new_last, latest_seq=latest_seq, added_bytes=added_bytes, now=now)
  152. return {
  153. "room_id": room_id, "stored_seq": new_last, "ingested": len(new_events), "authority": authority,
  154. "caught_up": new_last >= max(latest_seq, new_last)}
  155. def replica_state(db_path: DbPath, *, room_id: Any) -> dict[str, Any]:
  156. """Return the stored replica's coverage and authority lineage."""
  157. room_id = _room_id(room_id)
  158. with _replica_transaction(db_path) as conn:
  159. row = conn.execute(_SELECT_REPLICA, (room_id,)).fetchone()
  160. if row is None:
  161. raise ReplicaError("replica not found")
  162. return {
  163. "room_id": row["room_id"], "name": row["name"], "members": json.loads(row["members_json"]),
  164. "authority": {"gateway_id": row["authority_gateway_id"], "epoch": int(row["authority_epoch"])},
  165. "last_seq": int(row["last_seq"]), "latest_seq": int(row["latest_seq"]), "event_bytes": int(row["event_bytes"]),
  166. "created_at": float(row["created_at"]), "updated_at": float(row["updated_at"])}
  167. def promote_replica(
  168. db_path: DbPath, *, room_id: Any, reason: Any = "authority-unreachable", now: float | None = None
  169. ) -> dict[str, Any]:
  170. """Continue a replicated room on THIS gateway at ``epoch + 1``.
  171. Copies the replica log into the authoritative store and appends a lineage-proving ``authority.claimed``
  172. event, so wherever the claim replicates the old epoch is stale and every fenced primitive rejects it.
  173. The caller decides takeover is safe; this makes it atomic and provable.
  174. """
  175. room_id = _room_id(room_id)
  176. if not isinstance(reason, str) or not reason or len(reason) > 200:
  177. raise ReplicaError("reason must be a non-empty string of at most 200 chars")
  178. now = clock(now)
  179. local_gateway = local_authority_gateway_id()
  180. with _replica_transaction(db_path) as conn:
  181. replica = conn.execute(_SELECT_REPLICA, (room_id,)).fetchone()
  182. if replica is None:
  183. raise ReplicaError("replica not found")
  184. if replica["authority_gateway_id"] == local_gateway:
  185. raise ReplicaError("this gateway already holds the room authority")
  186. if conn.execute("SELECT 1 FROM hosted_rooms WHERE room_id=?", (room_id,)).fetchone():
  187. raise RoomConflictError("room_id already exists in the local authoritative store")
  188. if conn.execute("SELECT 1 FROM hosted_room_retired_ids WHERE room_id=?", (room_id,)).fetchone():
  189. raise RoomConflictError("room_id belongs to a disbanded room")
  190. previous_gateway, previous_epoch = str(replica["authority_gateway_id"]), int(replica["authority_epoch"])
  191. target_epoch, claim_seq = previous_epoch + 1, int(replica["last_seq"]) + 1
  192. claim = _control_event("claimed", target_epoch, {
  193. "previous_gateway_id": previous_gateway, "authority_gateway_id": local_gateway,
  194. "authority_epoch": target_epoch, "promoted_from_replica": True, "reason": reason})
  195. conn.execute("""INSERT INTO hosted_rooms
  196. (room_id, name, members_json, authority_gateway_id, authority_epoch, next_seq, event_bytes,
  197. revision, created_at, updated_at, disbanded_at)
  198. VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, NULL)""",
  199. (
  200. room_id, replica["name"], replica["members_json"], local_gateway, target_epoch, claim_seq + 1,
  201. int(replica["event_bytes"]) + utf8_len(*claim), now, now))
  202. conn.execute(
  203. f"""INSERT INTO hosted_room_events {_EVENT_COLUMNS}
  204. SELECT room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at
  205. FROM hosted_room_replica_events WHERE room_id=?""", (room_id,))
  206. _append_control_event(conn, room_id, claim_seq, target_epoch, claim, now)
  207. conn.execute("DELETE FROM hosted_room_replica_events WHERE room_id=?", (room_id,))
  208. conn.execute("DELETE FROM hosted_room_replicas WHERE room_id=?", (room_id,))
  209. return {
  210. "room_id": room_id, "authority_gateway_id": local_gateway, "authority_epoch": target_epoch,
  211. "previous_gateway_id": previous_gateway, "previous_epoch": previous_epoch, "claim_seq": claim_seq,
  212. "latest_seq": claim_seq}
  213. def demote_room(
  214. db_path: DbPath, *, room_id: Any, observed_gateway_id: Any, observed_epoch: Any, now: float | None = None
  215. ) -> dict[str, Any]:
  216. """Fence THIS gateway's stale room authority against a proven newer epoch.
  217. When a returning gateway observes (replicated ``authority.claimed`` or a transport rejection) that another
  218. gateway owns the room at a higher epoch, append ``authority.lost`` and adopt the observed lineage so no
  219. local send can commit at the stale epoch. Idempotent per lineage.
  220. """
  221. room_id = _room_id(room_id)
  222. observed_gateway_id = _validate_identifier(
  223. observed_gateway_id, label="observed_gateway_id", max_chars=MAX_ACTOR_ID_CHARS)
  224. observed_epoch = _positive_int(observed_epoch, message="observed_epoch must be a positive integer")
  225. now = clock(now)
  226. local_gateway = local_authority_gateway_id()
  227. with _transaction(db_path, immediate=True) as conn:
  228. row = conn.execute("""SELECT authority_gateway_id, authority_epoch, next_seq
  229. FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", (room_id,)).fetchone()
  230. if row is None:
  231. raise ReplicaError("room not found in the local authoritative store")
  232. current_gateway, current_epoch = str(row["authority_gateway_id"]), int(row["authority_epoch"])
  233. if current_gateway == observed_gateway_id and current_epoch == observed_epoch:
  234. return {
  235. "room_id": room_id, "authority_gateway_id": current_gateway, "authority_epoch": current_epoch,
  236. "idempotent": True}
  237. if observed_epoch <= current_epoch:
  238. raise ReplicaEpochRegressionError("observed epoch does not supersede the stored authority")
  239. if current_gateway != local_gateway:
  240. raise ReplicaError("room is not locally authoritative; nothing to demote")
  241. lost = _control_event("lost", observed_epoch, {
  242. "previous_gateway_id": current_gateway, "authority_gateway_id": observed_gateway_id,
  243. "authority_epoch": observed_epoch})
  244. _append_control_event(conn, room_id, int(row["next_seq"]), observed_epoch, lost, now)
  245. conn.execute("""UPDATE hosted_rooms
  246. SET authority_gateway_id=?, authority_epoch=?, next_seq=next_seq+1, revision=revision+1, updated_at=?
  247. WHERE room_id=?""",
  248. (observed_gateway_id, observed_epoch, now, room_id))
  249. return {
  250. "room_id": room_id, "authority_gateway_id": observed_gateway_id, "authority_epoch": observed_epoch,
  251. "idempotent": False}