hosted_room_work_storage.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247
  1. """Producer-scoped passive evidence storage and durable older-writer guards."""
  2. import json
  3. from gateway.hosted_rooms_common import table_exists
  4. INVALID_TABLE = "hosted_room_work_records_invalid"
  5. def initialize(conn):
  6. if not conn.in_transaction:
  7. # Reserve the writer before schema reads. A deferred SAVEPOINT can lose
  8. # its write upgrade to another publisher without honoring busy_timeout.
  9. conn.execute("BEGIN IMMEDIATE")
  10. with conn:
  11. _initialize_locked(conn)
  12. return
  13. # Nested initialization must never commit or roll back caller-owned work.
  14. conn.execute("SAVEPOINT work_record_initialize")
  15. try:
  16. _initialize_locked(conn)
  17. except BaseException:
  18. conn.execute("ROLLBACK TO work_record_initialize")
  19. conn.execute("RELEASE work_record_initialize")
  20. raise
  21. conn.execute("RELEASE work_record_initialize")
  22. def _initialize_locked(conn):
  23. from gateway import hosted_room_work_records as work
  24. tables = (work.SOURCE_TABLE, work.PENDING_TABLE)
  25. # Orphans are opaque evidence, not snapshots. No FK or producer can honestly
  26. # be supplied for them. Preserve every original column and forbid mutation.
  27. conn.execute(f"""CREATE TABLE IF NOT EXISTS {INVALID_TABLE} (
  28. evidence_id INTEGER PRIMARY KEY, source_table TEXT NOT NULL,
  29. room_id TEXT NOT NULL, revision INTEGER NOT NULL, digest TEXT NOT NULL, record_json TEXT NOT NULL,
  30. target_install_id TEXT, route_generation TEXT, status TEXT,
  31. disposition TEXT NOT NULL CHECK(disposition='invalid'))""")
  32. for table in tables:
  33. pending = table == work.PENDING_TABLE
  34. parent = "hosted_room_replicas" if table == work.TARGET_TABLE else "hosted_rooms"
  35. columns = {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")}
  36. legacy = bool(columns) and "producer_gateway_id" not in columns
  37. # Preserve surrounding cleanup triggers by recreating the same table name
  38. # inside this transaction, rather than renaming its referents.
  39. rows = conn.execute(f"SELECT * FROM {table}").fetchall() if legacy else []
  40. triggers = conn.execute("SELECT sql FROM sqlite_master WHERE type='trigger' AND tbl_name=?", (table,)).fetchall() if legacy else []
  41. if legacy:
  42. conn.execute(f"DROP TABLE {table}")
  43. extra = "target_install_id TEXT NOT NULL, route_generation TEXT NOT NULL, status TEXT NOT NULL," if pending else ""
  44. key = ",target_install_id" if pending else ""
  45. conn.execute(f"""CREATE TABLE IF NOT EXISTS {table} (
  46. room_id TEXT NOT NULL, revision INTEGER NOT NULL, digest TEXT NOT NULL, record_json TEXT NOT NULL,
  47. producer_gateway_id TEXT NOT NULL, producer_epoch INTEGER NOT NULL,
  48. disposition TEXT NOT NULL CHECK(disposition IN ('current','superseded_authority','historical','invalid')),
  49. {extra} PRIMARY KEY(room_id,producer_gateway_id,producer_epoch{key}),
  50. FOREIGN KEY(room_id) REFERENCES {parent}(room_id) ON DELETE CASCADE)""")
  51. for row in rows:
  52. if not conn.execute(f"SELECT 1 FROM {parent} WHERE room_id=?", (row["room_id"],)).fetchone():
  53. fields = ["source_table", "room_id", "revision", "digest", "record_json", "disposition"]
  54. values = [table, row["room_id"], row["revision"], row["digest"], row["record_json"], "invalid"]
  55. if pending:
  56. fields += ["target_install_id", "route_generation", "status"]
  57. values += [row[k] for k in fields[-3:]]
  58. conn.execute(f"INSERT INTO {INVALID_TABLE} ({','.join(fields)}) VALUES ({','.join('?' for _ in fields)})", values)
  59. continue
  60. producer, epoch, disposition = "", 0, "invalid"
  61. try:
  62. record = work.validate(json.loads(row["record_json"]))
  63. if (record["version"] != 1 or record["room_id"] != row["room_id"]
  64. or record["revision"] != row["revision"] or record["digest"] != row["digest"]):
  65. raise work.WorkRecordError("legacy work scope conflicts")
  66. producer, epoch, disposition = record["home_install_id"], 1, "current"
  67. except (ValueError, TypeError, KeyError):
  68. pass # Explicitly invalid, with original bytes and no invented producer.
  69. fields = ["room_id", "revision", "digest", "record_json", "producer_gateway_id", "producer_epoch", "disposition"]
  70. values = [row[k] for k in fields[:4]] + [producer, epoch, disposition]
  71. if pending:
  72. fields += ["target_install_id", "route_generation", "status"]
  73. values += [row[k] for k in fields[-3:]]
  74. values[6] = scope_disposition(conn, table, dict(zip(fields, values)))
  75. conn.execute(f"INSERT INTO {table} ({','.join(fields)}) VALUES ({','.join('?' for _ in fields)})", values)
  76. for trigger in triggers:
  77. conn.execute(trigger[0])
  78. _guards(conn, work, tables)
  79. _invalid_guards(conn)
  80. # Ordinary initialization owns schema/guards only. Each consuming operation
  81. # validates its own row; authority/enrollment triggers freeze exact-room old
  82. # scopes. Legacy dispositions were resolved at insertion above.
  83. def scope_disposition(conn, table, row):
  84. """Read-only authority projection, also used when inserting migrated rows."""
  85. from gateway import hosted_room_work_records as work
  86. if row["disposition"] != "current":
  87. return row["disposition"]
  88. current = None
  89. current = conn.execute("SELECT authority_gateway_id,authority_epoch FROM hosted_rooms WHERE room_id=?",
  90. (row["room_id"],)).fetchone()
  91. if current is None or tuple(current) == (row["producer_gateway_id"], row["producer_epoch"]):
  92. return "current"
  93. return "superseded_authority" if table == work.PENDING_TABLE and row["status"] != "acked" else "historical"
  94. def _guards(conn, work, tables):
  95. total, count = usage_sql((*tables, INVALID_TABLE))
  96. for table in tables:
  97. for operation in ("INSERT", "UPDATE"):
  98. key = "room_id=NEW.room_id AND producer_gateway_id=NEW.producer_gateway_id AND producer_epoch=NEW.producer_epoch"
  99. if table == work.PENDING_TABLE:
  100. key += " AND target_install_id=NEW.target_install_id"
  101. old_size = (f"COALESCE((SELECT {row_size_sql(table)} FROM {table} WHERE {key}),0)"
  102. if operation == "INSERT" else row_size_sql(table, "OLD."))
  103. new_size = row_size_sql(table, "NEW.")
  104. old_count = f"(SELECT COUNT(*) FROM {table} WHERE {key})" if operation == "INSERT" else "1"
  105. conn.execute(f"DROP TRIGGER IF EXISTS trg_{table}_budget_v2_{operation.lower()}")
  106. conn.execute(f"DROP TRIGGER IF EXISTS trg_{table}_budget_v3_{operation.lower()}")
  107. conn.execute(f"DROP TRIGGER IF EXISTS trg_{table}_budget_v4_{operation.lower()}")
  108. conn.execute(f"""CREATE TRIGGER trg_{table}_budget_v4_{operation.lower()}
  109. BEFORE {operation} ON {table} WHEN (
  110. (({new_size})>({old_size}) AND ({total})-({old_size})+({new_size})>{work.MAX_STORE_BYTES})
  111. OR ({old_count}=0 AND ({total}>{work.MAX_STORE_BYTES} OR {count}+1>{work.MAX_STORE_ROWS})))
  112. BEGIN SELECT RAISE(ABORT, 'work record storage is full'); END""")
  113. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_{table}_immutable_v2
  114. BEFORE UPDATE ON {table} WHEN
  115. NEW.room_id!=OLD.room_id OR NEW.producer_gateway_id!=OLD.producer_gateway_id
  116. OR NEW.producer_epoch!=OLD.producer_epoch
  117. OR (OLD.disposition!='current' AND (NEW.record_json!=OLD.record_json OR NEW.revision!=OLD.revision
  118. OR NEW.digest!=OLD.digest OR NEW.disposition!=OLD.disposition))
  119. BEGIN SELECT RAISE(ABORT, 'historical work record is immutable'); END""")
  120. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_{table}_delete_v2 BEFORE DELETE ON {table}
  121. WHEN OLD.disposition!='current' AND EXISTS (SELECT 1 FROM hosted_rooms
  122. WHERE room_id=OLD.room_id AND disbanded_at IS NULL)
  123. BEGIN SELECT RAISE(ABORT, 'historical work record is immutable'); END""")
  124. # INSERT OR REPLACE performs a delete internally. Refuse it before the
  125. # uniqueness conflict can remove an immutable snapshot.
  126. extra = " AND p.target_install_id=NEW.target_install_id" if table == work.PENDING_TABLE else ""
  127. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_{table}_replace_v2
  128. BEFORE INSERT ON {table} WHEN EXISTS (SELECT 1 FROM {table} p WHERE p.room_id=NEW.room_id
  129. AND p.producer_gateway_id=NEW.producer_gateway_id AND p.producer_epoch=NEW.producer_epoch
  130. {extra} AND p.disposition!='current')
  131. BEGIN SELECT RAISE(ABORT, 'historical work record is immutable'); END""")
  132. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_work_pending_outcome_v2 BEFORE UPDATE ON {work.PENDING_TABLE}
  133. WHEN OLD.disposition!='current' AND (NEW.status!=OLD.status OR NEW.route_generation!=OLD.route_generation
  134. OR NEW.target_install_id!=OLD.target_install_id)
  135. BEGIN SELECT RAISE(ABORT, 'historical delivery outcome is immutable'); END""")
  136. # A raw authority update by an older owner must also freeze old evidence.
  137. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_work_authority_transition_v2
  138. AFTER UPDATE OF authority_gateway_id,authority_epoch ON hosted_rooms BEGIN
  139. UPDATE {work.SOURCE_TABLE} SET disposition='historical' WHERE room_id=NEW.room_id AND disposition='current'
  140. AND (producer_gateway_id!=NEW.authority_gateway_id OR producer_epoch!=NEW.authority_epoch);
  141. UPDATE {work.PENDING_TABLE} SET disposition=CASE WHEN status='acked' THEN 'historical' ELSE 'superseded_authority' END
  142. WHERE room_id=NEW.room_id AND disposition='current'
  143. AND (producer_gateway_id!=NEW.authority_gateway_id OR producer_epoch!=NEW.authority_epoch);
  144. END""")
  145. def retained_fields(table):
  146. from gateway import hosted_room_work_records as work
  147. fields = ["room_id", "revision", "digest", "record_json"]
  148. fields += ["source_table"] if table == INVALID_TABLE else ["producer_gateway_id", "producer_epoch"]
  149. if table in (work.PENDING_TABLE, INVALID_TABLE):
  150. fields += ["target_install_id", "route_generation", "status"]
  151. return fields
  152. def row_size_sql(table, prefix=""):
  153. # Reserve the longest bounded disposition up front so freezing/cleanup can
  154. # still run at capacity. Every original variable column is charged as bytes.
  155. return "20+" + "+".join(f"COALESCE(length(CAST({prefix}{k} AS BLOB)),0)" for k in retained_fields(table))
  156. def usage_sql(tables):
  157. return (" + ".join(f"(SELECT COALESCE(SUM({row_size_sql(t)}),0) FROM {t})" for t in tables),
  158. " + ".join(f"(SELECT COUNT(*) FROM {t})" for t in tables))
  159. def validate_stored(row):
  160. from gateway import hosted_room_work_records as work
  161. try:
  162. record = work.validate(json.loads(row["record_json"]))
  163. if (row["disposition"] == "invalid"
  164. or scope(record) != (row["room_id"], row["producer_gateway_id"], row["producer_epoch"])
  165. or record["revision"] != row["revision"] or record["digest"] != row["digest"]):
  166. raise work.WorkRecordError("stored work metadata conflicts with record scope")
  167. except (ValueError, TypeError, KeyError, RecursionError) as exc:
  168. raise work.InvalidStoredWorkRecord("stored work evidence is invalid") from exc
  169. return record
  170. def validate_stored_locked(conn, table, row):
  171. """Validate the consumed row; invalidate only its unchanged current version.
  172. The savepoint owns a standalone classification or joins the caller's write
  173. transaction. A caller catching InvalidStoredWorkRecord must commit that
  174. transaction before reporting controlled failure. Historical rows are never
  175. rewritten, and read-only projections use validate_stored instead.
  176. """
  177. from gateway import hosted_room_work_records as work
  178. try:
  179. return validate_stored(row)
  180. except work.InvalidStoredWorkRecord:
  181. if row["disposition"] == "current":
  182. # Match all examined values, not merely a key that may since have
  183. # been replaced. IS preserves NULL/BLOB distinctions in old stores.
  184. fields = list(row.keys())
  185. where = " AND ".join(f'"{k}" IS ?' for k in fields)
  186. conn.execute("SAVEPOINT work_record_invalidate")
  187. try:
  188. conn.execute(f"UPDATE {table} SET disposition='invalid' WHERE {where}", [row[k] for k in fields])
  189. except BaseException:
  190. conn.execute("ROLLBACK TO work_record_invalidate")
  191. conn.execute("RELEASE work_record_invalidate")
  192. raise
  193. conn.execute("RELEASE work_record_invalidate")
  194. raise
  195. def scope(record):
  196. return record["room_id"], record["authority"]["gateway_id"], record["authority"]["epoch"]
  197. def save_locked(conn, table, record, *, target_install_id=None, route_generation=None):
  198. from gateway import hosted_room_work_records as work
  199. data = work.encode(record)
  200. fields = ["room_id", "producer_gateway_id", "producer_epoch", "revision", "digest", "record_json", "disposition"]
  201. values = [*scope(record), record["revision"], record["digest"], data, "current"]
  202. keys = fields[:3]
  203. if target_install_id is not None:
  204. fields += ["target_install_id", "route_generation", "status"]
  205. values += [target_install_id, route_generation, "pending"]
  206. keys += ["target_install_id"]
  207. work._budget(conn, table, dict(zip(fields, values)))
  208. updates = ','.join(f'{k}=excluded.{k}' for k in fields if k not in keys)
  209. conn.execute(f"INSERT INTO {table} ({','.join(fields)}) VALUES ({','.join('?' for _ in fields)}) "
  210. f"ON CONFLICT({','.join(keys)}) DO UPDATE SET {updates}", values)
  211. def _invalid_guards(conn):
  212. # There is no reviewed retirement grant in this runtime. Opaque evidence
  213. # cannot acquire a first-writer owner or be reclaimed by an ordinary writer.
  214. for operation in ("INSERT", "UPDATE", "DELETE"):
  215. conn.execute(f"""CREATE TRIGGER IF NOT EXISTS trg_work_invalid_{operation.lower()}
  216. BEFORE {operation} ON {INVALID_TABLE}
  217. BEGIN SELECT RAISE(ABORT, 'invalid work evidence is immutable'); END""")