hosted_room_attachments.py 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364
  1. """Durable, private attachment storage for gateway-owned Group Chats.
  2. The room log carries only server-minted attachment ids and bounded metadata.
  3. Canonical bytes live in a private directory beside ``state.db`` and are shared
  4. by SHA-256 through an internal blob table. Upload ids make client retries
  5. idempotent; send-time commitment binds each upload to one room event and the
  6. room's frozen recipient roster.
  7. """
  8. from __future__ import annotations
  9. from gateway.hosted_rooms import EventAttachmentConflictError
  10. from hermes_dir_acl import make_user_dir
  11. import base64
  12. import binascii
  13. import hashlib
  14. import json
  15. import os
  16. import re
  17. import secrets
  18. import sqlite3
  19. import stat
  20. import threading
  21. import time
  22. import unicodedata
  23. from collections import Counter
  24. from contextlib import contextmanager
  25. from dataclasses import dataclass
  26. from pathlib import Path
  27. from typing import Any, Callable, Iterator, Mapping, Sequence
  28. MAX_ATTACHMENTS_PER_MESSAGE = 8
  29. MAX_ATTACHMENT_BYTES = 15_000_000
  30. MAX_MESSAGE_ATTACHMENT_BYTES = 25_000_000
  31. MAX_ROOM_ATTACHMENT_BYTES = 512 * 1024 * 1024
  32. MAX_GATEWAY_BLOB_BYTES = 2 * 1024 * 1024 * 1024
  33. MAX_ROOM_ATTACHMENT_COUNT = 50_000
  34. MAX_GATEWAY_ATTACHMENT_COUNT = 200_000
  35. MAX_ROOM_UNCOMMITTED_BYTES = 100 * 1024 * 1024
  36. MAX_ROOM_UNCOMMITTED_COUNT = 64
  37. MAX_TASK_ATTACHMENTS = 16
  38. MAX_TASK_ATTACHMENT_BYTES = 50_000_000
  39. UNCOMMITTED_TTL_SECONDS = 60 * 60
  40. DISBANDED_GRACE_SECONDS = 15 * 60
  41. CLASSIC_ATTACHMENT_TTL_SECONDS = 7 * 24 * 60 * 60
  42. _SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
  43. MAX_ATTACHMENT_NAME_CHARS = 255
  44. MAX_ATTACHMENT_MIME_CHARS = 127
  45. MAX_IDENTIFIER_CHARS = 128
  46. _ATTACHMENT_FIELDS = frozenset({"attachment_id", "kind", "name", "size", "mime"})
  47. _ATTACHMENT_KINDS = frozenset({"image", "pdf", "file"})
  48. _ATTACHMENT_ID_RE = re.compile(r"^att_[0-9a-f]{32}$")
  49. _BLOB_ID_RE = re.compile(r"^blob_[0-9a-f]{32}$")
  50. _IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
  51. _MIME_RE = re.compile(
  52. r"^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$"
  53. )
  54. class AttachmentError(ValueError):
  55. """Base class for hosted-room attachment contract failures."""
  56. class AttachmentNotFoundError(AttachmentError):
  57. """The attachment id is unknown, expired, or unavailable to this caller."""
  58. class AttachmentConflictError(AttachmentError):
  59. """An idempotency or ownership key was reused with different content."""
  60. class AttachmentQuotaError(AttachmentError):
  61. """A bounded room or gateway quota would be exceeded."""
  62. class AttachmentIntegrityError(AttachmentError):
  63. """Canonical blob bytes no longer match their durable metadata."""
  64. @dataclass(frozen=True)
  65. class AttachmentData:
  66. """Verified canonical bytes plus safe attachment metadata."""
  67. attachment: dict[str, Any]
  68. data: bytes
  69. def default_attachment_root(db_path: Path | str) -> Path:
  70. """Return the private byte-store root beside the room state database."""
  71. return Path(db_path).parent / "hosted-room-attachments"
  72. def _identifier(value: Any, *, label: str) -> str:
  73. if not isinstance(value, str):
  74. raise AttachmentError(f"{label} must be a string")
  75. normalized = value.strip()
  76. if (
  77. not normalized
  78. or len(normalized) > MAX_IDENTIFIER_CHARS
  79. or _IDENTIFIER_RE.fullmatch(normalized) is None
  80. ):
  81. raise AttachmentError(f"invalid {label}")
  82. return normalized
  83. def _attachment_id(value: Any) -> str:
  84. if not isinstance(value, str) or _ATTACHMENT_ID_RE.fullmatch(value) is None:
  85. raise AttachmentError("invalid attachment_id")
  86. return value
  87. def _name(value: Any) -> str:
  88. if not isinstance(value, str):
  89. raise AttachmentError("attachment name must be a string")
  90. normalized = value.strip()
  91. try:
  92. normalized.encode("utf-8")
  93. except UnicodeError:
  94. raise AttachmentError("attachment name must contain valid Unicode") from None
  95. if (
  96. not normalized
  97. or len(normalized) > MAX_ATTACHMENT_NAME_CHARS
  98. or normalized in {".", ".."}
  99. or any(token in normalized for token in ("/", "\\", "\x00", "\n", "\r"))
  100. ):
  101. raise AttachmentError("attachment name must be a bounded basename")
  102. return normalized
  103. def _mime(value: Any) -> str:
  104. if not isinstance(value, str):
  105. raise AttachmentError("attachment mime must be a string")
  106. normalized = value.strip().lower()
  107. if (
  108. not normalized
  109. or len(normalized) > MAX_ATTACHMENT_MIME_CHARS
  110. or _MIME_RE.fullmatch(normalized) is None
  111. ):
  112. raise AttachmentError("attachment mime is invalid")
  113. return normalized
  114. def _kind(value: Any) -> str:
  115. if not isinstance(value, str) or value not in _ATTACHMENT_KINDS:
  116. raise AttachmentError("attachment kind must be image, pdf, or file")
  117. return value
  118. def _sniff_known_mime(data: bytes) -> str:
  119. if data.startswith(b"\x89PNG\r\n\x1a\n"):
  120. return "image/png"
  121. if data.startswith(b"\xff\xd8\xff"):
  122. return "image/jpeg"
  123. if data.startswith((b"GIF87a", b"GIF89a")):
  124. return "image/gif"
  125. if data.startswith(b"BM"):
  126. return "image/bmp"
  127. if len(data) >= 12 and data.startswith(b"RIFF") and data[8:12] == b"WEBP":
  128. return "image/webp"
  129. if data.startswith(b"%PDF-"):
  130. return "application/pdf"
  131. if data.startswith(b"PK\x03\x04"):
  132. return "application/zip"
  133. if data.startswith(b"\x1f\x8b"):
  134. return "application/gzip"
  135. if b"\x00" not in data:
  136. try:
  137. data.decode("utf-8")
  138. except UnicodeDecodeError:
  139. pass
  140. else:
  141. return "text/plain"
  142. return "application/octet-stream"
  143. def _validate_mime_and_kind(data: bytes, *, kind: str, mime: str) -> None:
  144. sniffed = _sniff_known_mime(data)
  145. if kind == "image":
  146. if not mime.startswith("image/") or sniffed != mime:
  147. raise AttachmentError("image bytes do not match the declared MIME type")
  148. return
  149. if kind == "pdf":
  150. if mime != "application/pdf" or sniffed != "application/pdf":
  151. raise AttachmentError("PDF bytes do not match application/pdf")
  152. return
  153. if mime.startswith("image/") or mime == "application/pdf":
  154. raise AttachmentError("image and PDF MIME types require their matching kind")
  155. if mime.startswith("text/") and sniffed != "text/plain":
  156. raise AttachmentError("text bytes do not match the declared MIME type")
  157. # Generic files are opaque bytes. ZIP and gzip are container formats used
  158. # by many non-archive MIME types (DOCX, XLSX, PPTX, EPUB, JAR, APK, etc.).
  159. # Only image and PDF payloads are security-sensitive enough to require an
  160. # exact magic/MIME match here; text remains strict to avoid binary prompts.
  161. def decode_content_base64(value: Any) -> bytes:
  162. """Strictly decode one bounded RPC upload body without accepting data URLs."""
  163. if not isinstance(value, str) or not value:
  164. raise AttachmentError("content_base64 is required")
  165. if value.lstrip().lower().startswith("data:"):
  166. raise AttachmentError("data URLs are not accepted")
  167. maximum_encoded = ((MAX_ATTACHMENT_BYTES + 2) // 3) * 4
  168. if len(value) > maximum_encoded + 4:
  169. raise AttachmentError("attachment exceeds the per-file size limit")
  170. try:
  171. data = base64.b64decode(value, validate=True)
  172. except (binascii.Error, ValueError) as exc:
  173. raise AttachmentError("content_base64 is not valid base64") from exc
  174. if not data:
  175. raise AttachmentError("attachment must not be empty")
  176. if len(data) > MAX_ATTACHMENT_BYTES:
  177. raise AttachmentError("attachment exceeds the per-file size limit")
  178. return data
  179. def encode_content_base64(data: bytes) -> str:
  180. """Encode verified bytes for the bounded attachment read RPC."""
  181. return base64.b64encode(data).decode("ascii")
  182. def _safe_manifest_entry(value: Any) -> dict[str, Any]:
  183. if not isinstance(value, Mapping):
  184. raise AttachmentError("each attachment manifest entry must be an object")
  185. fields = frozenset(value)
  186. if fields != _ATTACHMENT_FIELDS:
  187. unknown = fields - _ATTACHMENT_FIELDS
  188. missing = _ATTACHMENT_FIELDS - fields
  189. if unknown:
  190. raise AttachmentError(
  191. f"attachment manifest has unknown fields: {', '.join(sorted(unknown))}"
  192. )
  193. raise AttachmentError(
  194. f"attachment manifest is missing fields: {', '.join(sorted(missing))}"
  195. )
  196. size = value["size"]
  197. if isinstance(size, bool) or not isinstance(size, int) or not 0 < size <= MAX_ATTACHMENT_BYTES:
  198. raise AttachmentError("attachment size is outside the per-file limit")
  199. kind = _kind(value["kind"])
  200. mime = _mime(value["mime"])
  201. if kind == "image" and not mime.startswith("image/"):
  202. raise AttachmentError("image attachments require an image MIME type")
  203. if kind == "pdf" and mime != "application/pdf":
  204. raise AttachmentError("PDF attachments require application/pdf")
  205. return {
  206. "attachment_id": _attachment_id(value["attachment_id"]),
  207. "kind": kind,
  208. "name": _name(value["name"]),
  209. "size": size,
  210. "mime": mime,
  211. }
  212. def validate_manifest(value: Any) -> list[dict[str, Any]]:
  213. if not isinstance(value, list):
  214. raise AttachmentError("attachments must be a list")
  215. if len(value) > MAX_ATTACHMENTS_PER_MESSAGE:
  216. raise AttachmentError(
  217. f"attachments must contain at most {MAX_ATTACHMENTS_PER_MESSAGE} entries"
  218. )
  219. normalized = [_safe_manifest_entry(entry) for entry in value]
  220. if len({entry["attachment_id"] for entry in normalized}) != len(normalized):
  221. raise AttachmentError("attachment ids must be unique within one message")
  222. if sum(entry["size"] for entry in normalized) > MAX_MESSAGE_ATTACHMENT_BYTES:
  223. raise AttachmentError("attachments exceed the per-message total size limit")
  224. return normalized
  225. def validate_task_manifest(value: Any) -> list[dict[str, Any]]:
  226. """Validate a bounded aggregate of several already-valid message manifests."""
  227. if not isinstance(value, list):
  228. raise AttachmentError("task attachments must be a list")
  229. if len(value) > MAX_TASK_ATTACHMENTS:
  230. raise AttachmentError(
  231. f"task attachments must contain at most {MAX_TASK_ATTACHMENTS} entries"
  232. )
  233. normalized = [_safe_manifest_entry(entry) for entry in value]
  234. if len({entry["attachment_id"] for entry in normalized}) != len(normalized):
  235. raise AttachmentError("task attachment ids must be unique")
  236. if sum(entry["size"] for entry in normalized) > MAX_TASK_ATTACHMENT_BYTES:
  237. raise AttachmentError("task attachments exceed the aggregate byte limit")
  238. return normalized
  239. # (state.db, blob root) pairs whose schema, event reconciliation and prune already ran in this
  240. # process. Callers build a store per operation (one per attachment chunk); startup work and
  241. # its write locks must not repeat on every read.
  242. _STARTED: set[tuple[str, str, int, int] | None] = set()
  243. _STARTED_LOCK = threading.Lock()
  244. class HostedRoomAttachmentStore:
  245. """SQLite-owned metadata and private, content-deduplicated blob bytes."""
  246. def __init__(
  247. self,
  248. db_path: Path | str,
  249. *,
  250. root: Path | str | None = None,
  251. clock: Callable[[], float] = time.time,
  252. room_quota_bytes: int = MAX_ROOM_ATTACHMENT_BYTES,
  253. gateway_quota_bytes: int = MAX_GATEWAY_BLOB_BYTES,
  254. room_quota_count: int = MAX_ROOM_ATTACHMENT_COUNT,
  255. gateway_quota_count: int = MAX_GATEWAY_ATTACHMENT_COUNT,
  256. ) -> None:
  257. self.db_path = Path(db_path)
  258. self.root = Path(root or default_attachment_root(self.db_path))
  259. self.blob_root = self.root / "blobs"
  260. self.clock = clock
  261. self.room_quota_bytes = max(1, int(room_quota_bytes))
  262. self.gateway_quota_bytes = max(1, int(gateway_quota_bytes))
  263. self.room_quota_count = max(1, int(room_quota_count))
  264. self.gateway_quota_count = max(1, int(gateway_quota_count))
  265. self._lock = threading.RLock()
  266. self._prepare_private_root()
  267. with _STARTED_LOCK:
  268. if self._startup_key() in _STARTED:
  269. return
  270. conn = self._connect()
  271. try:
  272. conn.execute("BEGIN IMMEDIATE")
  273. self._initialize(conn)
  274. conn.commit()
  275. finally:
  276. conn.close()
  277. self.reconcile_room_events()
  278. self.prune()
  279. _STARTED.add(self._startup_key())
  280. def _startup_key(self) -> tuple[str, str, int, int] | None:
  281. """Identity of the database FILE, so a recreated state.db is set up again."""
  282. try:
  283. info = self.db_path.stat()
  284. except OSError:
  285. return None
  286. return str(self.db_path.resolve()), str(self.root.resolve()), info.st_dev, info.st_ino
  287. def _prepare_private_root(self) -> None:
  288. make_user_dir(self.root, parents=True, exist_ok=True, mode=0o700)
  289. make_user_dir(self.blob_root, parents=True, exist_ok=True, mode=0o700)
  290. for path in (self.root, self.blob_root):
  291. try:
  292. os.chmod(path, 0o700)
  293. except OSError:
  294. pass
  295. def _initialize(self, conn: sqlite3.Connection) -> None:
  296. conn.execute(
  297. """CREATE TABLE IF NOT EXISTS hosted_room_attachment_blobs (
  298. blob_id TEXT PRIMARY KEY,
  299. sha256 TEXT NOT NULL UNIQUE,
  300. size INTEGER NOT NULL CHECK (size > 0),
  301. ref_count INTEGER NOT NULL CHECK (ref_count > 0),
  302. created_at REAL NOT NULL
  303. )"""
  304. )
  305. conn.execute(
  306. """CREATE TABLE IF NOT EXISTS hosted_room_attachments (
  307. attachment_id TEXT PRIMARY KEY,
  308. upload_id TEXT NOT NULL,
  309. room_id TEXT NOT NULL,
  310. event_id TEXT,
  311. kind TEXT NOT NULL,
  312. name TEXT NOT NULL,
  313. size INTEGER NOT NULL CHECK (size > 0),
  314. mime TEXT NOT NULL,
  315. sha256 TEXT NOT NULL,
  316. blob_id TEXT NOT NULL,
  317. recipient_member_ids_json TEXT NOT NULL DEFAULT '[]',
  318. viewer_access INTEGER NOT NULL DEFAULT 0 CHECK (viewer_access IN (0, 1)),
  319. state TEXT NOT NULL CHECK (state IN ('uploaded', 'committed', 'disbanded')),
  320. created_at REAL NOT NULL,
  321. updated_at REAL NOT NULL,
  322. expires_at REAL,
  323. UNIQUE (room_id, upload_id),
  324. FOREIGN KEY (blob_id) REFERENCES hosted_room_attachment_blobs(blob_id)
  325. )"""
  326. )
  327. columns = {
  328. str(row["name"])
  329. for row in conn.execute("PRAGMA table_info(hosted_room_attachments)")
  330. }
  331. if "viewer_access" not in columns:
  332. conn.execute(
  333. """ALTER TABLE hosted_room_attachments
  334. ADD COLUMN viewer_access INTEGER NOT NULL DEFAULT 0"""
  335. )
  336. if "catalog_name" not in columns:
  337. conn.execute("ALTER TABLE hosted_room_attachments ADD COLUMN catalog_name TEXT")
  338. reader = conn.execute("SELECT attachment_id, name FROM hosted_room_attachments")
  339. while rows := reader.fetchmany(512):
  340. conn.executemany(
  341. "UPDATE hosted_room_attachments SET catalog_name=? WHERE attachment_id=?",
  342. ((fold_catalog_text(str(row["name"])), row["attachment_id"]) for row in rows),
  343. )
  344. conn.execute(
  345. """CREATE INDEX IF NOT EXISTS idx_hosted_room_attachments_room_state
  346. ON hosted_room_attachments(room_id, state, created_at)"""
  347. )
  348. conn.execute(
  349. """CREATE INDEX IF NOT EXISTS idx_hosted_room_attachments_expiry
  350. ON hosted_room_attachments(expires_at)"""
  351. )
  352. conn.execute(
  353. """CREATE INDEX IF NOT EXISTS idx_hosted_room_attachments_event
  354. ON hosted_room_attachments(room_id, event_id)"""
  355. )
  356. def _connect(self) -> sqlite3.Connection:
  357. from hermes_state_wal import apply_wal_with_fallback
  358. self.db_path.parent.mkdir(parents=True, exist_ok=True)
  359. conn = sqlite3.connect(self.db_path, timeout=10)
  360. conn.row_factory = sqlite3.Row
  361. try:
  362. apply_wal_with_fallback(conn, db_label="state.db (hosted room attachments)")
  363. conn.execute("PRAGMA foreign_keys=ON")
  364. except Exception:
  365. conn.close()
  366. raise
  367. return conn
  368. @contextmanager
  369. def _transaction(self, *, immediate: bool = False) -> Iterator[sqlite3.Connection]:
  370. conn = self._connect()
  371. try:
  372. if immediate:
  373. conn.execute("BEGIN IMMEDIATE")
  374. yield conn
  375. conn.commit()
  376. except Exception:
  377. conn.rollback()
  378. raise
  379. finally:
  380. conn.close()
  381. def _blob_path(self, blob_id: str) -> Path:
  382. if _BLOB_ID_RE.fullmatch(blob_id) is None:
  383. raise AttachmentIntegrityError("stored blob id is invalid")
  384. return self.blob_root / blob_id
  385. def _write_blob(self, target: Path, data: bytes) -> None:
  386. from utils import atomic_write_bytes
  387. # ``.tmp-`` staging names are what the orphan sweep reclaims after a crash.
  388. atomic_write_bytes(target, data, tmp_prefix=".tmp-", mode=0o600, fsync_dir=True)
  389. def _open_blob(self, *, blob_id: str, size: int) -> int:
  390. """Open the regular blob file and fence its size against the durable row."""
  391. path = self._blob_path(blob_id)
  392. flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
  393. try:
  394. descriptor = os.open(path, flags)
  395. except OSError as exc:
  396. raise AttachmentIntegrityError("canonical attachment blob is unavailable") from exc
  397. try:
  398. info = os.fstat(descriptor)
  399. if not stat.S_ISREG(info.st_mode) or info.st_size != size:
  400. raise AttachmentIntegrityError("canonical attachment blob size changed")
  401. except BaseException:
  402. os.close(descriptor)
  403. raise
  404. return descriptor
  405. def _read_blob(self, *, blob_id: str, size: int, sha256: str) -> bytes:
  406. descriptor = self._open_blob(blob_id=blob_id, size=size)
  407. try:
  408. with os.fdopen(descriptor, "rb", closefd=False) as handle:
  409. data = handle.read(MAX_ATTACHMENT_BYTES + 1)
  410. finally:
  411. os.close(descriptor)
  412. if len(data) != size or hashlib.sha256(data).hexdigest() != sha256:
  413. raise AttachmentIntegrityError("canonical attachment blob failed SHA-256 validation")
  414. return data
  415. def _read_blob_range(self, *, blob_id: str, size: int, offset: int, length: int) -> bytes:
  416. """One slice of the blob; the row's SHA-256 is the caller's whole-file digest and the
  417. receiver verifies the reassembled bytes against it, so no slice is hashed here."""
  418. descriptor = self._open_blob(blob_id=blob_id, size=size)
  419. try:
  420. with os.fdopen(descriptor, "rb", closefd=False) as handle:
  421. handle.seek(offset)
  422. data = handle.read(length)
  423. finally:
  424. os.close(descriptor)
  425. if len(data) != min(length, size - offset):
  426. raise AttachmentIntegrityError("canonical attachment blob size changed")
  427. return data
  428. @staticmethod
  429. def _metadata(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]:
  430. result = {
  431. "attachment_id": str(row["attachment_id"]),
  432. "kind": str(row["kind"]),
  433. "name": str(row["name"]),
  434. "size": int(row["size"]),
  435. "mime": str(row["mime"]),
  436. "sha256": str(row["sha256"]),
  437. "state": str(row["state"]),
  438. "created_at": float(row["created_at"]),
  439. "idempotent": idempotent,
  440. }
  441. if row["event_id"] is not None:
  442. result["event_id"] = str(row["event_id"])
  443. return result
  444. def put(
  445. self,
  446. *,
  447. room_id: Any,
  448. upload_id: Any,
  449. kind: Any,
  450. name: Any,
  451. mime: Any,
  452. data: bytes,
  453. ) -> dict[str, Any]:
  454. room_id = _identifier(room_id, label="room_id")
  455. upload_id = _identifier(upload_id, label="upload_id")
  456. kind = _kind(kind)
  457. name = _name(name)
  458. mime = _mime(mime)
  459. if not isinstance(data, bytes) or not data:
  460. raise AttachmentError("attachment bytes must not be empty")
  461. if len(data) > MAX_ATTACHMENT_BYTES:
  462. raise AttachmentError("attachment exceeds the per-file size limit")
  463. _validate_mime_and_kind(data, kind=kind, mime=mime)
  464. digest = hashlib.sha256(data).hexdigest()
  465. now = float(self.clock())
  466. # Long-lived gateways keep one store instance. Reclaim abandoned
  467. # uploads before applying quotas so an expired failed send cannot
  468. # permanently exhaust the room without a process restart.
  469. self.prune(now=now)
  470. with self._lock, self._transaction(immediate=True) as conn:
  471. existing = conn.execute(
  472. "SELECT * FROM hosted_room_attachments WHERE room_id=? AND upload_id=?",
  473. (room_id, upload_id),
  474. ).fetchone()
  475. if existing is not None:
  476. if (
  477. str(existing["kind"]) != kind
  478. or str(existing["name"]) != name
  479. or str(existing["mime"]) != mime
  480. or int(existing["size"]) != len(data)
  481. or str(existing["sha256"]) != digest
  482. ):
  483. raise AttachmentConflictError(
  484. "upload_id was already used for different attachment content"
  485. )
  486. self._read_blob(
  487. blob_id=str(existing["blob_id"]),
  488. size=int(existing["size"]),
  489. sha256=str(existing["sha256"]),
  490. )
  491. return self._metadata(existing, idempotent=True)
  492. room_totals = conn.execute(
  493. """SELECT COALESCE(SUM(size), 0) AS bytes, COUNT(*) AS count
  494. FROM hosted_room_attachments
  495. WHERE room_id=? AND state!='disbanded'""",
  496. (room_id,),
  497. ).fetchone()
  498. uncommitted = conn.execute(
  499. """SELECT COALESCE(SUM(size), 0) AS bytes, COUNT(*) AS count
  500. FROM hosted_room_attachments
  501. WHERE room_id=? AND state='uploaded'""",
  502. (room_id,),
  503. ).fetchone()
  504. if int(room_totals["bytes"]) + len(data) > self.room_quota_bytes:
  505. raise AttachmentQuotaError("room attachment quota exceeded")
  506. if int(room_totals["count"]) >= self.room_quota_count:
  507. raise AttachmentQuotaError("room attachment count quota exceeded")
  508. total_count = int(conn.execute("SELECT COUNT(*) FROM hosted_room_attachments").fetchone()[0])
  509. if total_count >= self.gateway_quota_count:
  510. raise AttachmentQuotaError("gateway attachment count quota exceeded")
  511. if (
  512. int(uncommitted["bytes"]) + len(data) > MAX_ROOM_UNCOMMITTED_BYTES
  513. or int(uncommitted["count"]) + 1 > MAX_ROOM_UNCOMMITTED_COUNT
  514. ):
  515. raise AttachmentQuotaError("room uncommitted upload quota exceeded")
  516. blob = conn.execute(
  517. "SELECT * FROM hosted_room_attachment_blobs WHERE sha256=?",
  518. (digest,),
  519. ).fetchone()
  520. if blob is None:
  521. physical = int(
  522. conn.execute(
  523. "SELECT COALESCE(SUM(size), 0) FROM hosted_room_attachment_blobs"
  524. ).fetchone()[0]
  525. )
  526. if physical + len(data) > self.gateway_quota_bytes:
  527. raise AttachmentQuotaError("gateway attachment quota exceeded")
  528. blob_id = f"blob_{secrets.token_hex(16)}"
  529. target = self._blob_path(blob_id)
  530. self._write_blob(target, data)
  531. conn.execute(
  532. """INSERT INTO hosted_room_attachment_blobs
  533. (blob_id, sha256, size, ref_count, created_at)
  534. VALUES (?, ?, ?, 1, ?)""",
  535. (blob_id, digest, len(data), now),
  536. )
  537. else:
  538. blob_id = str(blob["blob_id"])
  539. self._read_blob(blob_id=blob_id, size=int(blob["size"]), sha256=digest)
  540. conn.execute(
  541. "UPDATE hosted_room_attachment_blobs SET ref_count=ref_count+1 WHERE blob_id=?",
  542. (blob_id,),
  543. )
  544. attachment_id = f"att_{secrets.token_hex(16)}"
  545. conn.execute(
  546. """INSERT INTO hosted_room_attachments
  547. (attachment_id, upload_id, room_id, event_id, kind, name, catalog_name, size,
  548. mime, sha256, blob_id, recipient_member_ids_json, state,
  549. created_at, updated_at, expires_at)
  550. VALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, '[]', 'uploaded', ?, ?, ?)""",
  551. (
  552. attachment_id,
  553. upload_id,
  554. room_id,
  555. kind,
  556. name,
  557. fold_catalog_text(name),
  558. len(data),
  559. mime,
  560. digest,
  561. blob_id,
  562. now,
  563. now,
  564. now + UNCOMMITTED_TTL_SECONDS,
  565. ),
  566. )
  567. row = conn.execute(
  568. "SELECT * FROM hosted_room_attachments WHERE attachment_id=?",
  569. (attachment_id,),
  570. ).fetchone()
  571. if row is None: # pragma: no cover - guarded by insert
  572. raise RuntimeError("stored attachment could not be reloaded")
  573. return self._metadata(row)
  574. def find_upload(self, *, room_id: Any, upload_id: Any) -> dict[str, Any] | None:
  575. """Return verified metadata for an idempotent upload retry, if it exists."""
  576. room_id = _identifier(room_id, label="room_id")
  577. upload_id = _identifier(upload_id, label="upload_id")
  578. now = float(self.clock())
  579. with self._transaction() as conn:
  580. row = conn.execute(
  581. """SELECT * FROM hosted_room_attachments
  582. WHERE room_id=? AND upload_id=?""",
  583. (room_id, upload_id),
  584. ).fetchone()
  585. if row is None or str(row["state"]) == "disbanded":
  586. return None
  587. if row["expires_at"] is not None and float(row["expires_at"]) <= now:
  588. return None
  589. self._read_blob(
  590. blob_id=str(row["blob_id"]),
  591. size=int(row["size"]),
  592. sha256=str(row["sha256"]),
  593. )
  594. return self._metadata(row, idempotent=True)
  595. def commit_message(
  596. self,
  597. *,
  598. room_id: Any,
  599. event_id: Any,
  600. manifest: Any,
  601. recipient_member_ids: Sequence[str],
  602. viewer_access: bool = False,
  603. retention_seconds: float | None = None,
  604. hold_until_event: bool = False,
  605. ) -> list[dict[str, Any]]:
  606. normalized, _transitioned = self.commit_message_with_receipt(
  607. room_id=room_id,
  608. event_id=event_id,
  609. manifest=manifest,
  610. recipient_member_ids=recipient_member_ids,
  611. viewer_access=viewer_access,
  612. retention_seconds=retention_seconds,
  613. hold_until_event=hold_until_event,
  614. )
  615. return normalized
  616. def commit_message_with_receipt(
  617. self,
  618. *,
  619. room_id: Any,
  620. event_id: Any,
  621. manifest: Any,
  622. recipient_member_ids: Sequence[str],
  623. viewer_access: bool = False,
  624. retention_seconds: float | None = None,
  625. hold_until_event: bool = False,
  626. ) -> tuple[list[dict[str, Any]], tuple[str, ...]]:
  627. """Commit a manifest and report only rows transitioned by this call."""
  628. room_id = _identifier(room_id, label="room_id")
  629. event_id = _identifier(event_id, label="event_id")
  630. normalized = validate_manifest(manifest)
  631. recipients = tuple(
  632. _identifier(member_id, label="recipient_member_id")
  633. for member_id in recipient_member_ids
  634. )
  635. if not recipients:
  636. raise AttachmentError("attachment commitment requires recipient ids")
  637. if len(set(recipients)) != len(recipients):
  638. raise AttachmentError("recipient ids must be unique")
  639. recipients_json = json.dumps(sorted(recipients), separators=(",", ":"))
  640. now = float(self.clock())
  641. expires_at = (
  642. now + UNCOMMITTED_TTL_SECONDS
  643. if hold_until_event
  644. else None
  645. if retention_seconds is None
  646. else now + float(retention_seconds)
  647. )
  648. transitioned: list[str] = []
  649. with self._lock, self._transaction(immediate=True) as conn:
  650. for entry in normalized:
  651. row = conn.execute(
  652. "SELECT * FROM hosted_room_attachments WHERE attachment_id=?",
  653. (entry["attachment_id"],),
  654. ).fetchone()
  655. if row is None or str(row["room_id"]) != room_id:
  656. raise AttachmentNotFoundError("attachment is not uploaded for this room")
  657. durable = {
  658. "attachment_id": str(row["attachment_id"]),
  659. "kind": str(row["kind"]),
  660. "name": str(row["name"]),
  661. "size": int(row["size"]),
  662. "mime": str(row["mime"]),
  663. }
  664. if durable != entry:
  665. raise AttachmentConflictError(
  666. "attachment manifest metadata does not match the uploaded bytes"
  667. )
  668. if row["expires_at"] is not None and float(row["expires_at"]) <= now:
  669. raise AttachmentNotFoundError("attachment has expired")
  670. state = str(row["state"])
  671. if state == "disbanded":
  672. raise AttachmentNotFoundError("attachment belongs to a disbanded room")
  673. if state == "committed" and (
  674. str(row["event_id"] or "") != event_id
  675. or str(row["recipient_member_ids_json"]) != recipients_json
  676. or bool(row["viewer_access"]) != bool(viewer_access)
  677. ):
  678. raise AttachmentConflictError(
  679. "attachment is already owned by a different room event"
  680. )
  681. if state == "uploaded":
  682. transitioned.append(entry["attachment_id"])
  683. self._read_blob(
  684. blob_id=str(row["blob_id"]),
  685. size=int(row["size"]),
  686. sha256=str(row["sha256"]),
  687. )
  688. for entry in normalized:
  689. if entry["attachment_id"] not in transitioned:
  690. continue
  691. updated = conn.execute(
  692. """UPDATE hosted_room_attachments
  693. SET event_id=?, recipient_member_ids_json=?, viewer_access=?, state='committed',
  694. updated_at=?, expires_at=?
  695. WHERE attachment_id=? AND state='uploaded'""",
  696. (
  697. event_id,
  698. recipients_json,
  699. 1 if viewer_access else 0,
  700. now,
  701. expires_at,
  702. entry["attachment_id"],
  703. ),
  704. )
  705. if updated.rowcount != 1:
  706. raise AttachmentConflictError(
  707. "attachment changed during message commitment"
  708. )
  709. return normalized, tuple(transitioned)
  710. def abort_message_commit(
  711. self,
  712. *,
  713. room_id: Any,
  714. event_id: Any,
  715. attachment_ids: Sequence[str],
  716. ) -> int:
  717. """Return a failed pre-event commitment to its bounded upload state."""
  718. room_id = _identifier(room_id, label="room_id")
  719. event_id = _identifier(event_id, label="event_id")
  720. attachment_ids = [_attachment_id(value) for value in attachment_ids]
  721. if not attachment_ids:
  722. return 0
  723. now = float(self.clock())
  724. with self._transaction(immediate=True) as conn:
  725. owner = _owner_event(conn, room_id, event_id)
  726. if owner is not None:
  727. published_ids = {
  728. item.get("attachment_id")
  729. for item in json.loads(owner["payload_json"]).get("attachments", [])
  730. if isinstance(item, Mapping)
  731. }
  732. attachment_ids = [
  733. value for value in attachment_ids if value not in published_ids
  734. ]
  735. if not attachment_ids:
  736. return 0
  737. placeholders = ",".join("?" for _ in attachment_ids)
  738. changed = conn.execute(
  739. f"""UPDATE hosted_room_attachments
  740. SET event_id=NULL, recipient_member_ids_json='[]', viewer_access=0,
  741. state='uploaded', updated_at=?, expires_at=?
  742. WHERE room_id=? AND event_id=? AND state='committed'
  743. AND attachment_id IN ({placeholders})""",
  744. (
  745. now,
  746. now + UNCOMMITTED_TTL_SECONDS,
  747. room_id,
  748. event_id,
  749. *attachment_ids,
  750. ),
  751. )
  752. return int(changed.rowcount)
  753. def retain_event(self, *, room_id: Any, event_id: Any) -> int:
  754. """Retain committed blobs after their immutable room event is durable."""
  755. room_id = _identifier(room_id, label="room_id")
  756. event_id = _identifier(event_id, label="event_id")
  757. now = float(self.clock())
  758. with self._transaction(immediate=True) as conn:
  759. event = conn.execute(
  760. """SELECT 1 FROM hosted_room_events
  761. WHERE room_id=? AND event_id=? LIMIT 1""",
  762. (room_id, event_id),
  763. ).fetchone()
  764. if event is None:
  765. raise AttachmentConflictError(
  766. "attachment owner event is not durable in the room log"
  767. )
  768. changed = conn.execute(
  769. """UPDATE hosted_room_attachments
  770. SET expires_at=NULL, updated_at=?
  771. WHERE room_id=? AND event_id=? AND state='committed'""",
  772. (now, room_id, event_id),
  773. )
  774. return int(changed.rowcount)
  775. def read(
  776. self,
  777. *,
  778. room_id: Any,
  779. attachment_id: Any,
  780. recipient_member_id: Any,
  781. event_id: Any | None = None,
  782. viewer: bool = False,
  783. ) -> AttachmentData:
  784. room_id = _identifier(room_id, label="room_id")
  785. attachment_id = _attachment_id(attachment_id)
  786. recipient_member_id = (
  787. _identifier(recipient_member_id, label="recipient_member_id")
  788. if recipient_member_id is not None
  789. else ""
  790. )
  791. normalized_event = _identifier(event_id, label="event_id") if event_id is not None else None
  792. now = float(self.clock())
  793. with self._transaction() as conn:
  794. row = self._read_committed_row(
  795. conn, room_id=room_id, attachment_id=attachment_id,
  796. recipient_member_id=recipient_member_id,
  797. normalized_event=normalized_event, viewer=viewer, now=now,
  798. )
  799. data = self._read_blob(
  800. blob_id=str(row["blob_id"]),
  801. size=int(row["size"]),
  802. sha256=str(row["sha256"]),
  803. )
  804. return AttachmentData(self._metadata(row), data)
  805. def describe(
  806. self,
  807. *,
  808. room_id: Any,
  809. attachment_id: Any,
  810. recipient_member_id: Any,
  811. event_id: Any,
  812. ) -> dict[str, Any]:
  813. """``read`` without bytes: the metadata (including the upload-verified SHA-256) of a
  814. committed attachment this recipient may read for this event."""
  815. room_id = _identifier(room_id, label="room_id")
  816. attachment_id = _attachment_id(attachment_id)
  817. recipient_member_id = _identifier(recipient_member_id, label="recipient_member_id")
  818. normalized_event = _identifier(event_id, label="event_id")
  819. with self._transaction() as conn:
  820. row = self._read_committed_row(
  821. conn, room_id=room_id, attachment_id=attachment_id,
  822. recipient_member_id=recipient_member_id,
  823. normalized_event=normalized_event, viewer=False, now=float(self.clock()),
  824. )
  825. return self._metadata(row)
  826. def read_range(
  827. self,
  828. *,
  829. room_id: Any,
  830. attachment_id: Any,
  831. recipient_member_id: Any,
  832. event_id: Any,
  833. offset: int,
  834. length: int,
  835. ) -> AttachmentData:
  836. """``read`` for one slice: same authorization, metadata carries the stored whole-file
  837. SHA-256 and the receiver verifies the reassembled bytes against it."""
  838. room_id = _identifier(room_id, label="room_id")
  839. attachment_id = _attachment_id(attachment_id)
  840. recipient_member_id = _identifier(recipient_member_id, label="recipient_member_id")
  841. normalized_event = _identifier(event_id, label="event_id")
  842. if type(offset) is not int or type(length) is not int or offset < 0 or length <= 0:
  843. raise AttachmentError("attachment range must be non-negative integers")
  844. with self._transaction() as conn:
  845. row = self._read_committed_row(
  846. conn, room_id=room_id, attachment_id=attachment_id,
  847. recipient_member_id=recipient_member_id,
  848. normalized_event=normalized_event, viewer=False, now=float(self.clock()),
  849. )
  850. if offset >= int(row["size"]):
  851. raise AttachmentError("attachment range starts past the end")
  852. data = self._read_blob_range(
  853. blob_id=str(row["blob_id"]), size=int(row["size"]), offset=offset, length=length)
  854. return AttachmentData(self._metadata(row), data)
  855. def reconcile_room_events(self) -> int:
  856. """Recover only attachment commitments named by the durable event payload."""
  857. changed = 0
  858. now = float(self.clock())
  859. with self._transaction(immediate=True) as conn:
  860. has_events = conn.execute(
  861. """SELECT 1 FROM sqlite_master
  862. WHERE type='table' AND name='hosted_room_events'"""
  863. ).fetchone()
  864. if has_events is None:
  865. return 0
  866. rows = conn.execute(
  867. """SELECT attachment.attachment_id, attachment.event_id,
  868. event.payload_json
  869. FROM hosted_room_attachments AS attachment
  870. JOIN hosted_room_events AS event
  871. ON event.room_id=attachment.room_id
  872. AND event.event_id=attachment.event_id
  873. WHERE attachment.state='committed'
  874. AND attachment.expires_at IS NOT NULL"""
  875. ).fetchall()
  876. for row in rows:
  877. try:
  878. payload = json.loads(str(row["payload_json"]))
  879. ids = {
  880. str(item.get("attachment_id") or "")
  881. for item in payload.get("attachments", [])
  882. if isinstance(item, Mapping)
  883. }
  884. # Malformed row: bad/over-nested JSON, a non-object payload, or non-iterable attachments.
  885. except (ValueError, TypeError, AttributeError, RecursionError):
  886. continue
  887. if str(row["attachment_id"]) not in ids:
  888. continue
  889. changed += conn.execute(
  890. """UPDATE hosted_room_attachments
  891. SET expires_at=NULL, updated_at=?
  892. WHERE attachment_id=? AND state='committed'""",
  893. (now, str(row["attachment_id"])),
  894. ).rowcount
  895. return int(changed)
  896. def mark_room_disbanded(self, room_id: Any) -> int:
  897. room_id = _identifier(room_id, label="room_id")
  898. now = float(self.clock())
  899. with self._transaction(immediate=True) as conn:
  900. changed = conn.execute(
  901. """UPDATE hosted_room_attachments
  902. SET state='disbanded', updated_at=?, expires_at=?
  903. WHERE room_id=? AND state!='disbanded'""",
  904. (now, now + DISBANDED_GRACE_SECONDS, room_id),
  905. )
  906. return int(changed.rowcount)
  907. def prune(self, *, now: float | None = None) -> int:
  908. now = float(self.clock()) if now is None else float(now)
  909. removed_blob_ids: list[str] = []
  910. removed = 0
  911. with self._lock, self._transaction(immediate=True) as conn:
  912. has_rooms = conn.execute(
  913. """SELECT 1 FROM sqlite_master
  914. WHERE type='table' AND name='hosted_rooms'"""
  915. ).fetchone()
  916. if has_rooms is not None:
  917. conn.execute(
  918. """UPDATE hosted_room_attachments
  919. SET state='disbanded',
  920. expires_at=(
  921. SELECT room.disbanded_at + ? FROM hosted_rooms AS room
  922. WHERE room.room_id=hosted_room_attachments.room_id
  923. ),
  924. updated_at=?
  925. WHERE state='committed' AND EXISTS (
  926. SELECT 1 FROM hosted_rooms AS room
  927. WHERE room.room_id=hosted_room_attachments.room_id
  928. AND room.disbanded_at IS NOT NULL
  929. )""",
  930. (DISBANDED_GRACE_SECONDS, now),
  931. )
  932. rows = conn.execute(
  933. """SELECT attachment_id, blob_id FROM hosted_room_attachments
  934. WHERE expires_at IS NOT NULL AND expires_at <= ?""",
  935. (now,),
  936. ).fetchall()
  937. released = Counter(str(row["blob_id"]) for row in rows)
  938. for row in rows:
  939. conn.execute(
  940. "DELETE FROM hosted_room_attachments WHERE attachment_id=?",
  941. (str(row["attachment_id"]),),
  942. )
  943. removed += 1
  944. for blob_id, count in released.items():
  945. blob = conn.execute(
  946. "SELECT ref_count FROM hosted_room_attachment_blobs WHERE blob_id=?",
  947. (blob_id,),
  948. ).fetchone()
  949. if blob is None:
  950. continue
  951. if int(blob["ref_count"]) <= count:
  952. removed_blob_ids.append(blob_id)
  953. else:
  954. conn.execute(
  955. """UPDATE hosted_room_attachment_blobs
  956. SET ref_count=ref_count-?
  957. WHERE blob_id=?""",
  958. (count, blob_id),
  959. )
  960. if removed_blob_ids:
  961. conn.executemany(
  962. "DELETE FROM hosted_room_attachment_blobs WHERE blob_id=?",
  963. ((blob_id,) for blob_id in removed_blob_ids),
  964. )
  965. for blob_id in removed_blob_ids:
  966. self._blob_path(blob_id).unlink(missing_ok=True)
  967. self._sweep_orphans()
  968. return removed
  969. def _sweep_orphans(self) -> None:
  970. quarantined: list[Path] = []
  971. with self._lock:
  972. try:
  973. candidates = tuple(self.blob_root.iterdir())
  974. except OSError:
  975. return
  976. with self._transaction(immediate=True) as conn:
  977. live = {
  978. str(row["blob_id"])
  979. for row in conn.execute(
  980. "SELECT blob_id FROM hosted_room_attachment_blobs"
  981. )
  982. }
  983. for index, path in enumerate(candidates):
  984. if not (
  985. path.name.startswith((".tmp-", ".prune-"))
  986. or (
  987. _BLOB_ID_RE.fullmatch(path.name) is not None
  988. and path.name not in live
  989. )
  990. ):
  991. continue
  992. quarantine = self.blob_root / (
  993. f".prune-{os.getpid()}-{threading.get_ident()}-"
  994. f"{time.time_ns()}-{index}"
  995. )
  996. try:
  997. os.replace(path, quarantine)
  998. except OSError:
  999. continue
  1000. quarantined.append(quarantine)
  1001. # Renaming while the write transaction is active is the atomic
  1002. # reference fence; unlinking the private quarantine can be slow and
  1003. # therefore happens only after SQLite has committed.
  1004. for path in quarantined:
  1005. path.unlink(missing_ok=True)
  1006. def stats(self, *, room_id: str | None = None) -> dict[str, int]:
  1007. with self._transaction() as conn:
  1008. where = " WHERE room_id=?" if room_id is not None else ""
  1009. params = (room_id,) if room_id is not None else ()
  1010. row = conn.execute(
  1011. f"SELECT COUNT(*) AS count, COALESCE(SUM(size), 0) AS bytes "
  1012. f"FROM hosted_room_attachments{where}",
  1013. params,
  1014. ).fetchone()
  1015. blob = conn.execute(
  1016. "SELECT COUNT(*) AS count, COALESCE(SUM(size), 0) AS bytes "
  1017. "FROM hosted_room_attachment_blobs"
  1018. ).fetchone()
  1019. return {
  1020. "attachments": int(row["count"]),
  1021. "logical_bytes": int(row["bytes"]),
  1022. "blobs": int(blob["count"]),
  1023. "physical_bytes": int(blob["bytes"]),
  1024. }
  1025. def read_viewer(
  1026. self,
  1027. *,
  1028. room_id: Any,
  1029. attachment_id: Any,
  1030. event_id: Any | None = None,
  1031. recipient_member_id: Any = None,
  1032. authority_gateway_id: Any,
  1033. authority_epoch: Any,
  1034. ) -> AttachmentData:
  1035. """Read for a live hosted viewer, fencing revocation across byte I/O."""
  1036. room_id = _identifier(room_id, label="room_id")
  1037. attachment_id = _attachment_id(attachment_id)
  1038. normalized_event = _identifier(event_id, label="event_id") if event_id is not None else None
  1039. recipient_member_id = (
  1040. _identifier(recipient_member_id, label="recipient_member_id")
  1041. if recipient_member_id is not None else ""
  1042. )
  1043. authority_gateway_id = _identifier(authority_gateway_id, label="authority_gateway_id")
  1044. if isinstance(authority_epoch, bool) or not isinstance(authority_epoch, int) or authority_epoch < 1:
  1045. raise AttachmentError("authority_epoch must be a positive integer")
  1046. scope = {
  1047. "room_id": room_id,
  1048. "authority_gateway_id": authority_gateway_id,
  1049. "authority_epoch": authority_epoch,
  1050. }
  1051. selection = {
  1052. "room_id": room_id,
  1053. "attachment_id": attachment_id,
  1054. "recipient_member_id": recipient_member_id,
  1055. "normalized_event": normalized_event,
  1056. "viewer": True,
  1057. }
  1058. with self._transaction() as conn:
  1059. conn.execute("BEGIN")
  1060. self._require_viewer_room(conn, **scope)
  1061. row = self._read_committed_row(conn, **selection, now=float(self.clock()))
  1062. # No execution lock or SQLite snapshot spans filesystem/hash work.
  1063. data = self._read_blob(
  1064. blob_id=str(row["blob_id"]), size=int(row["size"]), sha256=str(row["sha256"]),
  1065. )
  1066. with self._transaction() as conn:
  1067. conn.execute("BEGIN")
  1068. self._require_viewer_room(conn, **scope)
  1069. current = self._read_committed_row(conn, **selection, now=float(self.clock()))
  1070. if current["blob_id"] != row["blob_id"] or self._metadata(current) != self._metadata(row):
  1071. raise AttachmentNotFoundError("attachment changed during viewer read")
  1072. return AttachmentData(self._metadata(current), data)
  1073. @staticmethod
  1074. def _require_viewer_room(
  1075. conn: sqlite3.Connection,
  1076. *,
  1077. room_id: str,
  1078. authority_gateway_id: str,
  1079. authority_epoch: int,
  1080. ) -> None:
  1081. from gateway.hosted_rooms_common import table_exists as _table_exists
  1082. if not _table_exists(conn, "hosted_rooms"):
  1083. raise AttachmentNotFoundError("Group Chat is unavailable to viewers")
  1084. room = conn.execute(
  1085. """SELECT authority_gateway_id, authority_epoch, disbanded_at
  1086. FROM hosted_rooms WHERE room_id=?""",
  1087. (room_id,),
  1088. ).fetchone()
  1089. if (
  1090. room is None
  1091. or room["disbanded_at"] is not None
  1092. or room["authority_gateway_id"] != authority_gateway_id
  1093. or room["authority_epoch"] != authority_epoch
  1094. ):
  1095. raise AttachmentNotFoundError("Group Chat viewer authority changed")
  1096. # Files-only sources need not install the optional safety/retirement schema.
  1097. for table in ("hosted_room_quarantine", "hosted_room_disband_fences"):
  1098. if _table_exists(conn, table) and conn.execute(
  1099. f"SELECT 1 FROM {table} WHERE room_id=?", (room_id,),
  1100. ).fetchone() is not None:
  1101. raise AttachmentNotFoundError("Group Chat is unavailable to viewers")
  1102. @staticmethod
  1103. def _read_committed_row(
  1104. conn: sqlite3.Connection,
  1105. *,
  1106. room_id: str,
  1107. attachment_id: str,
  1108. recipient_member_id: str,
  1109. normalized_event: str | None,
  1110. viewer: bool,
  1111. now: float,
  1112. ) -> sqlite3.Row:
  1113. row = conn.execute(
  1114. "SELECT * FROM hosted_room_attachments WHERE attachment_id=? AND room_id=?",
  1115. (attachment_id, room_id),
  1116. ).fetchone()
  1117. if row is None or str(row["state"]) != "committed":
  1118. raise AttachmentNotFoundError("attachment is not committed for this room")
  1119. if row["expires_at"] is not None and float(row["expires_at"]) <= now:
  1120. raise AttachmentNotFoundError("attachment has expired")
  1121. if normalized_event is not None and str(row["event_id"] or "") != normalized_event:
  1122. raise AttachmentNotFoundError("attachment is not owned by this room event")
  1123. if int(row["viewer_access"] or 0):
  1124. # Room-visible files need a published owner even for member reads.
  1125. # A pre-event commitment is staging, not authority to expose bytes.
  1126. if (
  1127. conn.execute(
  1128. "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_events'"
  1129. ).fetchone()
  1130. is None
  1131. ):
  1132. raise AttachmentNotFoundError(
  1133. "attachment is not published in this room"
  1134. )
  1135. owner = conn.execute(
  1136. """SELECT payload_json FROM hosted_room_events
  1137. WHERE room_id=? AND event_id=? AND kind IN ('message.user', 'message.member')""",
  1138. (room_id, str(row["event_id"] or "")),
  1139. ).fetchone()
  1140. manifest = {
  1141. key: row[key]
  1142. for key in ("attachment_id", "kind", "name", "size", "mime")
  1143. }
  1144. payload = (
  1145. json.loads(owner["payload_json"]) if owner is not None else None
  1146. )
  1147. published = (
  1148. payload.get("attachments") if isinstance(payload, Mapping) else None
  1149. )
  1150. if not isinstance(published, list) or manifest not in published:
  1151. raise AttachmentNotFoundError(
  1152. "attachment is not published in this room"
  1153. )
  1154. recipients = json.loads(str(row["recipient_member_ids_json"]))
  1155. if viewer:
  1156. if int(row["viewer_access"] or 0) != 1:
  1157. raise AttachmentNotFoundError(
  1158. "attachment is unavailable to Group Chat viewers"
  1159. )
  1160. elif recipient_member_id not in recipients:
  1161. raise AttachmentNotFoundError("attachment is unavailable to this recipient")
  1162. return row
  1163. __all__ = [
  1164. "CLASSIC_ATTACHMENT_TTL_SECONDS",
  1165. "DISBANDED_GRACE_SECONDS",
  1166. "MAX_ATTACHMENTS_PER_MESSAGE",
  1167. "MAX_ATTACHMENT_BYTES",
  1168. "MAX_MESSAGE_ATTACHMENT_BYTES",
  1169. "MAX_TASK_ATTACHMENTS",
  1170. "MAX_TASK_ATTACHMENT_BYTES",
  1171. "UNCOMMITTED_TTL_SECONDS",
  1172. "AttachmentConflictError",
  1173. "AttachmentData",
  1174. "AttachmentError",
  1175. "AttachmentIntegrityError",
  1176. "AttachmentNotFoundError",
  1177. "AttachmentQuotaError",
  1178. "HostedRoomAttachmentStore",
  1179. "decode_content_base64",
  1180. "default_attachment_root",
  1181. "encode_content_base64",
  1182. "validate_manifest",
  1183. "validate_task_manifest",
  1184. ]
  1185. def _owner_event(
  1186. conn: sqlite3.Connection, room_id: str, event_id: str
  1187. ) -> sqlite3.Row | None:
  1188. # The standalone private-transport store need not have a canonical room log.
  1189. if (
  1190. conn.execute(
  1191. "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_events'"
  1192. ).fetchone()
  1193. is None
  1194. ):
  1195. return None
  1196. return conn.execute(
  1197. "SELECT payload_json FROM hosted_room_events WHERE room_id=? AND event_id=?",
  1198. (room_id, event_id),
  1199. ).fetchone()
  1200. def retain_message_attachments(
  1201. conn: sqlite3.Connection,
  1202. *,
  1203. room_id: str,
  1204. event_id: str,
  1205. manifest: Any,
  1206. now: float,
  1207. ) -> None:
  1208. """Validate and retain exact commitments inside the canonical append transaction."""
  1209. normalized = validate_manifest(manifest)
  1210. if not normalized:
  1211. return
  1212. if normalized != manifest:
  1213. raise AttachmentError("published attachment metadata must be canonical")
  1214. if (
  1215. conn.execute(
  1216. "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_attachments'"
  1217. ).fetchone()
  1218. is None
  1219. ):
  1220. raise EventAttachmentConflictError("message attachments are not committed")
  1221. for entry in normalized:
  1222. row = conn.execute(
  1223. "SELECT * FROM hosted_room_attachments WHERE attachment_id=?",
  1224. (entry["attachment_id"],),
  1225. ).fetchone()
  1226. if (
  1227. row is None
  1228. or row["room_id"] != room_id
  1229. or row["event_id"] != event_id
  1230. or row["state"] != "committed"
  1231. or row["viewer_access"] != 1
  1232. or (row["expires_at"] is not None and float(row["expires_at"]) <= now)
  1233. or any(row[key] != value for key, value in entry.items())
  1234. ):
  1235. raise EventAttachmentConflictError(
  1236. "attachment commitment changed before event publication"
  1237. )
  1238. # Rollback and expiry use the same SQL write lock. No byte or peer I/O is
  1239. # needed here; import verified the bytes before staging these commitments.
  1240. conn.executemany(
  1241. """UPDATE hosted_room_attachments SET expires_at=NULL, updated_at=?
  1242. WHERE attachment_id=?""",
  1243. ((now, entry["attachment_id"]) for entry in normalized),
  1244. )
  1245. def fold_catalog_text(value: str) -> str:
  1246. if value.isascii():
  1247. return value.lower()
  1248. return "".join(
  1249. char for char in unicodedata.normalize("NFKD", value).casefold()
  1250. if not unicodedata.combining(char)
  1251. )