realtimeBroker.ts 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450
  1. import { randomUUID } from 'node:crypto'
  2. import WebSocket from 'ws'
  3. import { HttpError } from './errors.js'
  4. import { checkedChatFrame, CHAT_MAX_PAYLOAD, GroupFrameValidator } from './realtimeProtocol.js'
  5. import { RealtimeReceipts, type CommandReceipt } from './realtimeReceipts.js'
  6. type Frame = Record<string, any>
  7. export interface RealtimePrincipal {
  8. key: string
  9. upstreamKey: string
  10. instanceKey?: string
  11. paired: boolean
  12. valid(): boolean
  13. authorize?(kind: 'chat' | 'groups', method?: string): void
  14. url(channel: 'chat' | 'groups', anchor?: { epoch: string; cursor: number }): Promise<URL>
  15. observeCommand?(frame: string): void
  16. observeEvent?(frame: string): void
  17. }
  18. export interface StreamEntry { id: string; event: 'frame' | 'reset'; data: string; at: number; bytes: number }
  19. export interface RealtimeChannel {
  20. id: string; kind: 'chat' | 'groups'; principal: RealtimePrincipal
  21. seq: number; droppedThrough: number; entries: StreamEntry[]; routes: Set<string>
  22. listeners: Set<(entry: StreamEntry) => void>; touched: number; bytes: number
  23. group?: { epoch: string; cursor: number }; socket?: WebSocket
  24. }
  25. interface Route { profile: string; stored: string; runtime: string; active: boolean; seq: number; observers: Map<string, RealtimePrincipal> }
  26. interface Pending { resolve(frame: Frame): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout> }
  27. interface Upstream {
  28. principal: RealtimePrincipal; socket?: WebSocket; connecting?: Promise<void>; ready?: Frame
  29. pending: Map<string, Pending>; routes: Map<string, Route>; byRuntime: Map<string, Route>
  30. tails: Map<string, Promise<unknown>>; early: Map<string, Frame[]>; earlyBytes: number
  31. retry?: ReturnType<typeof setTimeout>; attempts: number; epoch?: string; recovering: boolean
  32. lastUsed: number; ping?: ReturnType<typeof setInterval>
  33. deferred: Frame[]
  34. recovery?: Promise<void>
  35. }
  36. /** Server-owned WS transports. Downstream detach never tears down an active upstream. */
  37. export class RealtimeBroker {
  38. readonly epoch = randomUUID()
  39. readonly channels = new Map<string, RealtimeChannel>()
  40. private upstreams = new Map<string, Upstream>()
  41. private commands = new Map<string, Promise<CommandReceipt>>()
  42. private completed = new Map<string, { receipt: CommandReceipt; at: number; bytes: number }>()
  43. private receipts: RealtimeReceipts
  44. private timer: ReturnType<typeof setInterval>
  45. private closed = false
  46. constructor(home: string, readonly now = Date.now) {
  47. this.receipts = new RealtimeReceipts(home)
  48. this.timer = setInterval(() => this.sweep(), 5_000)
  49. this.timer.unref()
  50. }
  51. async create(principal: RealtimePrincipal, kind: 'chat' | 'groups', group?: { epoch: string; cursor: number }): Promise<RealtimeChannel> {
  52. if (this.closed) throw new HttpError(503, 'Realtime broker is stopping', 'broker_stopping')
  53. principal.authorize?.(kind)
  54. if (!principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  55. // Closed pages leave resumable channels behind. They must not consume the live-stream
  56. // quota; evict oldest detached handles under pressure, never a running upstream.
  57. let own = [...this.channels.values()].filter(c => c.principal.key === principal.key)
  58. while (own.length >= 32) {
  59. const stale = own.filter(c => !c.listeners.size).sort((a, b) => a.touched - b.touched)[0]
  60. if (!stale) break
  61. this.remove(stale); own = own.filter(c => c !== stale)
  62. }
  63. if (own.length >= 32 || this.channels.size >= 512) {
  64. throw new HttpError(429, 'Too many realtime channels', 'channel_limit')
  65. }
  66. const c: RealtimeChannel = { id: randomUUID(), kind, principal, seq: 0, droppedThrough: 0,
  67. entries: [], routes: new Set(), listeners: new Set(), touched: this.now(), bytes: 0, group }
  68. this.channels.set(c.id, c)
  69. try {
  70. if (kind === 'groups') {
  71. if (!group) throw new HttpError(400, 'Group anchor is required', 'invalid_anchor')
  72. await this.connectGroup(c)
  73. } else {
  74. const u = this.upstream(principal)
  75. await this.connect(u)
  76. this.emit(c, 'frame', u.ready!)
  77. }
  78. return c
  79. } catch (e) { this.remove(c); throw e }
  80. }
  81. get(principal: RealtimePrincipal, id: string): RealtimeChannel {
  82. const c = this.channels.get(id)
  83. if (!c || c.principal.key !== principal.key) throw new HttpError(404, 'Channel not found', 'channel_not_found')
  84. if (!principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  85. c.touched = this.now()
  86. return c
  87. }
  88. detach(principal: RealtimePrincipal, id: string): void { this.remove(this.get(principal, id)) }
  89. release(c: RealtimeChannel): void { if (this.channels.get(c.id) === c) this.remove(c) }
  90. private remove(c: RealtimeChannel): void {
  91. this.channels.delete(c.id)
  92. c.socket?.close()
  93. for (const listener of c.listeners) {
  94. try { listener({ id: '', event: 'reset', data: '{"reason":"channel_closed"}', at: this.now(), bytes: 0 }) } catch { /* a closed consumer must not affect the upstream */ }
  95. }
  96. c.listeners.clear()
  97. }
  98. subscribe(c: RealtimeChannel, cursor: string | undefined, listener: (entry: StreamEntry) => void): () => void {
  99. if ([...this.channels.values()].filter(v => v.principal.key === c.principal.key).reduce((n, v) => n + v.listeners.size, 0) >= 8) {
  100. throw new HttpError(429, 'Too many active event streams', 'stream_limit')
  101. }
  102. this.trim(c)
  103. let seq = 0
  104. if (cursor) {
  105. const prefix = `${this.epoch}:`
  106. if (!cursor.startsWith(prefix) || !/^\d+$/.test(cursor.slice(prefix.length))) {
  107. throw new HttpError(409, 'Stream epoch changed', 'reset_required')
  108. }
  109. seq = Number(cursor.slice(prefix.length))
  110. }
  111. if (!Number.isSafeInteger(seq) || seq < c.droppedThrough || seq > c.seq) {
  112. throw new HttpError(409, 'Replay window expired', 'reset_required')
  113. }
  114. c.listeners.add(listener)
  115. for (const entry of c.entries) if (Number(entry.id.split(':').at(-1)) > seq) listener(entry)
  116. return () => { c.listeners.delete(listener); c.touched = this.now() }
  117. }
  118. receipt(p: RealtimePrincipal, id: string): CommandReceipt {
  119. if (!p.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  120. const receipt = this.completed.get(`${p.key}:${id}`)?.receipt ?? this.receipts.lookup(p.key, id)
  121. if (!receipt) throw new HttpError(404, 'Receipt not found; do not automatically resubmit an attempted command', 'receipt_not_found')
  122. return receipt
  123. }
  124. command(c: RealtimeChannel, requestId: string, input: Frame): Promise<CommandReceipt> {
  125. if (this.closed) throw new HttpError(503, 'Realtime broker is stopping', 'broker_stopping')
  126. if (c.kind !== 'chat') throw new HttpError(403, 'Group event channel is read-only', 'read_only')
  127. const normalized = checkedChatFrame(Buffer.from(JSON.stringify(input)), false, c.principal.paired)
  128. const frame = JSON.parse(normalized) as Frame
  129. c.principal.authorize?.('chat', String(frame.method))
  130. delete frame.id
  131. const fingerprint = JSON.stringify(frame)
  132. if (!this.receipts.lookup(c.principal.key, requestId) && (this.commands.size >= 1024
  133. || [...this.commands.keys()].filter(k => k.startsWith(`${c.principal.key}:`)).length >= 32)) {
  134. throw new HttpError(429, 'Too many pending commands', 'command_limit')
  135. }
  136. const existing = this.receipts.reserve(c.principal.key, requestId, fingerprint)
  137. const key = `${c.principal.key}:${requestId}`
  138. if (existing) return this.commands.get(key) ?? Promise.resolve(this.completed.get(key)?.receipt ?? existing)
  139. const work = this.perform(c, frame).then(response => {
  140. const state = response.error ? 'rejected' : 'confirmed'
  141. this.receipts.finish(c.principal.key, requestId, state, response)
  142. return { requestId, state, response } as CommandReceipt
  143. }, (error: unknown) => {
  144. if (error instanceof HttpError) {
  145. const response = { error: { code: error.code, message: error.message } }
  146. this.receipts.finish(c.principal.key, requestId, 'rejected', response)
  147. return { requestId, state: 'rejected', response } as CommandReceipt
  148. }
  149. this.receipts.finish(c.principal.key, requestId, 'unknown')
  150. return { requestId, state: 'unknown' } as CommandReceipt
  151. })
  152. this.commands.set(key, work)
  153. void work.then(receipt => {
  154. this.commands.delete(key)
  155. this.completed.set(key, { receipt, at: this.now(), bytes: Buffer.byteLength(JSON.stringify(receipt)) })
  156. while (this.completed.size > 10_000 || [...this.completed.values()].reduce((n, x) => n + x.bytes, 0) > 32 * 1024 * 1024) this.completed.delete(this.completed.keys().next().value!)
  157. })
  158. return work
  159. }
  160. private async perform(c: RealtimeChannel, frame: Frame): Promise<Frame> {
  161. const u = this.upstream(c.principal)
  162. const p = frame.params as Frame
  163. const method = String(frame.method)
  164. const opening = method === 'session.create' || method === 'session.resume'
  165. if (opening && (c.routes.size >= 128 || u.routes.size >= 512)) throw new HttpError(429, 'Route capacity reached', 'route_limit')
  166. let route = p.session_id ? u.byRuntime.get(p.session_id) : undefined
  167. if (!opening && !method.startsWith('profiles.')) {
  168. if (!route && (method === 'approval.respond' || method === 'clarify.respond')) {
  169. // These RPCs use request_id rather than session_id; resolve only a pending interaction seen by this channel.
  170. const match = [...c.routes].map(k => u.routes.get(k)).find(r => r && this.interactions.get(`${u.principal.upstreamKey}:${p.request_id}`) === r.runtime)
  171. route = match
  172. }
  173. if (!route || !c.routes.has(this.routeKey(route.profile, route.stored))) throw new HttpError(403, 'Session is not subscribed', 'route_forbidden')
  174. if (['approval.respond', 'clarify.respond'].includes(method)
  175. && this.interactions.get(`${u.principal.upstreamKey}:${p.request_id}`) !== route.runtime) {
  176. throw new HttpError(403, 'Interaction does not belong to this session', 'interaction_forbidden')
  177. }
  178. }
  179. const routeKey = opening ? this.routeKey(String(p.profile), String(p.session_id ?? c.id))
  180. : route ? this.routeKey(route.profile, route.stored) : 'profiles'
  181. const execute = async () => {
  182. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  183. if (method === 'session.resume') {
  184. const ownerKey = JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, p.profile, p.session_id])
  185. const owner = this.routeOwners.get(ownerKey)
  186. if (owner && owner !== c.principal.upstreamKey) throw new HttpError(409, 'Session is attached through another credential scope', 'session_scope_conflict')
  187. this.routeOwners.set(ownerKey, c.principal.upstreamKey)
  188. }
  189. await this.connect(u)
  190. await u.recovery
  191. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  192. const previouslyActive = route?.active ?? false
  193. if (route && (method === 'prompt.submit' || method === 'session.steer')) route.active = true
  194. const response = await this.rpc(u, method, p, f => c.principal.observeEvent?.(JSON.stringify(f)), f => c.principal.observeCommand?.(JSON.stringify(f)))
  195. if (response.error && route && (method === 'prompt.submit' || method === 'session.steer')) route.active = previouslyActive
  196. if (!response.error && route && (method === 'prompt.submit' || method === 'session.steer')) route.active = true
  197. if ((opening || method === 'session.branch') && response.result) {
  198. const result = response.result as Frame
  199. const info = result.info ?? {}
  200. const runtime = String(result.session_id ?? '')
  201. const stored = String(result.stored_session_id ?? result.session_key ?? info.stored_session_id ?? p.session_id ?? runtime)
  202. if (!runtime || !stored) throw new Error('Invalid upstream session identity')
  203. const profile = String(p.profile ?? route?.profile ?? 'default')
  204. const key = this.routeKey(profile, stored)
  205. this.routeOwners.set(JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, profile, stored]), c.principal.upstreamKey)
  206. const r: Route = u.routes.get(key) ?? { profile, stored, runtime, active: false, seq: 0, observers: new Map() }
  207. u.byRuntime.delete(r.runtime)
  208. r.runtime = runtime
  209. r.active = result.running === true || info.running === true
  210. u.routes.set(key, r); u.byRuntime.set(runtime, r); c.routes.add(key)
  211. r.observers.set(c.principal.key, c.principal)
  212. for (const pending of [result.pending_approval, result.pending_clarify, result.inflight?.pending_approval, result.inflight?.pending_clarify]) {
  213. if (pending?.request_id) this.interactions.set(`${u.principal.upstreamKey}:${pending.request_id}`, runtime)
  214. }
  215. const early = u.early.get(runtime) ?? []
  216. u.early.delete(runtime)
  217. for (const e of early) { u.earlyBytes -= Buffer.byteLength(JSON.stringify(e)); this.event(u, e) }
  218. }
  219. if (method === 'session.close' && route && !response.error) {
  220. u.byRuntime.delete(route.runtime); u.routes.delete(this.routeKey(route.profile, route.stored))
  221. this.routeOwners.delete(JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, route.profile, route.stored]))
  222. }
  223. u.lastUsed = this.now()
  224. return response
  225. }
  226. if (['session.interrupt', 'approval.respond', 'clarify.respond', 'session.steer'].includes(method)) return execute()
  227. const work = (u.tails.get(routeKey) ?? Promise.resolve()).catch(() => {}).then(execute)
  228. u.tails.set(routeKey, work)
  229. void work.finally(() => { if (u.tails.get(routeKey) === work) u.tails.delete(routeKey) }).catch(() => {})
  230. return work
  231. }
  232. private interactions = new Map<string, string>()
  233. private routeOwners = new Map<string, string>()
  234. private routeKey(profile: string, stored: string): string { return JSON.stringify([profile, stored]) }
  235. private upstream(p: RealtimePrincipal): Upstream {
  236. let u = this.upstreams.get(p.upstreamKey)
  237. if (!u) {
  238. if (this.upstreams.size >= 64) throw new HttpError(429, 'Upstream capacity reached', 'upstream_limit')
  239. u = { principal: p, pending: new Map(), routes: new Map(), byRuntime: new Map(), tails: new Map(),
  240. early: new Map(), earlyBytes: 0, attempts: 0, recovering: false, deferred: [], lastUsed: this.now() }
  241. this.upstreams.set(p.upstreamKey, u)
  242. }
  243. return u
  244. }
  245. private connect(u: Upstream): Promise<void> {
  246. if (u.connecting) return u.connecting
  247. if (u.socket?.readyState === WebSocket.OPEN && u.ready) return Promise.resolve()
  248. u.connecting = (async () => {
  249. const url = await u.principal.url('chat')
  250. if (this.closed) throw new Error('Broker closed')
  251. await new Promise<void>((resolve, reject) => {
  252. const ws = new WebSocket(url, { headers: { Origin: `${url.protocol === 'wss:' ? 'https:' : 'http:'}//${url.host}` }, maxPayload: CHAT_MAX_PAYLOAD, handshakeTimeout: 15_000 })
  253. u.socket = ws
  254. const timeout = setTimeout(() => { reject(new Error('Upstream ready timeout')); ws.terminate() }, 20_000)
  255. let ready = false
  256. ws.on('message', (data, binary) => {
  257. if (u.socket !== ws || binary) return
  258. let f: Frame
  259. try { f = JSON.parse(data.toString()) } catch { ws.terminate(); return }
  260. if (f.method === 'event' && f.params?.type === 'gateway.ready') {
  261. clearTimeout(timeout); ready = true; u.ready = f; u.attempts = 0
  262. const epoch = f.params.payload?.replay_epoch
  263. const changed = u.epoch !== undefined && epoch !== u.epoch
  264. u.epoch = epoch
  265. if (u.routes.size) u.recovery = this.recover(u, changed).finally(() => { u.recovery = undefined })
  266. resolve()
  267. } else if (f.id !== undefined) {
  268. const pending = u.pending.get(String(f.id))
  269. if (pending) { clearTimeout(pending.timer); u.pending.delete(String(f.id)); pending.resolve(f) }
  270. } else if (f.method === 'event') this.event(u, f)
  271. })
  272. let alive = true
  273. ws.on('pong', () => { alive = true })
  274. u.ping = setInterval(() => { if (!alive) ws.terminate(); else if (ws.readyState === WebSocket.OPEN) { alive = false; ws.ping() } }, 25_000)
  275. u.ping.unref()
  276. ws.on('error', () => { if (!ready) reject(new Error('Upstream connection failed')) })
  277. ws.on('close', () => {
  278. clearTimeout(timeout); clearInterval(u.ping)
  279. if (u.socket !== ws) return
  280. u.socket = undefined; u.ready = undefined
  281. if (!ready) reject(new Error('Upstream disconnected'))
  282. for (const pending of u.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Upstream disconnected; result unknown')) }
  283. u.pending.clear()
  284. if (!this.closed && this.upstreams.get(u.principal.upstreamKey) === u) {
  285. u.retry = setTimeout(() => { u.retry = undefined; void this.connect(u).catch(() => {}) }, Math.min(15_000, 500 * 2 ** Math.min(u.attempts++, 5)))
  286. u.retry.unref()
  287. }
  288. })
  289. })
  290. })().finally(() => { u.connecting = undefined })
  291. return u.connecting
  292. }
  293. private rpc(u: Upstream, method: string, params: Frame, observed?: (frame: Frame) => void, sent?: (frame: Frame) => void): Promise<Frame> {
  294. const id = randomUUID()
  295. if (u.socket?.readyState !== WebSocket.OPEN || u.socket.bufferedAmount > 8 * 1024 * 1024 || u.pending.size >= 128) throw new Error('Upstream unavailable or congested')
  296. const frame = { jsonrpc: '2.0', id, method, params }
  297. sent?.(frame)
  298. return new Promise((resolve, reject) => {
  299. const timer = setTimeout(() => { u.pending.delete(id); reject(new Error('Upstream command result unknown')) }, 120_000)
  300. timer.unref()
  301. u.pending.set(id, { timer, reject, resolve: response => { observed?.(response); resolve(response) } })
  302. u.socket!.send(JSON.stringify(frame), error => { if (error) { clearTimeout(timer); u.pending.delete(id); reject(error) } })
  303. })
  304. }
  305. private event(u: Upstream, f: Frame, replay = false): void {
  306. if (this.closed) return
  307. if (u.recovering && !replay) {
  308. if (u.deferred.length >= 512 || Buffer.byteLength(JSON.stringify([...u.deferred, f])) > 1024 * 1024) { u.deferred = []; this.reset(u, 'recovery_overflow') }
  309. else u.deferred.push(f)
  310. return
  311. }
  312. const p = f.params
  313. if (!p || typeof p.type !== 'string') return
  314. const sid = String(p.session_id ?? '')
  315. const r = u.byRuntime.get(sid)
  316. if (sid && !r) {
  317. const size = Buffer.byteLength(JSON.stringify(f))
  318. if (u.earlyBytes + size > 1024 * 1024) { u.early.clear(); u.earlyBytes = 0; this.reset(u, 'unmapped_event_overflow'); return }
  319. const list = u.early.get(sid) ?? []; list.push(f); u.early.set(sid, list); u.earlyBytes += size
  320. return
  321. }
  322. if (r) {
  323. if (Number.isSafeInteger(p.seq)) {
  324. if (p.seq <= r.seq) return
  325. if (r.seq > 0 && p.seq > r.seq + 1) this.reset(u, 'upstream_sequence_gap')
  326. r.seq = p.seq
  327. }
  328. if (['message.start', 'approval.request', 'clarify.request'].includes(p.type)) r.active = true
  329. if (p.type === 'message.complete') r.active = false
  330. if (p.payload?.request_id) this.interactions.set(`${u.principal.upstreamKey}:${p.payload.request_id}`, sid)
  331. while (this.interactions.size > 10_000) this.interactions.delete(this.interactions.keys().next().value!)
  332. }
  333. if (r) for (const principal of r.observers.values()) {
  334. if (principal.valid()) principal.observeEvent?.(JSON.stringify(f))
  335. }
  336. for (const c of this.channels.values()) {
  337. if (c.kind !== 'chat' || c.principal.upstreamKey !== u.principal.upstreamKey || !c.principal.valid()) continue
  338. if (r && !c.routes.has(this.routeKey(r.profile, r.stored))) continue
  339. if (!r && !['profiles.changed', 'sessions.changed', 'models.changed', 'pet.changed'].includes(p.type)) continue
  340. this.emit(c, 'frame', f)
  341. }
  342. }
  343. private async recover(u: Upstream, changed: boolean): Promise<void> {
  344. if (u.recovering) return
  345. u.recovering = true
  346. try {
  347. for (const r of u.routes.values()) {
  348. const lastSeen = r.seq
  349. const resumed = await this.rpc(u, 'session.resume', { session_id: r.stored, profile: r.profile, close_on_disconnect: false, omit_messages: true })
  350. if (typeof resumed.result?.running === 'boolean') r.active = resumed.result.running
  351. if (!resumed.result || resumed.result.session_id !== r.runtime || changed) {
  352. if (resumed.result?.session_id) { u.byRuntime.delete(r.runtime); r.runtime = resumed.result.session_id; u.byRuntime.set(r.runtime, r) }
  353. r.seq = 0; this.reset(u, 'upstream_reset'); continue
  354. }
  355. const replay = await this.rpc(u, 'session.events.since', { session_id: r.runtime, last_seen: lastSeen })
  356. if (replay.error || replay.result?.truncated || replay.result?.epoch !== u.epoch) this.reset(u, 'upstream_replay_unavailable')
  357. else for (const p of replay.result?.events ?? []) this.event(u, { jsonrpc: '2.0', method: 'event', params: p }, true)
  358. }
  359. } catch { this.reset(u, 'upstream_recovery_failed') }
  360. finally {
  361. u.recovering = false
  362. const deferred = u.deferred; u.deferred = []
  363. for (const frame of deferred) this.event(u, frame)
  364. }
  365. }
  366. private reset(u: Upstream, reason: string): void {
  367. for (const c of this.channels.values()) if (c.kind === 'chat' && c.principal.upstreamKey === u.principal.upstreamKey) this.emit(c, 'reset', { reason })
  368. }
  369. private async connectGroup(c: RealtimeChannel): Promise<void> {
  370. const url = await c.principal.url('groups', c.group)
  371. const validator = new GroupFrameValidator(c.group!.epoch, c.group!.cursor)
  372. await new Promise<void>((resolve, reject) => {
  373. const ws = new WebSocket(url, { headers: { Origin: `${url.protocol === 'wss:' ? 'https:' : 'http:'}//${url.host}` }, maxPayload: 2 * 1024 * 1024, handshakeTimeout: 15_000 })
  374. c.socket = ws
  375. const timer = setTimeout(() => { ws.terminate(); reject(new Error('Group ready timeout')) }, 20_000)
  376. ws.on('message', (data, binary) => {
  377. try {
  378. if (!validator.accept(data, binary)) return
  379. const frame = JSON.parse(data.toString()) as Frame
  380. if (frame.type === 'group.ready') { clearTimeout(timer); resolve() }
  381. if (frame.type === 'group.event') c.group!.cursor = frame.cursor
  382. this.emit(c, 'frame', frame)
  383. } catch { this.emit(c, 'reset', { reason: 'group_gap' }); ws.close() }
  384. })
  385. ws.on('error', () => { clearTimeout(timer); reject(new Error('Group connection failed')) })
  386. ws.on('close', () => { clearTimeout(timer); reject(new Error('Group disconnected')); if (this.channels.has(c.id)) this.emit(c, 'reset', { reason: 'group_disconnected' }) })
  387. })
  388. }
  389. private emit(c: RealtimeChannel, event: StreamEntry['event'], value: Frame): void {
  390. const data = JSON.stringify(value)
  391. const entry: StreamEntry = { id: `${this.epoch}:${++c.seq}`, event, data, at: this.now(), bytes: Buffer.byteLength(data) }
  392. c.entries.push(entry); c.bytes += entry.bytes
  393. this.trim(c)
  394. for (const listener of [...c.listeners]) {
  395. try { listener(entry) } catch { c.listeners.delete(listener) }
  396. }
  397. }
  398. private trim(c: RealtimeChannel): void {
  399. const own = [...this.channels.values()].filter(v => v.principal.key === c.principal.key)
  400. while (own.reduce((n, v) => n + v.bytes, 0) > 8 * 1024 * 1024 || own.reduce((n, v) => n + v.entries.length, 0) > 10_000) {
  401. const oldest = own.filter(v => v.entries.length).sort((a, b) => a.entries[0]!.at - b.entries[0]!.at)[0]
  402. if (!oldest) break
  403. this.evict(oldest)
  404. }
  405. while (c.entries[0] && c.entries[0].at < this.now() - 600_000) this.evict(c)
  406. while ([...this.channels.values()].reduce((n, v) => n + v.bytes, 0) > 128 * 1024 * 1024) {
  407. const oldest = [...this.channels.values()].filter(v => v.entries.length).sort((a, b) => a.entries[0]!.at - b.entries[0]!.at)[0]
  408. if (!oldest) break
  409. this.evict(oldest)
  410. }
  411. }
  412. private evict(c: RealtimeChannel): void {
  413. const entry = c.entries.shift()!
  414. c.bytes -= entry.bytes; c.droppedThrough = Number(entry.id.split(':').at(-1))
  415. }
  416. private sweep(): void {
  417. for (const [key, item] of this.completed) if (item.at < this.now() - 600_000) this.completed.delete(key)
  418. for (const c of this.channels.values()) {
  419. if (!c.principal.valid() || (!c.listeners.size && this.now() - c.touched > 600_000)) this.remove(c)
  420. else this.trim(c)
  421. }
  422. for (const [key, u] of this.upstreams) {
  423. if (u.pending.size || u.tails.size || [...u.routes.values()].some(r => r.active)) { u.lastUsed = this.now(); continue }
  424. if ([...this.channels.values()].some(c => c.principal.upstreamKey === key)) { u.lastUsed = this.now(); continue }
  425. if (this.now() - u.lastUsed > 300_000) {
  426. this.upstreams.delete(key); clearTimeout(u.retry); clearInterval(u.ping); u.socket?.close()
  427. for (const [route, owner] of this.routeOwners) if (owner === key) this.routeOwners.delete(route)
  428. }
  429. }
  430. }
  431. close(): void {
  432. if (this.closed) return
  433. this.closed = true; clearInterval(this.timer)
  434. for (const c of [...this.channels.values()]) this.remove(c)
  435. for (const u of this.upstreams.values()) {
  436. clearTimeout(u.retry); clearInterval(u.ping)
  437. for (const p of u.pending.values()) { clearTimeout(p.timer); p.reject(new Error('Broker stopped')) }
  438. u.pending.clear(); u.socket?.terminate()
  439. }
  440. // Pending command continuations may still update their receipts in the next microtask.
  441. void Promise.allSettled(this.commands.values()).then(() => this.receipts.close())
  442. }
  443. }