realtimeBroker.ts 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662
  1. import { randomUUID } from 'node:crypto'
  2. import WebSocket from 'ws'
  3. import { HttpError } from './errors.js'
  4. import { checkedChatFrame, CHAT_MAX_PAYLOAD, GroupFrameValidator } from './realtimeProtocol.js'
  5. import { RealtimeReceipts, type CommandReceipt } from './realtimeReceipts.js'
  6. import { applyWorkingDirectory, type AppliedWorkingDirectory } from './sessionWorkingDirectory.js'
  7. type Frame = Record<string, any>
  8. export interface RealtimeActivity { kind: 'command' | 'event' | 'group' | 'reset'; name: string; sessionId?: string; roomId?: string }
  9. export interface RealtimePrincipal {
  10. key: string
  11. upstreamKey: string
  12. instanceKey?: string
  13. paired: boolean
  14. nativeBot?: boolean
  15. agent?: WebSocket.ClientOptions['agent']
  16. valid(): boolean
  17. authorize?(kind: 'chat' | 'groups', method?: string): void
  18. canResume?(profile: string, sessionId: string): Promise<boolean>
  19. configuredWorkingDirectory?(profile: string): Promise<string | undefined>
  20. url(channel: 'chat' | 'groups', anchor?: { epoch: string; cursor: number }): Promise<URL>
  21. observeCommand?(frame: string): void
  22. observeEvent?(frame: string): void
  23. }
  24. export interface StreamEntry { id: string; event: 'frame' | 'reset'; data: string; at: number; bytes: number }
  25. export interface RealtimeChannel {
  26. id: string; kind: 'chat' | 'groups'; principal: RealtimePrincipal
  27. seq: number; droppedThrough: number; entries: StreamEntry[]; routes: Set<string>
  28. listeners: Set<(entry: StreamEntry) => void>; touched: number; bytes: number
  29. group?: { epoch: string; cursor: number }; socket?: WebSocket
  30. }
  31. interface Route { profile: string; stored: string; runtime: string; active: boolean; seq: number; observers: Map<string, RealtimePrincipal>; cwd?: string; workingDirectory?: AppliedWorkingDirectory }
  32. interface Pending { resolve(frame: Frame): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout> }
  33. interface Upstream {
  34. principal: RealtimePrincipal; socket?: WebSocket; connecting?: Promise<void>; ready?: Frame
  35. pending: Map<string, Pending>; routes: Map<string, Route>; byRuntime: Map<string, Route>
  36. tails: Map<string, Promise<unknown>>; early: Map<string, Frame[]>; earlyBytes: number
  37. retry?: ReturnType<typeof setTimeout>; attempts: number; epoch?: string; recovering: boolean
  38. lastUsed: number; ping?: ReturnType<typeof setInterval>
  39. deferred: Frame[]
  40. recovery?: Promise<void>
  41. }
  42. /** Server-owned WS transports. Downstream detach never tears down an active upstream. */
  43. export class RealtimeBroker {
  44. get idleForUpdate(): boolean {
  45. return this.commands.size === 0 && [...this.upstreams.values()].every(upstream => upstream.pending.size === 0 && [...upstream.routes.values()].every(route => !route.active))
  46. }
  47. protectedSession: (id: string) => boolean = () => false
  48. assertCanCommand:()=>void=()=>{}
  49. onNativeEvent: (owner: string, profile: string, storedId: string, frame: Frame) => void = () => {}
  50. onNativeGlobalEvent: (owner: string, type: string, upstreamInvalidation?: boolean) => void = () => {}
  51. onNativeCommand: (owner: string, profile: string, storedId: string, method: string, params: Frame) => void = () => {}
  52. onNativeRoute: (owner: string, profile: string, storedId: string, runtimeId: string, running?: boolean, snapshot?: Frame) => void = () => {}
  53. nativeSourceCursor: (owner:string,profile:string,storedId:string,runtimeId:string,epoch:string)=>number = () => 0
  54. readonly epoch = randomUUID()
  55. readonly channels = new Map<string, RealtimeChannel>()
  56. private upstreams = new Map<string, Upstream>()
  57. private commands = new Map<string, Promise<CommandReceipt>>()
  58. private completed = new Map<string, { receipt: CommandReceipt; at: number; bytes: number }>()
  59. private receipts: RealtimeReceipts
  60. private timer: ReturnType<typeof setInterval>
  61. private closed = false
  62. constructor(home: string, readonly now = Date.now, readonly onActivity: (activity: RealtimeActivity) => void = () => {}) {
  63. this.receipts = new RealtimeReceipts(home)
  64. this.timer = setInterval(() => this.sweep(), 5_000)
  65. this.timer.unref()
  66. }
  67. async create(principal: RealtimePrincipal, kind: 'chat' | 'groups', group?: { epoch: string; cursor: number }): Promise<RealtimeChannel> {
  68. if (String(kind) === 'groups') throw new HttpError(410, '请使用新版聊天事件接口', 'retired_group_stream')
  69. if (this.closed) throw new HttpError(503, 'Realtime broker is stopping', 'broker_stopping')
  70. principal.authorize?.(kind)
  71. if (!principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  72. // Closed pages leave resumable channels behind. They must not consume the live-stream
  73. // quota; evict oldest detached handles under pressure, never a running upstream.
  74. let own = [...this.channels.values()].filter(c => c.principal.key === principal.key)
  75. while (own.length >= 32) {
  76. const stale = own.filter(c => !c.listeners.size).sort((a, b) => a.touched - b.touched)[0]
  77. if (!stale) break
  78. this.remove(stale); own = own.filter(c => c !== stale)
  79. }
  80. if (own.length >= 32 || this.channels.size >= 512) {
  81. throw new HttpError(429, 'Too many realtime channels', 'channel_limit')
  82. }
  83. const c: RealtimeChannel = { id: randomUUID(), kind, principal, seq: 0, droppedThrough: 0,
  84. entries: [], routes: new Set(), listeners: new Set(), touched: this.now(), bytes: 0, group }
  85. this.channels.set(c.id, c)
  86. try {
  87. {
  88. const u = this.upstream(principal)
  89. await this.connect(u)
  90. this.emit(c, 'frame', u.ready!)
  91. }
  92. return c
  93. } catch (e) { this.remove(c); throw e }
  94. }
  95. get(principal: RealtimePrincipal, id: string): RealtimeChannel {
  96. const c = this.channels.get(id)
  97. if (!c || c.principal.key !== principal.key) throw new HttpError(404, 'Channel not found', 'channel_not_found')
  98. if (!principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  99. c.touched = this.now()
  100. return c
  101. }
  102. detach(principal: RealtimePrincipal, id: string): void { this.remove(this.get(principal, id)) }
  103. release(c: RealtimeChannel): void { if (this.channels.get(c.id) === c) this.remove(c) }
  104. private remove(c: RealtimeChannel): void {
  105. this.channels.delete(c.id)
  106. c.socket?.close()
  107. for (const listener of c.listeners) {
  108. try { listener({ id: '', event: 'reset', data: '{"reason":"channel_closed"}', at: this.now(), bytes: 0 }) } catch { /* a closed consumer must not affect the upstream */ }
  109. }
  110. c.listeners.clear()
  111. }
  112. subscribe(c: RealtimeChannel, cursor: string | undefined, listener: (entry: StreamEntry) => void): () => void {
  113. if ([...this.channels.values()].filter(v => v.principal.key === c.principal.key).reduce((n, v) => n + v.listeners.size, 0) >= 8) {
  114. throw new HttpError(429, 'Too many active event streams', 'stream_limit')
  115. }
  116. this.trim(c)
  117. let seq = 0
  118. if (cursor) {
  119. const prefix = `${this.epoch}:`
  120. if (!cursor.startsWith(prefix) || !/^\d+$/.test(cursor.slice(prefix.length))) {
  121. throw new HttpError(409, 'Stream epoch changed', 'reset_required')
  122. }
  123. seq = Number(cursor.slice(prefix.length))
  124. }
  125. if (!Number.isSafeInteger(seq) || seq < c.droppedThrough || seq > c.seq) {
  126. throw new HttpError(409, 'Replay window expired', 'reset_required')
  127. }
  128. c.listeners.add(listener)
  129. for (const entry of c.entries) if (Number(entry.id.split(':').at(-1)) > seq) listener(entry)
  130. return () => { c.listeners.delete(listener); c.touched = this.now() }
  131. }
  132. receipt(p: RealtimePrincipal, id: string): CommandReceipt {
  133. if (!p.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  134. const receipt = this.completed.get(`${p.key}:${id}`)?.receipt ?? this.receipts.lookup(p.key, id)
  135. if (!receipt) throw new HttpError(404, 'Receipt not found; do not automatically resubmit an attempted command', 'receipt_not_found')
  136. return receipt
  137. }
  138. command(c: RealtimeChannel, requestId: string, input: Frame): Promise<CommandReceipt> {
  139. if (this.closed) throw new HttpError(503, 'Realtime broker is stopping', 'broker_stopping')
  140. this.assertCanCommand()
  141. if (c.kind !== 'chat') throw new HttpError(403, 'Group event channel is read-only', 'read_only')
  142. const normalized = checkedChatFrame(Buffer.from(JSON.stringify(input)), false, c.principal.paired)
  143. const frame = JSON.parse(normalized) as Frame
  144. if (Object.values(frame.params?.preferred_session_ids ?? {}).some(id => typeof id === 'string' && this.protectedSession(id))) {
  145. throw new HttpError(404, '会话不存在', 'session_not_found')
  146. }
  147. c.principal.authorize?.('chat', String(frame.method))
  148. delete frame.id
  149. const fingerprint = JSON.stringify(frame)
  150. if (!this.receipts.lookup(c.principal.key, requestId) && (this.commands.size >= 1024
  151. || [...this.commands.keys()].filter(k => k.startsWith(`${c.principal.key}:`)).length >= 32)) {
  152. throw new HttpError(429, 'Too many pending commands', 'command_limit')
  153. }
  154. const existing = this.receipts.reserve(c.principal.key, requestId, fingerprint)
  155. const key = `${c.principal.key}:${requestId}`
  156. if (existing) return this.commands.get(key) ?? Promise.resolve(this.completed.get(key)?.receipt ?? existing)
  157. const work = this.perform(c, frame, requestId).then(response => {
  158. const state = response.error ? 'rejected' : 'confirmed'
  159. this.receipts.finish(c.principal.key, requestId, state, response)
  160. return { requestId, state, response } as CommandReceipt
  161. }, (error: unknown) => {
  162. if (error instanceof HttpError) {
  163. const response = { error: { code: error.code, message: error.message } }
  164. this.receipts.finish(c.principal.key, requestId, 'rejected', response)
  165. return { requestId, state: 'rejected', response } as CommandReceipt
  166. }
  167. this.receipts.finish(c.principal.key, requestId, 'unknown')
  168. return { requestId, state: 'unknown' } as CommandReceipt
  169. })
  170. this.commands.set(key, work)
  171. void work.then(receipt => {
  172. this.commands.delete(key)
  173. this.completed.set(key, { receipt, at: this.now(), bytes: Buffer.byteLength(JSON.stringify(receipt)) })
  174. while (this.completed.size > 10_000 || [...this.completed.values()].reduce((n, x) => n + x.bytes, 0) > 32 * 1024 * 1024) this.completed.delete(this.completed.keys().next().value!)
  175. })
  176. return work
  177. }
  178. private async perform(c: RealtimeChannel, frame: Frame, deliveryID: string): Promise<Frame> {
  179. const u = this.upstream(c.principal)
  180. const p = frame.params as Frame
  181. if (p.session_id && this.protectedSession(String(p.session_id))) throw new HttpError(404, '会话不存在', 'session_not_found')
  182. const method = String(frame.method)
  183. const opening = method === 'session.create' || method === 'session.resume'
  184. if (opening && (c.routes.size >= 128 || u.routes.size >= 512)) throw new HttpError(429, 'Route capacity reached', 'route_limit')
  185. let route = p.session_id ? u.byRuntime.get(p.session_id) : undefined
  186. if (!opening && !method.startsWith('profiles.')) {
  187. if (!route && (method === 'approval.respond' || method === 'clarify.respond')) {
  188. // These RPCs use request_id rather than session_id; resolve only a pending interaction seen by this channel.
  189. const match = [...c.routes].map(k => u.routes.get(k)).find(r => r && this.interactions.get(`${u.principal.upstreamKey}:${p.request_id}`) === r.runtime)
  190. route = match
  191. }
  192. if (!route || !c.routes.has(this.routeKey(route.profile, route.stored))) throw new HttpError(403, 'Session is not subscribed', 'route_forbidden')
  193. if (['approval.respond', 'clarify.respond'].includes(method)
  194. && this.interactions.get(`${u.principal.upstreamKey}:${p.request_id}`) !== route.runtime) {
  195. throw new HttpError(403, 'Interaction does not belong to this session', 'interaction_forbidden')
  196. }
  197. }
  198. const routeKey = opening ? this.routeKey(String(p.profile), String(p.session_id ?? c.id))
  199. : route ? this.routeKey(route.profile, route.stored) : 'profiles'
  200. const execute = async () => {
  201. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  202. if (method === 'session.resume') {
  203. let permitted = !c.principal.canResume || await c.principal.canResume(String(p.profile), String(p.session_id))
  204. if (c.principal.nativeBot) {
  205. await this.connect(u)
  206. const roster = await this.rpc(u, 'profiles.list', {include_sessions: false})
  207. const profiles = roster.result?.profiles
  208. const bot = Array.isArray(profiles) ? profiles.find((value: Frame) => value.name === p.profile) : undefined
  209. permitted = !roster.error && typeof p.session_id === 'string' && p.session_id.length > 0 &&
  210. bot?.ui_meta?.['hermes-bots']?.chat === p.session_id
  211. }
  212. if (!permitted) {
  213. throw new HttpError(403, '此会话属于历史记录,不能继续聊天', 'history_session_read_only')
  214. }
  215. const ownerKey = JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, p.profile, p.session_id])
  216. const owner = this.routeOwners.get(ownerKey)
  217. if (owner && owner !== c.principal.upstreamKey) throw new HttpError(409, 'Session is attached through another credential scope', 'session_scope_conflict')
  218. this.routeOwners.set(ownerKey, c.principal.upstreamKey)
  219. }
  220. await this.connect(u)
  221. await u.recovery
  222. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  223. if (!c.principal.nativeBot && c.principal.configuredWorkingDirectory) {
  224. if (method === 'session.create') {
  225. const cwd = await c.principal.configuredWorkingDirectory(String(p.profile))
  226. // Client cwd is never authoritative, including paired mobile clients.
  227. delete p.cwd
  228. if (cwd) p.cwd = cwd
  229. } else if (route && !route.active && ['prompt.submit', 'file.attach', 'image.attach_bytes', 'pdf.attach'].includes(method)) {
  230. const cwd = await c.principal.configuredWorkingDirectory(route.profile)
  231. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  232. route.workingDirectory = await applyWorkingDirectory(async (method, params) => {
  233. const response = await this.rpc(u, method, params)
  234. if (response.error) throw new HttpError(502, response.error.message || '无法切换到机器人工作目录', 'working_directory_rejected')
  235. return response.result
  236. }, route.runtime, cwd, route.workingDirectory, route.cwd)
  237. if (route.workingDirectory) route.cwd = route.workingDirectory.resolved
  238. }
  239. if (!c.principal.valid()) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  240. }
  241. const submittedAt = this.now()/1000
  242. const previouslyActive = route?.active ?? false
  243. if (route && (method === 'prompt.submit' || method === 'session.steer')) route.active = true
  244. const nativeOwner = this.nativeOwner(c.principal)
  245. if (nativeOwner && route && ['prompt.submit', 'session.steer'].includes(method)) {
  246. this.onNativeCommand(nativeOwner, route.profile, route.stored, method, {...p,_delivery_id:deliveryID})
  247. const peer={jsonrpc:'2.0',method:'event',params:{type:'run.peer_user_message',session_id:route.runtime,profile:route.profile,
  248. payload:{message_id:`user:${deliveryID}`,queue_id:deliveryID.replace(/^(web|ios):prompt:/,''),client_message_id:deliveryID.replace(/^(web|ios):prompt:/,''),text:p.text??'',timestamp:submittedAt}}}
  249. for(const channel of this.channels.values())if(channel.routes.has(this.routeKey(route.profile,route.stored))&&channel.principal.valid()&&this.nativeOwner(channel.principal)===nativeOwner)this.emit(channel,'frame',peer)
  250. }
  251. const activity: RealtimeActivity = { kind: 'command', name: method, sessionId: route?.stored }
  252. this.onActivity(activity)
  253. const response = await this.rpc(u, method, p, f => c.principal.observeEvent?.(JSON.stringify(f)), f => c.principal.observeCommand?.(JSON.stringify(f)))
  254. .finally(() => this.onActivity(activity))
  255. if (nativeOwner && route && ['prompt.submit','session.steer'].includes(method)) {
  256. this.onNativeEvent(nativeOwner,route.profile,route.stored,{type:response.error?'command.rejected':'command.confirmed',
  257. delivery_id:`command-result:${deliveryID}`,payload:{delivery_id:deliveryID,error:response.error?.message,status:response.result?.status}})
  258. const peer={jsonrpc:'2.0',method:'event',params:{type:'run.peer_user_message',session_id:route.runtime,profile:route.profile,
  259. payload:{message_id:`user:${deliveryID}`,queue_id:deliveryID.replace(/^(web|ios):prompt:/,''),client_message_id:deliveryID.replace(/^(web|ios):prompt:/,''),text:p.text??'',status:response.error?'failed':'accepted',timestamp:submittedAt}}}
  260. for(const channel of this.channels.values())if(channel.routes.has(this.routeKey(route.profile,route.stored))&&channel.principal.valid()&&this.nativeOwner(channel.principal)===nativeOwner)this.emit(channel,'frame',peer)
  261. }
  262. if (response.error && route && (method === 'prompt.submit' || method === 'session.steer')) route.active = previouslyActive
  263. if (!response.error && route && (method === 'prompt.submit' || method === 'session.steer')) route.active = true
  264. if ((opening || method === 'session.branch') && response.result) {
  265. const result = response.result as Frame
  266. const info = result.info ?? {}
  267. const runtime = String(result.session_id ?? '')
  268. const stored = String(result.stored_session_id ?? result.session_key ?? info.stored_session_id ?? p.session_id ?? runtime)
  269. if (!runtime || !stored) throw new Error('Invalid upstream session identity')
  270. const profile = String(p.profile ?? route?.profile ?? 'default')
  271. const key = this.routeKey(profile, stored)
  272. this.routeOwners.set(JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, profile, stored]), c.principal.upstreamKey)
  273. const r: Route = u.routes.get(key) ?? { profile, stored, runtime, active: false, seq: 0, observers: new Map() }
  274. u.byRuntime.delete(r.runtime)
  275. r.runtime = runtime
  276. r.active = result.running === true || info.running === true
  277. // Resume/reconnect may have restored a historical cwd or another runtime.
  278. r.workingDirectory = undefined
  279. r.cwd = typeof info.cwd === 'string' ? info.cwd : undefined
  280. u.routes.set(key, r); u.byRuntime.set(runtime, r); c.routes.add(key)
  281. r.observers.set(c.principal.key, c.principal)
  282. if (nativeOwner) {
  283. r.seq=Math.max(r.seq,this.nativeSourceCursor(nativeOwner,profile,stored,runtime,u.epoch??''))
  284. this.onNativeRoute(nativeOwner, profile, stored, runtime, r.active, result)
  285. }
  286. if (nativeOwner && (method === 'session.create' || method === 'session.branch')) {
  287. this.onNativeCommand(nativeOwner, profile, stored, method, {...p,_delivery_id:deliveryID})
  288. }
  289. for (const pending of [result.pending_approval, result.pending_clarify, result.inflight?.pending_approval, result.inflight?.pending_clarify]) {
  290. if (pending?.request_id) this.interactions.set(`${u.principal.upstreamKey}:${pending.request_id}`, runtime)
  291. }
  292. const early = u.early.get(runtime) ?? []
  293. u.early.delete(runtime)
  294. for (const e of early) { u.earlyBytes -= Buffer.byteLength(JSON.stringify(e)); this.event(u, e) }
  295. }
  296. if (method === 'session.close' && route && !response.error) {
  297. u.byRuntime.delete(route.runtime); u.routes.delete(this.routeKey(route.profile, route.stored))
  298. this.routeOwners.delete(JSON.stringify([c.principal.instanceKey ?? c.principal.upstreamKey, route.profile, route.stored]))
  299. }
  300. if (route && response.result && ['file.attach','image.attach_bytes','pdf.attach'].includes(method) && c.principal.key.startsWith('user:')) {
  301. this.onNativeEvent(c.principal.key.split(':')[1]!, route.profile, route.stored, { type:'attachment.staged', payload:response.result })
  302. }
  303. u.lastUsed = this.now()
  304. return this.withoutProtectedSessions(response) ?? { result: {} }
  305. }
  306. if (['session.interrupt', 'approval.respond', 'clarify.respond', 'session.steer'].includes(method)) return execute()
  307. const work = (u.tails.get(routeKey) ?? Promise.resolve()).catch(() => {}).then(execute)
  308. u.tails.set(routeKey, work)
  309. void work.finally(() => { if (u.tails.get(routeKey) === work) u.tails.delete(routeKey) }).catch(() => {})
  310. return work
  311. }
  312. private interactions = new Map<string, string>()
  313. private routeOwners = new Map<string, string>()
  314. private routeKey(profile: string, stored: string): string { return JSON.stringify([profile, stored]) }
  315. private upstream(p: RealtimePrincipal): Upstream {
  316. let u = this.upstreams.get(p.upstreamKey)
  317. if (!u) {
  318. if (this.upstreams.size >= 64) throw new HttpError(429, 'Upstream capacity reached', 'upstream_limit')
  319. u = { principal: p, pending: new Map(), routes: new Map(), byRuntime: new Map(), tails: new Map(),
  320. early: new Map(), earlyBytes: 0, attempts: 0, recovering: false, deferred: [], lastUsed: this.now() }
  321. this.upstreams.set(p.upstreamKey, u)
  322. }
  323. return u
  324. }
  325. private authorized(p: RealtimePrincipal): boolean {
  326. try { if (!p.valid()) return false; p.authorize?.('chat'); return true } catch { return false }
  327. }
  328. private connectionPrincipal(u: Upstream): RealtimePrincipal | undefined {
  329. if (this.authorized(u.principal)) return u.principal
  330. // A service transport may outlive its first browser session and serve other users.
  331. const candidates = [
  332. ...[...this.channels.values()].filter(c => c.principal.upstreamKey === u.principal.upstreamKey).map(c => c.principal),
  333. ...[...u.routes.values()].flatMap(r => [...r.observers.values()]),
  334. ]
  335. const principal = candidates.find(p => this.authorized(p))
  336. if (principal) u.principal = principal
  337. return principal
  338. }
  339. private scheduleReconnect(u: Upstream): void {
  340. if (this.closed || this.upstreams.get(u.principal.upstreamKey) !== u || u.retry || !this.connectionPrincipal(u)) return
  341. u.retry = setTimeout(() => {
  342. u.retry = undefined
  343. void this.connect(u).catch(() => {})
  344. }, Math.min(15_000, 500 * 2 ** Math.min(u.attempts++, 5)))
  345. u.retry.unref()
  346. }
  347. private connect(u: Upstream): Promise<void> {
  348. if (this.closed || this.upstreams.get(u.principal.upstreamKey) !== u) return Promise.reject(new Error('Broker transport closed'))
  349. if (u.connecting) return u.connecting
  350. if (u.socket?.readyState === WebSocket.OPEN && u.ready) return Promise.resolve()
  351. clearTimeout(u.retry); u.retry = undefined
  352. u.connecting = (async () => {
  353. // Let the previous socket's recovery unwind before starting another generation.
  354. await u.recovery?.catch(() => {})
  355. const principal = this.connectionPrincipal(u)
  356. if (!principal) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  357. const url = await principal.url('chat')
  358. if (this.closed || this.upstreams.get(principal.upstreamKey) !== u) throw new Error('Broker transport closed')
  359. if (!this.authorized(principal)) throw new HttpError(401, 'Authentication expired', 'authentication_required')
  360. await new Promise<void>((resolve, reject) => {
  361. const ws = new WebSocket(url, { agent: principal.agent, headers: { Origin: `${url.protocol === 'wss:' ? 'https:' : 'http:'}//${url.host}` }, maxPayload: CHAT_MAX_PAYLOAD, handshakeTimeout: 15_000 })
  362. u.socket = ws
  363. const timeout = setTimeout(() => { reject(new Error('Upstream ready timeout')); ws.terminate() }, 20_000)
  364. let ready = false
  365. ws.on('message', (data, binary) => {
  366. if (u.socket !== ws || binary) return
  367. let f: Frame
  368. try { f = JSON.parse(data.toString()) } catch { ws.terminate(); return }
  369. if (f.method === 'event' && f.params?.type === 'gateway.ready') {
  370. if (ready) return
  371. clearTimeout(timeout); ready = true; u.ready = f
  372. const epoch = f.params.payload?.replay_epoch
  373. const changed = u.epoch !== undefined && epoch !== u.epoch
  374. u.epoch = epoch
  375. const recovery = u.routes.size ? this.recover(u, changed) : Promise.resolve()
  376. u.recovery = recovery
  377. void recovery.then(() => {
  378. if (this.closed || u.socket !== ws || ws.readyState !== WebSocket.OPEN) throw new Error('Upstream disconnected during recovery')
  379. u.attempts = 0
  380. resolve()
  381. }).catch(error => { reject(error); ws.terminate() }).finally(() => {
  382. if (u.recovery === recovery) u.recovery = undefined
  383. })
  384. } else if (f.id !== undefined) {
  385. const pending = u.pending.get(String(f.id))
  386. if (pending) { clearTimeout(pending.timer); u.pending.delete(String(f.id)); pending.resolve(f) }
  387. } else if (f.method === 'event') this.event(u, f)
  388. })
  389. let alive = true
  390. ws.on('pong', () => { alive = true })
  391. const ping = setInterval(() => { if (!alive) ws.terminate(); else if (ws.readyState === WebSocket.OPEN) { alive = false; ws.ping() } }, 25_000)
  392. u.ping = ping; ping.unref()
  393. ws.on('error', () => { if (!ready) reject(new Error('Upstream connection failed')) })
  394. ws.on('close', () => {
  395. clearTimeout(timeout); clearInterval(ping)
  396. if (u.socket !== ws) return
  397. u.socket = undefined; u.ready = undefined
  398. reject(new Error('Upstream disconnected'))
  399. for (const pending of u.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Upstream disconnected; result unknown')) }
  400. u.pending.clear()
  401. this.scheduleReconnect(u)
  402. })
  403. })
  404. })().catch(error => {
  405. // Credentials and readiness can fail before any WebSocket close callback exists.
  406. this.scheduleReconnect(u)
  407. throw error
  408. }).finally(() => { u.connecting = undefined })
  409. return u.connecting
  410. }
  411. private rpc(u: Upstream, method: string, params: Frame, observed?: (frame: Frame) => void, sent?: (frame: Frame) => void): Promise<Frame> {
  412. const id = randomUUID()
  413. if (u.socket?.readyState !== WebSocket.OPEN || u.socket.bufferedAmount > 8 * 1024 * 1024 || u.pending.size >= 128) throw new Error('Upstream unavailable or congested')
  414. const frame = { jsonrpc: '2.0', id, method, params }
  415. sent?.(frame)
  416. return new Promise((resolve, reject) => {
  417. const timer = setTimeout(() => { u.pending.delete(id); reject(new Error('Upstream command result unknown')) }, 120_000)
  418. timer.unref()
  419. u.pending.set(id, { timer, reject, resolve: response => { observed?.(response); resolve(response) } })
  420. u.socket!.send(JSON.stringify(frame), error => { if (error) { clearTimeout(timer); u.pending.delete(id); reject(error) } })
  421. })
  422. }
  423. private event(u: Upstream, f: Frame, replay = false): void {
  424. if (this.closed) return
  425. if (u.recovering && !replay) {
  426. if (u.deferred.length >= 512 || Buffer.byteLength(JSON.stringify([...u.deferred, f])) > 1024 * 1024) { u.deferred = []; this.reset(u, 'recovery_overflow') }
  427. else u.deferred.push(f)
  428. return
  429. }
  430. const p = f.params
  431. if (!p || typeof p.type !== 'string') return
  432. const sid = String(p.session_id ?? '')
  433. const r = u.byRuntime.get(sid)
  434. this.onActivity({ kind: 'event', name: p.type, sessionId: r?.stored })
  435. if (sid && !r) {
  436. const size = Buffer.byteLength(JSON.stringify(f))
  437. if (u.earlyBytes + size > 1024 * 1024) { u.early.clear(); u.earlyBytes = 0; this.reset(u, 'unmapped_event_overflow'); return }
  438. const list = u.early.get(sid) ?? []; list.push(f); u.early.set(sid, list); u.earlyBytes += size
  439. return
  440. }
  441. if (r) {
  442. if (Number.isSafeInteger(p.seq)) {
  443. if (p.seq <= r.seq) return
  444. if (r.seq > 0 && p.seq > r.seq + 1) this.reset(u, 'upstream_sequence_gap')
  445. r.seq = p.seq
  446. }
  447. if(p.type==='session.info'){
  448. const stored=p.payload?.stored_session_id??p.payload?.session_key??p.payload?.info?.stored_session_id
  449. if(typeof stored==='string'&&stored&&stored!==r.stored){
  450. const oldKey=this.routeKey(r.profile,r.stored),newKey=this.routeKey(r.profile,stored)
  451. if(!u.routes.has(newKey)){
  452. this.routeOwners.delete(JSON.stringify([u.principal.instanceKey??u.principal.upstreamKey,r.profile,r.stored]));
  453. this.routeOwners.set(JSON.stringify([u.principal.instanceKey??u.principal.upstreamKey,r.profile,stored]),u.principal.upstreamKey);
  454. u.routes.delete(oldKey);r.stored=stored;u.routes.set(newKey,r)
  455. for(const channel of this.channels.values())if(channel.principal.upstreamKey===u.principal.upstreamKey&&channel.routes.delete(oldKey))channel.routes.add(newKey)
  456. for(const principal of r.observers.values()){
  457. const owner=this.nativeOwner(principal)
  458. if(owner&&principal.valid())this.onNativeRoute(owner,r.profile,r.stored,r.runtime,r.active)
  459. }
  460. }
  461. }
  462. }
  463. if (p.type === 'session.info' && typeof p.payload?.cwd === 'string') {
  464. r.cwd = p.payload.cwd
  465. if (r.cwd !== r.workingDirectory?.resolved) r.workingDirectory = undefined
  466. }
  467. if (['message.start', 'approval.request', 'clarify.request'].includes(p.type)) r.active = true
  468. if (p.type === 'message.complete') r.active = false
  469. if (p.payload?.request_id) this.interactions.set(`${u.principal.upstreamKey}:${p.payload.request_id}`, sid)
  470. while (this.interactions.size > 10_000) this.interactions.delete(this.interactions.keys().next().value!)
  471. }
  472. const deliveryID = Number.isSafeInteger(p.seq) ? `${u.epoch??this.epoch}:${sid}:${p.seq}` : randomUUID()
  473. const persistedOwners = new Set<string>()
  474. if (r) for (const principal of r.observers.values()) {
  475. if (principal.valid()) {
  476. principal.observeEvent?.(JSON.stringify(f))
  477. const owner = this.nativeOwner(principal)
  478. if (owner && !persistedOwners.has(owner)) { persistedOwners.add(owner); this.onNativeEvent(owner, r.profile, r.stored, {...p,delivery_id:deliveryID,epoch:u.epoch??this.epoch}) }
  479. }
  480. }
  481. if (!r && p.type === 'sessions.changed') {
  482. const owners = new Set([...this.channels.values()].flatMap(channel => {
  483. const owner = this.nativeOwner(channel.principal)
  484. return owner && channel.principal.valid() ? [owner] : []
  485. }))
  486. for (const owner of owners) this.onNativeGlobalEvent(owner, p.type, true)
  487. }
  488. for (const c of this.channels.values()) {
  489. if (c.kind !== 'chat' || c.principal.upstreamKey !== u.principal.upstreamKey || !c.principal.valid()) continue
  490. if (r && !c.routes.has(this.routeKey(r.profile, r.stored))) continue
  491. if (!r && !['profiles.changed', 'sessions.changed', 'models.changed', 'pet.changed'].includes(p.type)) continue
  492. this.emit(c, 'frame', f)
  493. }
  494. if (r && (p.type === 'session.title' || p.type === 'session.title.updated')) {
  495. const owners = new Set([...r.observers.values()].flatMap(principal => {
  496. const owner = principal.valid() ? this.nativeOwner(principal) : undefined
  497. return owner ? [owner] : []
  498. }))
  499. for (const owner of owners) this.publishOwnerSessionsChanged(owner)
  500. }
  501. }
  502. publishOwnerSessionsChanged(owner: string, reason = 'session.title', profile?: string, sessionID?: string): void {
  503. if (this.closed) return
  504. this.onNativeGlobalEvent(owner, 'sessions.changed')
  505. const frame = {
  506. jsonrpc: '2.0',
  507. method: 'event',
  508. params: { type: 'sessions.changed', payload: { reason, profile, session_id: sessionID } },
  509. }
  510. for (const channel of this.channels.values()) {
  511. if (channel.kind !== 'chat' || !channel.principal.valid()
  512. || this.nativeOwner(channel.principal) !== owner) continue
  513. this.emit(channel, 'frame', frame)
  514. }
  515. }
  516. private nativeOwner(principal: RealtimePrincipal): string | undefined {
  517. const device = /^device:[^:]+(?::agent:[^:]+)?$/.test(principal.key)
  518. if (device) return principal.key
  519. const match = /^(?:user|push):([^:]+)/.exec(principal.key)
  520. return match?.[1]
  521. }
  522. private async recover(u: Upstream, changed: boolean): Promise<void> {
  523. if (u.recovering) return
  524. u.recovering = true
  525. try {
  526. for (const r of u.routes.values()) {
  527. const lastSeen = r.seq
  528. const resumed = await this.rpc(u, 'session.resume', { session_id: r.stored, profile: r.profile, close_on_disconnect: false, omit_messages: true })
  529. if (resumed.error?.code === 4007 || resumed.error?.code === 'session_not_found') {
  530. // Hermes' session.resume uses 4007 for a missing stored session. Retire
  531. // only that route; a deleted session must not reconnect the whole service forever.
  532. this.reset(u, 'upstream_session_missing')
  533. this.event(u, { method: 'event', params: { type: 'error', session_id: r.runtime,
  534. payload: { code: 'session_not_found', message: 'Hermes 中的原会话已不存在,无法恢复。' } } }, true)
  535. r.active = false
  536. const key = this.routeKey(r.profile, r.stored)
  537. u.routes.delete(key); u.byRuntime.delete(r.runtime)
  538. this.routeOwners.delete(JSON.stringify([u.principal.instanceKey ?? u.principal.upstreamKey, r.profile, r.stored]))
  539. for (const c of this.channels.values()) if (c.principal.upstreamKey === u.principal.upstreamKey) c.routes.delete(key)
  540. for (const [id, runtime] of this.interactions) if (id.startsWith(`${u.principal.upstreamKey}:`) && runtime === r.runtime) this.interactions.delete(id)
  541. continue
  542. }
  543. if (resumed.error || typeof resumed.result?.session_id !== 'string' || !resumed.result.session_id) {
  544. throw new Error('Upstream session recovery failed')
  545. }
  546. r.workingDirectory = undefined
  547. r.cwd = typeof resumed.result?.info?.cwd === 'string' ? resumed.result.info.cwd : undefined
  548. if (typeof resumed.result?.running === 'boolean') r.active = resumed.result.running
  549. const restore=(snapshot:boolean)=>{
  550. for(const principal of r.observers.values()){
  551. const owner=this.nativeOwner(principal)
  552. if(owner&&principal.valid()&&resumed.result)this.onNativeRoute(owner,r.profile,r.stored,String(resumed.result.session_id??r.runtime),r.active,
  553. snapshot?resumed.result:{...resumed.result,inflight:undefined})
  554. }
  555. }
  556. if (resumed.result.session_id !== r.runtime || changed) {
  557. if (resumed.result?.session_id) { u.byRuntime.delete(r.runtime); r.runtime = resumed.result.session_id; u.byRuntime.set(r.runtime, r) }
  558. r.seq = 0; this.reset(u, 'upstream_reset'); restore(true); continue
  559. }
  560. const replay = await this.rpc(u, 'session.events.since', { session_id: r.runtime, last_seen: lastSeen })
  561. if (replay.error || replay.result?.truncated || replay.result?.epoch !== u.epoch) {this.reset(u, 'upstream_replay_unavailable');restore(true)}
  562. else {for (const p of replay.result?.events ?? []) this.event(u, { jsonrpc: '2.0', method: 'event', params: p }, true);restore(false)}
  563. }
  564. } catch (error) { this.reset(u, 'upstream_recovery_failed'); throw error }
  565. finally {
  566. u.recovering = false
  567. const deferred = u.deferred; u.deferred = []
  568. for (const frame of deferred) this.event(u, frame)
  569. }
  570. }
  571. private reset(u: Upstream, reason: string): void {
  572. this.onActivity({ kind: 'reset', name: reason })
  573. for (const route of u.routes.values()) {
  574. for (const principal of route.observers.values()) {
  575. const owner = this.nativeOwner(principal)
  576. if (owner && principal.valid()) {
  577. this.onNativeEvent(owner, route.profile, route.stored, {
  578. type: 'gateway.reset', payload: { reason }, seq: route.seq,
  579. })
  580. }
  581. }
  582. }
  583. for (const c of this.channels.values()) if (c.kind === 'chat' && c.principal.upstreamKey === u.principal.upstreamKey) this.emit(c, 'reset', { reason })
  584. }
  585. publishServerIdentity(identity: import('../shared/serverIdentity.js').ServerIdentity, instanceKey: string): void {
  586. for (const channel of this.channels.values()) {
  587. if (channel.principal.valid() && channel.principal.instanceKey === instanceKey) {
  588. this.emit(channel, 'frame', { jsonrpc: '2.0', method: 'event', params: { type: 'server.identity.changed', payload: identity } })
  589. }
  590. }
  591. }
  592. private emit(c: RealtimeChannel, event: StreamEntry['event'], value: Frame): void {
  593. const publicValue = this.withoutProtectedSessions(value)
  594. if (!publicValue) return
  595. const data = JSON.stringify(publicValue)
  596. const entry: StreamEntry = { id: `${this.epoch}:${++c.seq}`, event, data, at: this.now(), bytes: Buffer.byteLength(data) }
  597. c.entries.push(entry); c.bytes += entry.bytes
  598. this.trim(c)
  599. for (const listener of [...c.listeners]) {
  600. try { listener(entry) } catch { c.listeners.delete(listener) }
  601. }
  602. }
  603. private withoutProtectedSessions(value: any): any {
  604. if (Array.isArray(value)) return value.map(v => this.withoutProtectedSessions(v)).filter(v => v !== undefined)
  605. if (!value || typeof value !== 'object') return value
  606. if (value.source === 'yaoyao_workspace' || ['id','session_id','sessionId','stored_session_id','session_key'].some(k => typeof value[k] === 'string' && this.protectedSession(value[k]))) return undefined
  607. return Object.fromEntries(Object.entries(value).map(([k,v]) => [k,this.withoutProtectedSessions(v)]).filter(([,v]) => v !== undefined))
  608. }
  609. private trim(c: RealtimeChannel): void {
  610. const own = [...this.channels.values()].filter(v => v.principal.key === c.principal.key)
  611. while (own.reduce((n, v) => n + v.bytes, 0) > 8 * 1024 * 1024 || own.reduce((n, v) => n + v.entries.length, 0) > 10_000) {
  612. const oldest = own.filter(v => v.entries.length).sort((a, b) => a.entries[0]!.at - b.entries[0]!.at)[0]
  613. if (!oldest) break
  614. this.evict(oldest)
  615. }
  616. while (c.entries[0] && c.entries[0].at < this.now() - 600_000) this.evict(c)
  617. while ([...this.channels.values()].reduce((n, v) => n + v.bytes, 0) > 128 * 1024 * 1024) {
  618. const oldest = [...this.channels.values()].filter(v => v.entries.length).sort((a, b) => a.entries[0]!.at - b.entries[0]!.at)[0]
  619. if (!oldest) break
  620. this.evict(oldest)
  621. }
  622. }
  623. private evict(c: RealtimeChannel): void {
  624. const entry = c.entries.shift()!
  625. c.bytes -= entry.bytes; c.droppedThrough = Number(entry.id.split(':').at(-1))
  626. }
  627. private sweep(): void {
  628. for (const [key, item] of this.completed) if (item.at < this.now() - 600_000) this.completed.delete(key)
  629. for (const c of this.channels.values()) {
  630. if (!c.principal.valid() || (!c.listeners.size && this.now() - c.touched > 600_000)) this.remove(c)
  631. else this.trim(c)
  632. }
  633. for (const [key, u] of this.upstreams) {
  634. if (u.pending.size || u.tails.size || [...u.routes.values()].some(r => r.active)) { u.lastUsed = this.now(); continue }
  635. if ([...this.channels.values()].some(c => c.principal.upstreamKey === key)) { u.lastUsed = this.now(); continue }
  636. if (this.now() - u.lastUsed > 300_000) {
  637. this.upstreams.delete(key); clearTimeout(u.retry); clearInterval(u.ping); u.socket?.close()
  638. for (const [route, owner] of this.routeOwners) if (owner === key) this.routeOwners.delete(route)
  639. }
  640. }
  641. }
  642. close(): void {
  643. if (this.closed) return
  644. this.closed = true; clearInterval(this.timer)
  645. for (const c of [...this.channels.values()]) this.remove(c)
  646. for (const u of this.upstreams.values()) {
  647. clearTimeout(u.retry); clearInterval(u.ping)
  648. for (const p of u.pending.values()) { clearTimeout(p.timer); p.reject(new Error('Broker stopped')) }
  649. u.pending.clear(); u.socket?.terminate()
  650. }
  651. // Pending command continuations may still update their receipts in the next microtask.
  652. void Promise.allSettled(this.commands.values()).then(() => this.receipts.close())
  653. }
  654. }