pushEvents.ts 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687
  1. import { createHash, randomUUID } from 'node:crypto'
  2. import WebSocket, { type RawData } from 'ws'
  3. import { isSupportedGroupProtocolVersion } from '../shared/types.js'
  4. import type { ServerConfig } from './config.js'
  5. import type { UpstreamServiceSession } from './localAuth.js'
  6. import type { PushCoordinator as PersistentPushCoordinator } from './pushCoordinator.js'
  7. import { notificationPlainText } from './notificationText.js'
  8. type MaybePromise<T> = T | Promise<T>
  9. type JsonRecord = Record<string, unknown>
  10. export interface ChatRelayIdentity {
  11. localUserID: string
  12. connectionID?: string
  13. accountKey?: string
  14. source: 'web' | 'gateway' | 'paired'
  15. }
  16. interface ChatObservationBase {
  17. localUserID: string
  18. connectionID: string
  19. accountKey?: string
  20. source: ChatRelayIdentity['source']
  21. observedAt: number
  22. }
  23. export type ChatPushObservation = ChatObservationBase & (
  24. | {
  25. type: 'chat.session_opened'
  26. action: 'create' | 'resume'
  27. profile: string
  28. runtimeSessionID: string
  29. storedSessionID: string
  30. }
  31. | {
  32. type: 'chat.prompt'
  33. phase: 'submitted' | 'accepted' | 'rejected'
  34. requestID: string
  35. profile?: string
  36. runtimeSessionID: string
  37. storedSessionID?: string
  38. queued: boolean
  39. status?: string
  40. error?: string
  41. }
  42. | {
  43. type: 'chat.rpc_event'
  44. eventType: string
  45. profile?: string
  46. runtimeSessionID?: string
  47. storedSessionID?: string
  48. payload: JsonRecord
  49. }
  50. | {
  51. type: 'chat.disconnected'
  52. activeSessions: Array<{
  53. profile: string
  54. runtimeSessionID: string
  55. storedSessionID: string
  56. }>
  57. }
  58. )
  59. type ChatObservationPayload = ChatPushObservation extends infer Observation
  60. ? Observation extends ChatObservationBase
  61. ? Omit<Observation, keyof ChatObservationBase>
  62. : never
  63. : never
  64. export interface GroupWatchAnchor {
  65. epoch: string
  66. cursor: number
  67. }
  68. export interface GroupWatchReset extends GroupWatchAnchor {
  69. reason: string
  70. }
  71. export interface ChatPushJob {
  72. id: string
  73. localUserID: string
  74. profile?: string
  75. runtimeSessionID: string
  76. storedSessionID?: string
  77. requestID: string
  78. queued: boolean
  79. phase: 'submitted' | 'accepted' | 'watching'
  80. submittedAt: number
  81. expiresAt: number
  82. metadata?: Record<string, unknown>
  83. }
  84. interface NotificationCandidateBase {
  85. eventID: string
  86. localUserID: string
  87. title: string
  88. body: string
  89. collapseID: string
  90. data: Record<string, string>
  91. }
  92. export interface ChatPushCandidate extends NotificationCandidateBase {
  93. kind: 'chat.completed' | 'chat.failed' | 'chat.approval.requested' | 'chat.clarification.requested'
  94. jobID: string
  95. profile?: string
  96. sessionID?: string
  97. requestID?: string
  98. }
  99. export interface GroupPushCandidate {
  100. eventID: NotificationCandidateBase['eventID']
  101. localUserID: NotificationCandidateBase['localUserID']
  102. title: NotificationCandidateBase['title']
  103. body: NotificationCandidateBase['body']
  104. collapseID: NotificationCandidateBase['collapseID']
  105. data: NotificationCandidateBase['data']
  106. kind: 'group.message.completed' | 'group.message.failed' | 'group.interaction.requested'
  107. epoch?: string
  108. cursor?: number
  109. roomID: string
  110. topicID?: string
  111. messageID?: string
  112. messageSequence?: number
  113. interactionID?: string
  114. interactionKind?: 'approval' | 'clarification' | 'unknown'
  115. senderName?: string
  116. content?: string
  117. error?: string
  118. occurredAt?: number
  119. }
  120. export type PushNotificationCandidate = ChatPushCandidate | GroupPushCandidate
  121. /**
  122. * Persistence and delivery boundary used by the relay and the group watcher.
  123. * Implementations must durably deduplicate `enqueueNotification` by
  124. * `localUserID + eventID`. `advanceGroupCursor` is called only after every
  125. * candidate for that cursor has been durably accepted by the outbox.
  126. */
  127. export interface PushEventCoordinator {
  128. observeChat?(observation: ChatPushObservation): MaybePromise<void>
  129. saveChatJob(job: ChatPushJob): MaybePromise<void>
  130. completeChatJob(jobID: string): MaybePromise<void>
  131. pendingChatJobs(): MaybePromise<readonly ChatPushJob[]>
  132. recoverChatJob?(job: ChatPushJob): MaybePromise<void>
  133. promptDigest?(localUserID: string, prompt: string): MaybePromise<string>
  134. canRecoverChatJob?(job: ChatPushJob): MaybePromise<boolean>
  135. enqueueNotification(candidate: PushNotificationCandidate): MaybePromise<'enqueued' | 'duplicate' | 'ignored' | void>
  136. groupWatchAnchor(): MaybePromise<GroupWatchAnchor | undefined>
  137. groupSubscribers(roomID: string): MaybePromise<readonly string[]>
  138. advanceGroupCursor(anchor: GroupWatchAnchor): MaybePromise<void>
  139. resetGroupCursor(reset: GroupWatchReset): MaybePromise<GroupWatchAnchor | void>
  140. }
  141. /** Keep the event pipeline decoupled from the concrete persistent coordinator. */
  142. export function createPushEventCoordinator(push: PersistentPushCoordinator): PushEventCoordinator {
  143. return {
  144. saveChatJob: job => { push.saveChatJob(job) },
  145. completeChatJob: jobID => { push.completeChatJob(jobID) },
  146. pendingChatJobs: () => push.pendingChatJobs(),
  147. promptDigest: (localUserID, prompt) => push.promptDigest(localUserID, prompt),
  148. canRecoverChatJob: job => push.chatJobRecoveryAllowed(job),
  149. enqueueNotification: candidate => push.enqueueNotification(candidate),
  150. groupWatchAnchor: () => push.groupWatchAnchor(),
  151. groupSubscribers: roomID => push.subscribedUserIds(roomID),
  152. advanceGroupCursor: anchor => { push.advanceGroupCursor(anchor) },
  153. resetGroupCursor: reset => push.resetGroupCursor(reset),
  154. }
  155. }
  156. function record(value: unknown): JsonRecord {
  157. return value && typeof value === 'object' && !Array.isArray(value)
  158. ? value as JsonRecord
  159. : {}
  160. }
  161. function string(value: unknown): string {
  162. return typeof value === 'string' ? value.trim() : ''
  163. }
  164. function requestID(value: unknown): string {
  165. return typeof value === 'string' || typeof value === 'number' ? String(value) : ''
  166. }
  167. function errorMessage(value: unknown): string {
  168. const source = record(value)
  169. return string(source.message ?? source.error) || (value == null ? '' : String(value))
  170. }
  171. function summary(value: unknown, fallback: string): string {
  172. return notificationPlainText(value, { fallback, maximum: 180 })
  173. }
  174. function eventTimestamp(value: unknown): number | undefined {
  175. if (typeof value === 'number' && Number.isFinite(value)) {
  176. return value < 10_000_000_000 ? value * 1_000 : value
  177. }
  178. if (typeof value !== 'string' || !value.trim()) return undefined
  179. const numeric = Number(value)
  180. if (Number.isFinite(numeric)) return numeric < 10_000_000_000 ? numeric * 1_000 : numeric
  181. const parsed = Date.parse(value)
  182. return Number.isNaN(parsed) ? undefined : parsed
  183. }
  184. function collapseID(prefix: string, value: string): string {
  185. return `${prefix}:${createHash('sha256').update(value, 'utf8').digest('base64url').slice(0, 32)}`
  186. }
  187. function notificationData(
  188. eventID: string,
  189. kind: PushNotificationCandidate['kind'],
  190. localUserID: string,
  191. values: Record<string, string | undefined>,
  192. ): Record<string, string> {
  193. return Object.fromEntries(Object.entries({
  194. version: '1', eventId: eventID, kind, clientAccountId: localUserID, ...values,
  195. }).filter((entry): entry is [string, string] => typeof entry[1] === 'string' && Boolean(entry[1])))
  196. }
  197. interface SessionIdentity {
  198. profile: string
  199. runtimeSessionID: string
  200. storedSessionID: string
  201. baseline?: ChatMessageBaseline
  202. baselinePromise?: Promise<ChatMessageBaseline>
  203. }
  204. interface PendingSessionOpen {
  205. action: 'create' | 'resume'
  206. profile: string
  207. requestedStoredSessionID: string
  208. }
  209. interface PendingPrompt {
  210. requestID: string
  211. runtimeSessionID: string
  212. storedSessionID?: string
  213. profile?: string
  214. queued: boolean
  215. job: ChatPushJob
  216. }
  217. export interface ChatTerminalContent {
  218. title?: string
  219. body?: string
  220. messageID?: string
  221. timestamp?: number
  222. confirmed?: boolean
  223. failed?: boolean
  224. error?: string
  225. correlated?: boolean
  226. }
  227. export interface ChatMessageBaseline {
  228. messageID?: string
  229. sequence?: number
  230. total: number
  231. assistantCount: number
  232. lastRowSequence?: number
  233. }
  234. export interface ChatNotificationResolver {
  235. captureBaseline?(job: ChatPushJob): Promise<ChatMessageBaseline>
  236. resolveTerminal(job: ChatPushJob, eventType: string, payload: JsonRecord): Promise<ChatTerminalContent>
  237. }
  238. /** Observes one already-authorized chat relay without changing its protocol. */
  239. export class ChatPushRelayObserver {
  240. readonly #connectionID: string
  241. readonly #sessions = new Map<string, SessionIdentity>()
  242. readonly #pendingSessionOpens = new Map<string, PendingSessionOpen>()
  243. readonly #pendingPrompts = new Map<string, PendingPrompt>()
  244. readonly #jobsByRuntimeSession = new Map<string, ChatPushJob[]>()
  245. readonly #terminalJobIDs = new Set<string>()
  246. #closed = false
  247. #delivery = Promise.resolve()
  248. constructor(
  249. readonly coordinator: PushEventCoordinator,
  250. readonly identity: ChatRelayIdentity,
  251. readonly now: () => number = Date.now,
  252. readonly notificationResolver?: ChatNotificationResolver,
  253. ) {
  254. this.#connectionID = identity.connectionID ?? randomUUID().toLowerCase()
  255. }
  256. observeClientFrame(frameText: string): void {
  257. if (this.#closed) return
  258. let frame: JsonRecord
  259. try { frame = record(JSON.parse(frameText)) } catch { return }
  260. const method = string(frame.method)
  261. const id = requestID(frame.id)
  262. const params = record(frame.params)
  263. if ((method === 'session.create' || method === 'session.resume') && id) {
  264. this.#pendingSessionOpens.set(id, {
  265. action: method === 'session.create' ? 'create' : 'resume',
  266. profile: string(params.profile),
  267. requestedStoredSessionID: method === 'session.resume' ? string(params.session_id) : '',
  268. })
  269. return
  270. }
  271. if (method !== 'prompt.submit' || !id) return
  272. const runtimeSessionID = string(params.session_id)
  273. if (!runtimeSessionID) return
  274. const session = this.#sessions.get(runtimeSessionID)
  275. let job: ChatPushJob = {
  276. id: randomUUID().toLowerCase(),
  277. localUserID: this.identity.localUserID,
  278. profile: session?.profile,
  279. runtimeSessionID,
  280. storedSessionID: session?.storedSessionID,
  281. requestID: id,
  282. queued: params.queued === true,
  283. phase: 'submitted',
  284. submittedAt: this.now(),
  285. expiresAt: this.now() + 24 * 60 * 60 * 1_000,
  286. }
  287. if (this.coordinator.promptDigest) {
  288. try {
  289. const digest = this.coordinator.promptDigest(
  290. this.identity.localUserID,
  291. string(params.text),
  292. )
  293. if (typeof digest === 'string' && digest) {
  294. job = { ...job, metadata: { ...job.metadata, promptDigest: digest } }
  295. }
  296. } catch { /* Missing correlation fails closed without blocking chat. */ }
  297. }
  298. if (session?.baseline) {
  299. const baseline = session.baseline
  300. job = {
  301. ...job,
  302. metadata: {
  303. ...job.metadata,
  304. baselineCaptured: true,
  305. baselineMessageID: baseline.messageID,
  306. baselineSequence: baseline.sequence,
  307. baselineTotal: baseline.total,
  308. baselineAssistantCount: baseline.assistantCount,
  309. baselineRowSequence: baseline.lastRowSequence,
  310. },
  311. }
  312. } else {
  313. job = { ...job, metadata: { ...job.metadata, baselineCaptured: false } }
  314. }
  315. const prompt: PendingPrompt = {
  316. requestID: id,
  317. runtimeSessionID,
  318. storedSessionID: session?.storedSessionID,
  319. profile: session?.profile,
  320. queued: params.queued === true,
  321. job,
  322. }
  323. this.#pendingPrompts.set(id, prompt)
  324. this.#jobsByRuntimeSession.set(runtimeSessionID, [
  325. ...(this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []),
  326. prompt.job,
  327. ])
  328. this.#enqueue(() => this.coordinator.saveChatJob(prompt.job))
  329. if (!session?.baseline && session?.baselinePromise) {
  330. void session.baselinePromise.then(baseline => {
  331. this.#applyBaseline(runtimeSessionID, prompt.job.id, baseline)
  332. }).catch(() => undefined)
  333. }
  334. this.#emit({
  335. type: 'chat.prompt',
  336. phase: 'submitted',
  337. requestID: prompt.requestID,
  338. runtimeSessionID: prompt.runtimeSessionID,
  339. storedSessionID: prompt.storedSessionID,
  340. profile: prompt.profile,
  341. queued: prompt.queued,
  342. })
  343. }
  344. ownsJob(jobID: string): boolean {
  345. return !this.#closed && [...this.#jobsByRuntimeSession.values()].some(jobs => jobs.some(job => job.id === jobID))
  346. }
  347. observeUpstreamFrame(data: RawData, isBinary: boolean): void {
  348. if (this.#closed || isBinary) return
  349. let frame: JsonRecord
  350. try { frame = record(JSON.parse(Buffer.from(data as Uint8Array).toString('utf8'))) } catch { return }
  351. const id = requestID(frame.id)
  352. if (id) {
  353. const pendingOpen = this.#pendingSessionOpens.get(id)
  354. if (pendingOpen) {
  355. this.#pendingSessionOpens.delete(id)
  356. if (!frame.error) {
  357. const result = record(frame.result)
  358. const runtimeSessionID = string(result.session_id ?? result.sessionId)
  359. const storedSessionID = string(
  360. result.stored_session_id ?? result.storedSessionId ?? result.session_key,
  361. ) || pendingOpen.requestedStoredSessionID
  362. if (runtimeSessionID && storedSessionID && pendingOpen.profile) {
  363. const session: SessionIdentity = {
  364. profile: pendingOpen.profile,
  365. runtimeSessionID,
  366. storedSessionID,
  367. ...(pendingOpen.action === 'create'
  368. ? { baseline: { total: 0, assistantCount: 0, lastRowSequence: 0 } }
  369. : {}),
  370. }
  371. this.#sessions.set(runtimeSessionID, session)
  372. if (pendingOpen.action === 'resume' && this.notificationResolver?.captureBaseline) {
  373. const baselineJob: ChatPushJob = {
  374. id: `baseline:${this.#connectionID}:${runtimeSessionID}`,
  375. localUserID: this.identity.localUserID,
  376. profile: session.profile,
  377. runtimeSessionID,
  378. storedSessionID,
  379. requestID: 'baseline',
  380. queued: false,
  381. phase: 'submitted',
  382. submittedAt: this.now(),
  383. expiresAt: this.now() + 60_000,
  384. }
  385. const baselinePromise = this.notificationResolver.captureBaseline(baselineJob)
  386. session.baselinePromise = baselinePromise
  387. void baselinePromise.then(baseline => {
  388. if (this.#sessions.get(runtimeSessionID) === session) session.baseline = baseline
  389. }).catch(() => undefined)
  390. }
  391. this.#emit({
  392. type: 'chat.session_opened',
  393. profile: session.profile,
  394. runtimeSessionID,
  395. storedSessionID,
  396. action: pendingOpen.action,
  397. })
  398. }
  399. }
  400. }
  401. const pendingPrompt = this.#pendingPrompts.get(id)
  402. if (pendingPrompt) {
  403. this.#pendingPrompts.delete(id)
  404. const result = record(frame.result)
  405. const status = string(result.status) || (frame.error ? 'rejected' : 'accepted')
  406. this.#emit({
  407. type: 'chat.prompt',
  408. phase: frame.error ? 'rejected' : 'accepted',
  409. requestID: pendingPrompt.requestID,
  410. runtimeSessionID: pendingPrompt.runtimeSessionID,
  411. storedSessionID: pendingPrompt.storedSessionID,
  412. profile: pendingPrompt.profile,
  413. queued: pendingPrompt.queued,
  414. status,
  415. ...(frame.error ? { error: errorMessage(frame.error) || 'Hermes rejected the prompt' } : {}),
  416. })
  417. if (this.#terminalJobIDs.has(pendingPrompt.job.id)) {
  418. // A terminal event can legally overtake its JSON-RPC receipt.
  419. // Never recreate a job that is already being completed.
  420. } else if (frame.error) {
  421. this.#removeJob(pendingPrompt.job)
  422. this.#enqueue(() => this.coordinator.completeChatJob(pendingPrompt.job.id))
  423. } else {
  424. const runID = string(result.run_id ?? result.runId ?? result.turn_id ?? result.turnId)
  425. const accepted: ChatPushJob = {
  426. ...pendingPrompt.job,
  427. phase: 'accepted',
  428. ...(runID ? { metadata: { ...pendingPrompt.job.metadata, runID } } : {}),
  429. }
  430. this.#replaceJob(accepted)
  431. this.#enqueue(() => this.coordinator.saveChatJob(accepted))
  432. }
  433. }
  434. }
  435. if (string(frame.method) !== 'event') return
  436. const params = record(frame.params)
  437. const eventType = string(params.type)
  438. if (!eventType || eventType === 'gateway.ready') return
  439. const runtimeSessionID = string(params.session_id)
  440. const session = runtimeSessionID ? this.#sessions.get(runtimeSessionID) : undefined
  441. this.#emit({
  442. type: 'chat.rpc_event',
  443. eventType,
  444. profile: string(params.profile) || session?.profile || undefined,
  445. runtimeSessionID: runtimeSessionID || undefined,
  446. storedSessionID: session?.storedSessionID,
  447. payload: record(params.payload),
  448. })
  449. if (runtimeSessionID) this.#observeRPCEvent(
  450. eventType,
  451. runtimeSessionID,
  452. record(params.payload),
  453. )
  454. }
  455. disconnected(): void {
  456. if (this.#closed) return
  457. this.#closed = true
  458. this.#emit({
  459. type: 'chat.disconnected',
  460. activeSessions: [...this.#sessions.values()],
  461. })
  462. for (const jobs of this.#jobsByRuntimeSession.values()) {
  463. for (const job of [...jobs]) {
  464. if (this.#terminalJobIDs.has(job.id)) continue
  465. const disconnectedJob: ChatPushJob = {
  466. ...job,
  467. metadata: { ...job.metadata, disconnectedAt: this.now() },
  468. }
  469. this.#replaceJob(disconnectedJob)
  470. this.#enqueue(() => this.coordinator.saveChatJob(disconnectedJob))
  471. if (this.coordinator.recoverChatJob) {
  472. this.#enqueue(() => this.coordinator.recoverChatJob!(disconnectedJob))
  473. }
  474. }
  475. }
  476. }
  477. async flush(): Promise<void> {
  478. await Promise.allSettled(
  479. [...this.#sessions.values()].flatMap(session => session.baselinePromise ? [session.baselinePromise] : []),
  480. )
  481. await Promise.resolve()
  482. await this.#delivery
  483. }
  484. #emit(observation: ChatObservationPayload): void {
  485. const event = {
  486. ...observation,
  487. localUserID: this.identity.localUserID,
  488. connectionID: this.#connectionID,
  489. accountKey: this.identity.accountKey,
  490. source: this.identity.source,
  491. observedAt: this.now(),
  492. } as ChatPushObservation
  493. if (this.coordinator.observeChat) this.#enqueue(() => this.coordinator.observeChat!(event))
  494. }
  495. #enqueue(operation: () => MaybePromise<void>): void {
  496. this.#delivery = this.#delivery.then(operation).catch(() => undefined)
  497. }
  498. #replaceJob(job: ChatPushJob): void {
  499. const jobs = this.#jobsByRuntimeSession.get(job.runtimeSessionID) ?? []
  500. const index = jobs.findIndex(candidate => candidate.id === job.id)
  501. if (index >= 0) jobs[index] = job
  502. else jobs.push(job)
  503. this.#jobsByRuntimeSession.set(job.runtimeSessionID, jobs)
  504. }
  505. #applyBaseline(runtimeSessionID: string, jobID: string, baseline: ChatMessageBaseline): void {
  506. const jobs = this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []
  507. const index = jobs.findIndex(job => job.id === jobID)
  508. if (index < 0 || this.#terminalJobIDs.has(jobID)) return
  509. const current = jobs[index]!
  510. const updated: ChatPushJob = {
  511. ...current,
  512. metadata: {
  513. ...current.metadata,
  514. baselineCaptured: true,
  515. baselineMessageID: baseline.messageID,
  516. baselineSequence: baseline.sequence,
  517. baselineTotal: baseline.total,
  518. baselineAssistantCount: baseline.assistantCount,
  519. baselineRowSequence: baseline.lastRowSequence,
  520. },
  521. }
  522. jobs[index] = updated
  523. for (const pending of this.#pendingPrompts.values()) {
  524. if (pending.job.id === jobID) pending.job = updated
  525. }
  526. this.#enqueue(() => this.coordinator.saveChatJob(updated))
  527. }
  528. #removeJob(job: ChatPushJob): void {
  529. const jobs = (this.#jobsByRuntimeSession.get(job.runtimeSessionID) ?? [])
  530. .filter(candidate => candidate.id !== job.id)
  531. if (jobs.length) this.#jobsByRuntimeSession.set(job.runtimeSessionID, jobs)
  532. else this.#jobsByRuntimeSession.delete(job.runtimeSessionID)
  533. }
  534. #observeRPCEvent(eventType: string, runtimeSessionID: string, payload: JsonRecord): void {
  535. const jobs = this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []
  536. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  537. let job = eventRunID
  538. ? jobs.find(candidate => string(candidate.metadata?.runID) === eventRunID)
  539. ?? jobs.find(candidate => !string(candidate.metadata?.runID))
  540. : jobs[0]
  541. if (!job) return
  542. if (eventRunID && !string(job.metadata?.runID)) {
  543. job = { ...job, metadata: { ...job.metadata, runID: eventRunID } }
  544. this.#replaceJob(job)
  545. const boundJob = job
  546. this.#enqueue(() => this.coordinator.saveChatJob(boundJob))
  547. }
  548. if (['approval.request', 'approval.requested', 'clarify.request', 'clarify.requested'].includes(eventType)) {
  549. const requestIDValue = string(payload.request_id ?? payload.requestId ?? payload.id)
  550. if (!requestIDValue) return
  551. const clarification = eventType.startsWith('clarify.')
  552. const kind: ChatPushCandidate['kind'] = clarification
  553. ? 'chat.clarification.requested' : 'chat.approval.requested'
  554. const eventID = `chat-interaction:${job.id}:${requestIDValue}`
  555. const candidate: ChatPushCandidate = {
  556. eventID,
  557. localUserID: job.localUserID,
  558. kind,
  559. jobID: job.id,
  560. profile: job.profile,
  561. sessionID: job.storedSessionID,
  562. requestID: requestIDValue,
  563. title: clarification ? 'Agent 需要补充信息' : 'Agent 请求审批',
  564. body: summary(payload.question ?? payload.message ?? payload.prompt, clarification ? '请打开夭夭继续处理' : '请打开夭夭审批'),
  565. collapseID: collapseID('chat-interaction', `${job.storedSessionID ?? runtimeSessionID}:${requestIDValue}`),
  566. data: notificationData(eventID, kind, job.localUserID, {
  567. profile: job.profile, sessionId: job.storedSessionID, requestId: requestIDValue,
  568. }),
  569. }
  570. this.#enqueue(async () => {
  571. if (!await this.#jobOwnershipAllowed(job)) return
  572. if (!eventRunID) {
  573. let ownership: ChatTerminalContent = {}
  574. try {
  575. ownership = await this.notificationResolver?.resolveTerminal(job, eventType, payload) ?? {}
  576. } catch { return }
  577. if (ownership.correlated !== true) return
  578. }
  579. await this.coordinator.enqueueNotification(candidate)
  580. })
  581. return
  582. }
  583. const terminal = ['message.complete', 'run.completed', 'run.failed', 'error'].includes(eventType)
  584. if (!terminal) return
  585. if (this.#terminalJobIDs.has(job.id)) return
  586. this.#terminalJobIDs.add(job.id)
  587. this.#enqueue(async () => {
  588. if (!await this.#jobOwnershipAllowed(job)) return
  589. let resolved: ChatTerminalContent | undefined
  590. if (!eventRunID) {
  591. try { resolved = await this.notificationResolver?.resolveTerminal(job, eventType, payload) ?? {} } catch { resolved = {} }
  592. const failed = eventType === 'run.failed' || eventType === 'error' || Boolean(payload.error)
  593. if (resolved.confirmed !== true && !(failed && resolved.correlated === true)) {
  594. this.#terminalJobIDs.delete(job.id)
  595. return
  596. }
  597. }
  598. const candidate = await chatTerminalCandidate(job, eventType, payload, this.notificationResolver, resolved)
  599. await this.coordinator.enqueueNotification(candidate)
  600. await this.coordinator.completeChatJob(job.id)
  601. this.#removeJob(job)
  602. })
  603. }
  604. async #jobOwnershipAllowed(job: ChatPushJob): Promise<boolean> {
  605. if (!this.coordinator.canRecoverChatJob) return true
  606. try { return await this.coordinator.canRecoverChatJob(job) } catch { return false }
  607. }
  608. }
  609. export interface GroupEventEnvelope {
  610. type: 'group.ready' | 'group.event' | 'group.heartbeat' | 'group.reset_required'
  611. epoch?: string
  612. cursor?: number
  613. heartbeatSeconds?: number
  614. roomId?: string
  615. event?: string
  616. payload?: unknown
  617. reason?: string
  618. }
  619. function canonicalAnchor(value: GroupWatchAnchor): GroupWatchAnchor {
  620. if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value.epoch)
  621. || !Number.isSafeInteger(value.cursor) || value.cursor < 0) {
  622. throw new Error('Invalid Yaoyao group event anchor')
  623. }
  624. return { epoch: value.epoch, cursor: value.cursor }
  625. }
  626. function eventRoomID(envelope: GroupEventEnvelope, payload: JsonRecord): string {
  627. return string(envelope.roomId ?? payload.roomId ?? payload.room_id)
  628. }
  629. function terminalGroupCandidate(
  630. envelope: GroupEventEnvelope,
  631. anchor: GroupWatchAnchor,
  632. ): Omit<GroupPushCandidate, 'localUserID' | 'data'> | undefined {
  633. if (envelope.type !== 'group.event') return undefined
  634. const payload = record(envelope.payload)
  635. const roomID = eventRoomID(envelope, payload)
  636. if (!roomID) return undefined
  637. if (envelope.event === 'message.upsert') {
  638. const senderKind = string(payload.senderKind ?? payload.sender_kind).toLowerCase()
  639. const status = string(payload.status).toLowerCase()
  640. const messageID = string(payload.id)
  641. if (senderKind !== 'agent' || !messageID || (status !== 'completed' && status !== 'failed')) return undefined
  642. const kind: GroupPushCandidate['kind'] = status === 'completed'
  643. ? 'group.message.completed' : 'group.message.failed'
  644. const eventID = `group-message:${messageID}:${status}`
  645. const content = string(payload.content)
  646. const error = string(payload.error)
  647. return {
  648. eventID,
  649. kind,
  650. epoch: anchor.epoch,
  651. cursor: anchor.cursor + 1,
  652. roomID,
  653. topicID: string(payload.topicId ?? payload.topic_id) || undefined,
  654. messageID,
  655. messageSequence: Number.isSafeInteger(Number(payload.seq ?? payload.sequence))
  656. ? Number(payload.seq ?? payload.sequence) : undefined,
  657. senderName: string(payload.senderName ?? payload.sender_name) || undefined,
  658. content: content || undefined,
  659. error: error || undefined,
  660. occurredAt: eventTimestamp(payload.updatedAt ?? payload.updated_at ?? payload.createdAt ?? payload.created_at),
  661. title: status === 'completed'
  662. ? `${string(payload.senderName ?? payload.sender_name) || 'Agent'} 已回复`
  663. : `${string(payload.senderName ?? payload.sender_name) || 'Agent'} 执行失败`,
  664. body: summary(error || content, status === 'completed' ? '打开夭夭查看团队消息' : '打开夭夭查看失败详情'),
  665. collapseID: collapseID('group-message', messageID),
  666. }
  667. }
  668. if (envelope.event !== 'interaction.requested') return undefined
  669. const interactionID = string(payload.id)
  670. const status = string(payload.status).toLowerCase()
  671. if (!interactionID || (status && status !== 'pending')) return undefined
  672. const rawKind = string(payload.kind).toLowerCase()
  673. const interactionKind = rawKind === 'approval' || rawKind === 'clarification' ? rawKind : 'unknown'
  674. const interactionPayload = record(payload.payload)
  675. const eventID = `group-interaction:${interactionID}`
  676. const content = string(interactionPayload.question ?? interactionPayload.message ?? interactionPayload.prompt)
  677. return {
  678. eventID,
  679. kind: 'group.interaction.requested',
  680. epoch: anchor.epoch,
  681. cursor: anchor.cursor + 1,
  682. roomID,
  683. topicID: string(payload.topicId ?? payload.topic_id) || undefined,
  684. interactionID,
  685. interactionKind,
  686. content: content || undefined,
  687. occurredAt: eventTimestamp(payload.updatedAt ?? payload.updated_at ?? payload.createdAt ?? payload.created_at),
  688. title: interactionKind === 'clarification' ? '团队 Agent 需要补充信息' : '团队 Agent 请求审批',
  689. body: summary(content, '请打开夭夭继续处理'),
  690. collapseID: collapseID('group-interaction', interactionID),
  691. }
  692. }
  693. export type GroupEnvelopeResult = 'ready' | 'heartbeat' | 'advanced' | 'ignored' | 'reset'
  694. /** Applies strict epoch/cursor continuity and extracts only push-worthy events. */
  695. export class GroupPushEventProcessor {
  696. #anchor: GroupWatchAnchor
  697. readonly #recent = new Map<string, true>()
  698. constructor(
  699. anchor: GroupWatchAnchor,
  700. readonly coordinator: PushEventCoordinator,
  701. readonly recentLimit = 4_096,
  702. ) {
  703. this.#anchor = canonicalAnchor(anchor)
  704. }
  705. get anchor(): GroupWatchAnchor { return { ...this.#anchor } }
  706. async accept(envelope: GroupEventEnvelope): Promise<GroupEnvelopeResult> {
  707. if (envelope.type === 'group.reset_required') {
  708. const proposed = canonicalAnchor({
  709. epoch: string(envelope.epoch) || this.#anchor.epoch,
  710. cursor: envelope.cursor ?? this.#anchor.cursor,
  711. })
  712. const recovered = await this.coordinator.resetGroupCursor({
  713. ...proposed,
  714. reason: string(envelope.reason) || 'server_reset',
  715. })
  716. this.#anchor = canonicalAnchor(recovered ?? proposed)
  717. this.#recent.clear()
  718. return 'reset'
  719. }
  720. if (envelope.epoch && envelope.epoch !== this.#anchor.epoch) {
  721. throw new Error('Yaoyao group event epoch changed without reset')
  722. }
  723. if (envelope.type === 'group.ready') {
  724. if (envelope.cursor !== this.#anchor.cursor) throw new Error('Yaoyao group ready cursor mismatch')
  725. return 'ready'
  726. }
  727. if (envelope.type === 'group.heartbeat') {
  728. if (envelope.cursor != null && envelope.cursor !== this.#anchor.cursor) {
  729. throw new Error('Yaoyao group heartbeat cursor mismatch')
  730. }
  731. return 'heartbeat'
  732. }
  733. if (envelope.type !== 'group.event' || !Number.isSafeInteger(envelope.cursor) || envelope.cursor! < 0) {
  734. throw new Error('Invalid Yaoyao group event envelope')
  735. }
  736. if (envelope.cursor! <= this.#anchor.cursor) return 'ignored'
  737. if (envelope.cursor !== this.#anchor.cursor + 1) throw new Error('Yaoyao group event cursor gap')
  738. const candidate = terminalGroupCandidate(envelope, this.#anchor)
  739. if (candidate) {
  740. const subscribers = [...new Set((await this.coordinator.groupSubscribers(candidate.roomID))
  741. .map(value => value.trim()).filter(Boolean))]
  742. for (const localUserID of subscribers) {
  743. const dedupeKey = `${localUserID}:${candidate.eventID}`
  744. if (this.#recent.has(dedupeKey)) continue
  745. await this.coordinator.enqueueNotification({
  746. ...candidate,
  747. localUserID,
  748. data: notificationData(candidate.eventID, candidate.kind, localUserID, {
  749. roomId: candidate.roomID,
  750. topicId: candidate.topicID,
  751. messageId: candidate.messageID,
  752. interactionId: candidate.interactionID,
  753. }),
  754. })
  755. this.#remember(dedupeKey)
  756. }
  757. }
  758. const next = { epoch: this.#anchor.epoch, cursor: envelope.cursor }
  759. await this.coordinator.advanceGroupCursor(next)
  760. this.#anchor = next
  761. return 'advanced'
  762. }
  763. #remember(key: string): void {
  764. this.#recent.delete(key)
  765. this.#recent.set(key, true)
  766. while (this.#recent.size > this.recentLimit) {
  767. const oldest = this.#recent.keys().next().value as string | undefined
  768. if (!oldest) break
  769. this.#recent.delete(oldest)
  770. }
  771. }
  772. }
  773. export interface GroupEventConnection {
  774. close(): void
  775. }
  776. export interface GroupEventSource {
  777. currentAnchor(): Promise<GroupWatchAnchor>
  778. connect(
  779. anchor: GroupWatchAnchor,
  780. handlers: {
  781. onEnvelope(envelope: GroupEventEnvelope): void
  782. onClose(error?: Error): void
  783. },
  784. ): Promise<GroupEventConnection>
  785. }
  786. function parseUpstreamJSON(body: Buffer): JsonRecord {
  787. try { return record(JSON.parse(body.toString('utf8'))) } catch { throw new Error('Hermes returned invalid JSON') }
  788. }
  789. function values(value: unknown): unknown[] {
  790. if (Array.isArray(value)) return value
  791. const source = record(value)
  792. const data = source.data
  793. return Array.isArray(data) ? data : []
  794. }
  795. function messageContent(value: unknown): string {
  796. if (typeof value === 'string') return value.trim()
  797. if (!Array.isArray(value)) return ''
  798. return value.map(item => {
  799. if (typeof item === 'string') return item
  800. const part = record(item)
  801. return string(part.text ?? part.content)
  802. }).filter(Boolean).join('\n').trim()
  803. }
  804. function messageTimestamp(value: unknown): number | undefined {
  805. if (typeof value === 'number' && Number.isFinite(value)) return value < 1_000_000_000_000 ? value * 1_000 : value
  806. if (typeof value === 'string' && value.trim()) {
  807. const numeric = Number(value)
  808. if (Number.isFinite(numeric)) return numeric < 1_000_000_000_000 ? numeric * 1_000 : numeric
  809. const parsed = Date.parse(value)
  810. if (!Number.isNaN(parsed)) return parsed
  811. }
  812. return undefined
  813. }
  814. interface ChatHistoryRow {
  815. role: string
  816. content: string
  817. messageID?: string
  818. runID?: string
  819. sequence: number
  820. timestamp?: number
  821. failed: boolean
  822. error?: string
  823. }
  824. export class HermesChatNotificationResolver implements ChatNotificationResolver {
  825. private readonly titleCache = new Map<string, { value: string; expiresAt: number }>()
  826. constructor(
  827. readonly upstreamSession: UpstreamServiceSession,
  828. readonly digestPrompt?: (localUserID: string, prompt: string) => MaybePromise<string>,
  829. ) {}
  830. private async snapshot(job: ChatPushJob): Promise<{
  831. title?: string
  832. total: number
  833. rows: ChatHistoryRow[]
  834. assistants: ChatHistoryRow[]
  835. latest?: ChatHistoryRow
  836. }> {
  837. if (!job.storedSessionID) return { total: 0, rows: [], assistants: [] }
  838. const search = new URLSearchParams({
  839. offset: '0', limit: '500', order: 'latest', include_compacted: 'true',
  840. })
  841. if (job.profile) search.set('profile', job.profile)
  842. const response = await this.upstreamSession.request(
  843. `/api/sessions/${encodeURIComponent(job.storedSessionID)}/messages`,
  844. { search },
  845. )
  846. if (response.status < 200 || response.status >= 300) throw new Error(`Hermes history failed (${response.status})`)
  847. const source = parseUpstreamJSON(response.body)
  848. const session = record(source.session)
  849. const messages = values(source.messages ?? source.items ?? source.data ?? source)
  850. const rows = messages.map((raw, index): ChatHistoryRow => {
  851. const message = record(raw)
  852. const role = string(message.role ?? message.senderKind ?? message.sender_kind).toLowerCase()
  853. const content = messageContent(message.content ?? message.text ?? message.message)
  854. const error = string(message.error)
  855. const status = string(message.status).toLowerCase()
  856. const rowID = Number(message.id)
  857. const explicitSequence = Number(message.sequence ?? message.seq ?? message.index)
  858. const rawSequence = Number.isFinite(rowID)
  859. ? rowID : Number.isFinite(explicitSequence) ? explicitSequence : index
  860. return {
  861. role,
  862. content: content || error,
  863. messageID: requestID(message.id ?? message.message_id ?? message.messageId) || undefined,
  864. runID: string(message.run_id ?? message.runId ?? message.turn_id ?? message.turnId) || undefined,
  865. sequence: rawSequence,
  866. timestamp: messageTimestamp(message.timestamp ?? message.createdAt ?? message.created_at),
  867. failed: status === 'failed' || status === 'error' || Boolean(error),
  868. ...(error ? { error } : {}),
  869. }
  870. }).sort((left, right) => left.sequence - right.sequence || (left.timestamp ?? 0) - (right.timestamp ?? 0))
  871. const assistants = rows.filter(row => (row.role === 'assistant' || row.role === 'agent') && Boolean(row.content))
  872. const pagination = record(source.pagination)
  873. const totalValue = Number(pagination.total ?? source.total ?? messages.length)
  874. let title = string(session.title ?? source.title) || undefined
  875. if (!title) title = await this.sessionTitle(job)
  876. return {
  877. title,
  878. total: Number.isSafeInteger(totalValue) && totalValue >= 0 ? totalValue : messages.length,
  879. rows,
  880. assistants,
  881. latest: assistants.at(-1),
  882. }
  883. }
  884. private async sessionTitle(job: ChatPushJob): Promise<string | undefined> {
  885. if (!job.storedSessionID) return undefined
  886. const key = `${job.profile ?? 'default'}:${job.storedSessionID}`
  887. const cached = this.titleCache.get(key)
  888. if (cached && cached.expiresAt > Date.now()) return cached.value
  889. const search = new URLSearchParams()
  890. if (job.profile) search.set('profile', job.profile)
  891. try {
  892. const response = await this.upstreamSession.request(
  893. `/api/sessions/${encodeURIComponent(job.storedSessionID)}`,
  894. { search },
  895. )
  896. if (response.status < 200 || response.status >= 300) return undefined
  897. const payload = parseUpstreamJSON(response.body)
  898. const data = record(payload.data)
  899. const session = record(payload.session ?? data.session ?? payload.data ?? payload)
  900. const title = string(session.title)
  901. if (!title) return undefined
  902. this.titleCache.set(key, { value: title, expiresAt: Date.now() + 5 * 60 * 1_000 })
  903. return title
  904. } catch {
  905. return undefined
  906. }
  907. }
  908. async captureBaseline(job: ChatPushJob): Promise<ChatMessageBaseline> {
  909. if (!job.storedSessionID) {
  910. return { total: 0, assistantCount: 0, lastRowSequence: 0 }
  911. }
  912. const search = new URLSearchParams({
  913. offset: '0', limit: '1', order: 'latest', include_compacted: 'true',
  914. })
  915. if (job.profile) search.set('profile', job.profile)
  916. const response = await this.upstreamSession.request(
  917. `/api/sessions/${encodeURIComponent(job.storedSessionID)}/messages`,
  918. { search },
  919. )
  920. if (response.status < 200 || response.status >= 300) throw new Error(`Hermes history failed (${response.status})`)
  921. const source = parseUpstreamJSON(response.body)
  922. const messages = values(source.messages ?? source.items ?? source.data ?? source)
  923. const raw = messages.at(-1)
  924. const message = record(raw)
  925. const role = string(message.role ?? message.senderKind ?? message.sender_kind).toLowerCase()
  926. const rowID = Number(message.id)
  927. const explicitSequence = Number(message.sequence ?? message.seq ?? message.index)
  928. const sequence = Number.isFinite(rowID)
  929. ? rowID : Number.isFinite(explicitSequence) ? explicitSequence : 0
  930. const pagination = record(source.pagination)
  931. const totalValue = Number(pagination.total ?? source.total ?? messages.length)
  932. return {
  933. messageID: role === 'assistant' || role === 'agent'
  934. ? requestID(message.id ?? message.message_id ?? message.messageId) || undefined
  935. : undefined,
  936. sequence: role === 'assistant' || role === 'agent' ? sequence : undefined,
  937. total: Number.isSafeInteger(totalValue) && totalValue >= 0 ? totalValue : messages.length,
  938. assistantCount: role === 'assistant' || role === 'agent' ? 1 : 0,
  939. lastRowSequence: sequence,
  940. }
  941. }
  942. async resolveTerminal(job: ChatPushJob, _eventType: string, _payload: JsonRecord): Promise<ChatTerminalContent> {
  943. const snapshot = await this.snapshot(job)
  944. const expectedRunID = string(job.metadata?.runID)
  945. if (expectedRunID) {
  946. const latest = snapshot.assistants.filter(candidate => candidate.runID === expectedRunID).at(-1)
  947. const baselineSequence = Number(job.metadata?.baselineSequence)
  948. const baselineRowSequence = Number(job.metadata?.baselineRowSequence)
  949. const baselineMessageID = string(job.metadata?.baselineMessageID)
  950. const confirmed = job.metadata?.baselineCaptured === true && Boolean(latest) && (
  951. (Number.isFinite(baselineSequence) && latest!.sequence > baselineSequence)
  952. || (Number.isFinite(baselineRowSequence) && latest!.sequence > baselineRowSequence)
  953. || Boolean(baselineMessageID && latest!.messageID && latest!.messageID !== baselineMessageID)
  954. )
  955. return this.terminalContent(snapshot.title, latest, confirmed, confirmed)
  956. }
  957. const promptDigest = string(job.metadata?.promptDigest)
  958. const baselineRowSequence = Number(job.metadata?.baselineRowSequence)
  959. if (!this.digestPrompt || !promptDigest || !Number.isFinite(baselineRowSequence)) {
  960. return { title: snapshot.title, confirmed: false, correlated: false }
  961. }
  962. const matchingUsers: ChatHistoryRow[] = []
  963. for (const row of snapshot.rows) {
  964. if (row.role !== 'user' || row.sequence <= baselineRowSequence || !row.content) continue
  965. let digest = ''
  966. try { digest = await this.digestPrompt(job.localUserID, row.content) } catch { continue }
  967. if (digest === promptDigest) matchingUsers.push(row)
  968. }
  969. if (matchingUsers.length !== 1) {
  970. return { title: snapshot.title, confirmed: false, correlated: false }
  971. }
  972. const userRow = matchingUsers[0]!
  973. const nextUser = snapshot.rows.find(row => row.role === 'user' && row.sequence > userRow.sequence)
  974. const latest = snapshot.assistants
  975. .filter(row => row.sequence > userRow.sequence && (!nextUser || row.sequence < nextUser.sequence))
  976. .at(-1)
  977. return this.terminalContent(snapshot.title, latest, Boolean(latest), true)
  978. }
  979. private terminalContent(
  980. title: string | undefined,
  981. latest: ChatHistoryRow | undefined,
  982. confirmed: boolean,
  983. correlated: boolean,
  984. ): ChatTerminalContent {
  985. return {
  986. title,
  987. body: latest?.content,
  988. messageID: latest?.messageID,
  989. timestamp: latest?.timestamp,
  990. confirmed,
  991. correlated,
  992. ...(latest?.failed ? { failed: true } : {}),
  993. ...(latest?.error ? { error: latest.error } : {}),
  994. }
  995. }
  996. }
  997. function chatInteractionCandidate(
  998. job: ChatPushJob,
  999. eventType: string,
  1000. payload: JsonRecord,
  1001. ): ChatPushCandidate | undefined {
  1002. const requestIDValue = string(payload.request_id ?? payload.requestId ?? payload.id)
  1003. if (!requestIDValue) return undefined
  1004. const clarification = eventType.startsWith('clarify.')
  1005. || string(payload.kind).toLowerCase() === 'clarification'
  1006. const kind: ChatPushCandidate['kind'] = clarification
  1007. ? 'chat.clarification.requested' : 'chat.approval.requested'
  1008. const eventID = `chat-interaction:${job.id}:${requestIDValue}`
  1009. return {
  1010. eventID,
  1011. localUserID: job.localUserID,
  1012. kind,
  1013. jobID: job.id,
  1014. profile: job.profile,
  1015. sessionID: job.storedSessionID,
  1016. requestID: requestIDValue,
  1017. title: clarification ? 'Agent 需要补充信息' : 'Agent 请求审批',
  1018. body: summary(payload.question ?? payload.message ?? payload.prompt, clarification ? '请打开夭夭继续处理' : '请打开夭夭审批'),
  1019. collapseID: collapseID('chat-interaction', `${job.storedSessionID ?? job.runtimeSessionID}:${requestIDValue}`),
  1020. data: notificationData(eventID, kind, job.localUserID, {
  1021. profile: job.profile, sessionId: job.storedSessionID, requestId: requestIDValue,
  1022. }),
  1023. }
  1024. }
  1025. async function chatTerminalCandidate(
  1026. job: ChatPushJob,
  1027. eventType: string,
  1028. payload: JsonRecord,
  1029. resolver?: ChatNotificationResolver,
  1030. suppliedContent?: ChatTerminalContent,
  1031. ): Promise<ChatPushCandidate> {
  1032. const terminalStatus = string(payload.status).toLowerCase()
  1033. const failed = eventType === 'run.failed' || eventType === 'error'
  1034. || terminalStatus === 'failed' || terminalStatus === 'error' || Boolean(payload.error)
  1035. let resolved: ChatTerminalContent = suppliedContent ?? {}
  1036. if (!suppliedContent) {
  1037. try { resolved = await resolver?.resolveTerminal(job, eventType, payload) ?? {} } catch { /* payload fallback */ }
  1038. }
  1039. const kind: ChatPushCandidate['kind'] = failed ? 'chat.failed' : 'chat.completed'
  1040. const eventID = `chat-terminal:${job.id}:${failed ? 'failed' : 'completed'}`
  1041. const authoritativeBody = resolved.confirmed === true ? resolved.body : undefined
  1042. return {
  1043. eventID,
  1044. localUserID: job.localUserID,
  1045. kind,
  1046. jobID: job.id,
  1047. profile: job.profile,
  1048. sessionID: job.storedSessionID,
  1049. title: resolved.title || (failed ? '任务执行失败' : '任务已完成'),
  1050. body: failed
  1051. ? summary(payload.error ?? payload.message ?? authoritativeBody, '打开夭夭查看失败详情')
  1052. : summary(authoritativeBody ?? payload.text ?? payload.message, '打开夭夭查看结果'),
  1053. collapseID: collapseID('chat', job.storedSessionID ?? job.runtimeSessionID),
  1054. data: notificationData(eventID, kind, job.localUserID, {
  1055. profile: job.profile, sessionId: job.storedSessionID,
  1056. }),
  1057. }
  1058. }
  1059. /**
  1060. * Server-owned recovery connection for one persisted chat job. The caller
  1061. * controls the number of concurrently started watchers.
  1062. */
  1063. export interface ChatPushRecoveryTransport {
  1064. start(): void
  1065. send(data: string): void
  1066. close(code?: number, reason?: string): void
  1067. }
  1068. export type ChatPushTransportFactory = (job: ChatPushJob, onFrame: (frame: string) => void, onClose: () => void) => Promise<ChatPushRecoveryTransport>
  1069. export class HermesChatPushJobWatcher {
  1070. #job: ChatPushJob
  1071. #running = false
  1072. #generation = 0
  1073. #retryAttempt = 0
  1074. #retryTimer?: ReturnType<typeof setTimeout>
  1075. #socket?: WebSocket | ChatPushRecoveryTransport
  1076. #resumeRequestID = ''
  1077. #frames = Promise.resolve()
  1078. #finished = false
  1079. constructor(
  1080. job: ChatPushJob,
  1081. readonly config: ServerConfig,
  1082. readonly upstreamSession: UpstreamServiceSession,
  1083. readonly coordinator: PushEventCoordinator,
  1084. readonly resolver: ChatNotificationResolver = new HermesChatNotificationResolver(upstreamSession),
  1085. readonly now: () => number = Date.now,
  1086. readonly onFinished?: (jobID: string) => void,
  1087. readonly transportFactory?: ChatPushTransportFactory,
  1088. ) {
  1089. this.#job = { ...job }
  1090. }
  1091. start(): void {
  1092. if (this.#running) return
  1093. this.#running = true
  1094. this.#generation += 1
  1095. void this.#connect(this.#generation)
  1096. }
  1097. stop(): void {
  1098. this.#running = false
  1099. this.#generation += 1
  1100. if (this.#retryTimer) clearTimeout(this.#retryTimer)
  1101. this.#retryTimer = undefined
  1102. this.#socket?.close(1000, 'chat push watcher close')
  1103. this.#socket = undefined
  1104. }
  1105. async #connect(generation: number): Promise<void> {
  1106. if (!this.#job.storedSessionID || this.#job.expiresAt <= this.now()) {
  1107. await this.coordinator.completeChatJob(this.#job.id)
  1108. this.stop()
  1109. this.#notifyFinished()
  1110. return
  1111. }
  1112. try {
  1113. if (this.transportFactory) {
  1114. const transport = await this.transportFactory(this.#job,
  1115. frame => this.#queueFrame(generation, Buffer.from(frame), false),
  1116. () => this.#scheduleReconnect(generation))
  1117. if (!this.#running || generation !== this.#generation) { transport.close(); return }
  1118. this.#socket = transport
  1119. transport.start()
  1120. return
  1121. }
  1122. const ticketResponse = await this.upstreamSession.request('/api/auth/ws-ticket', { method: 'POST' })
  1123. if (ticketResponse.status < 200 || ticketResponse.status >= 300) {
  1124. throw new Error(`Hermes WebSocket ticket failed (${ticketResponse.status})`)
  1125. }
  1126. const ticket = string(parseUpstreamJSON(ticketResponse.body).ticket)
  1127. if (!ticket) throw new Error('Hermes returned an empty WebSocket ticket')
  1128. if (!this.#running || generation !== this.#generation) return
  1129. const url = new URL(this.config.upstream)
  1130. url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'
  1131. const prefix = this.config.upstream.pathname === '/' ? '' : this.config.upstream.pathname.replace(/\/$/, '')
  1132. url.pathname = `${prefix}/api/ws`
  1133. url.search = ''
  1134. url.searchParams.set('ticket', ticket)
  1135. const socket = new WebSocket(url, {
  1136. headers: { Origin: this.config.upstream.origin },
  1137. maxPayload: 36 * 1_024 * 1_024,
  1138. handshakeTimeout: 15_000,
  1139. })
  1140. this.#socket = socket
  1141. socket.on('message', (data, isBinary) => this.#queueFrame(generation, data, isBinary))
  1142. socket.once('error', () => {
  1143. if (socket.readyState !== WebSocket.OPEN) this.#scheduleReconnect(generation)
  1144. })
  1145. socket.once('close', () => this.#scheduleReconnect(generation))
  1146. } catch {
  1147. this.#scheduleReconnect(generation)
  1148. }
  1149. }
  1150. #queueFrame(generation: number, data: RawData, isBinary: boolean): void {
  1151. this.#frames = this.#frames.then(async () => {
  1152. if (!this.#running || generation !== this.#generation || isBinary) return
  1153. let frame: JsonRecord
  1154. try { frame = record(JSON.parse(Buffer.from(data as Uint8Array).toString('utf8'))) } catch { return }
  1155. if (string(frame.method) === 'event') {
  1156. const params = record(frame.params)
  1157. const eventType = string(params.type)
  1158. if (eventType === 'gateway.ready') {
  1159. this.#resumeRequestID = `push-resume:${this.#job.id}:${generation}`
  1160. this.#socket?.send(JSON.stringify({
  1161. jsonrpc: '2.0',
  1162. id: this.#resumeRequestID,
  1163. method: 'session.resume',
  1164. params: {
  1165. session_id: this.#job.storedSessionID,
  1166. profile: this.#job.profile,
  1167. source: 'web',
  1168. close_on_disconnect: false,
  1169. omit_messages: true,
  1170. cols: 80,
  1171. },
  1172. }))
  1173. return
  1174. }
  1175. const runtimeSessionID = string(params.session_id)
  1176. if (runtimeSessionID && this.#job.runtimeSessionID && runtimeSessionID !== this.#job.runtimeSessionID) return
  1177. const payload = record(params.payload)
  1178. const expectedRunID = string(this.#job.metadata?.runID)
  1179. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  1180. if (expectedRunID && eventRunID !== expectedRunID) return
  1181. if (['approval.request', 'approval.requested', 'clarify.request', 'clarify.requested'].includes(eventType)) {
  1182. if (!await this.#ownershipStillAllowed()) return
  1183. if (!expectedRunID) {
  1184. let ownership: ChatTerminalContent = {}
  1185. try { ownership = await this.resolver.resolveTerminal(this.#job, eventType, payload) } catch { return }
  1186. if (ownership.correlated !== true) return
  1187. }
  1188. const candidate = chatInteractionCandidate(this.#job, eventType, payload)
  1189. if (candidate) await this.coordinator.enqueueNotification(candidate)
  1190. return
  1191. }
  1192. if (['message.complete', 'run.completed', 'run.failed', 'error'].includes(eventType)) {
  1193. await this.#finishTerminal(eventType, payload)
  1194. }
  1195. return
  1196. }
  1197. if (requestID(frame.id) !== this.#resumeRequestID) return
  1198. if (frame.error) throw new Error(errorMessage(frame.error) || 'Unable to resume chat push job')
  1199. const result = record(frame.result)
  1200. const runtimeSessionID = string(result.session_id ?? result.sessionId)
  1201. const storedSessionID = string(result.stored_session_id ?? result.storedSessionId ?? result.session_key)
  1202. || this.#job.storedSessionID
  1203. const resumedRunID = string(result.run_id ?? result.runId ?? result.turn_id ?? result.turnId)
  1204. const expectedRunID = string(this.#job.metadata?.runID)
  1205. if (!runtimeSessionID) throw new Error('Hermes resume did not return a runtime session ID')
  1206. const promptDigest = string(this.#job.metadata?.promptDigest)
  1207. if ((!expectedRunID && !promptDigest) || (expectedRunID && resumedRunID && resumedRunID !== expectedRunID)) {
  1208. this.#socket?.close(1012, 'push run ownership is not confirmed')
  1209. return
  1210. }
  1211. this.#job = {
  1212. ...this.#job,
  1213. runtimeSessionID,
  1214. storedSessionID,
  1215. phase: 'watching',
  1216. metadata: {
  1217. ...this.#job.metadata,
  1218. ...(expectedRunID ? { runID: expectedRunID } : {}),
  1219. },
  1220. }
  1221. await this.coordinator.saveChatJob(this.#job)
  1222. const approval = record(result.pending_approval)
  1223. const clarification = record(result.pending_clarify)
  1224. const pending = string(approval.request_id ?? approval.requestId ?? approval.id)
  1225. ? chatInteractionCandidate(this.#job, 'approval.requested', approval)
  1226. : string(clarification.request_id ?? clarification.requestId ?? clarification.id)
  1227. ? chatInteractionCandidate(this.#job, 'clarify.requested', clarification)
  1228. : undefined
  1229. if (pending) {
  1230. if (!await this.#ownershipStillAllowed()) return
  1231. if (!expectedRunID) {
  1232. let ownership: ChatTerminalContent = {}
  1233. try { ownership = await this.resolver.resolveTerminal(this.#job, 'interaction.requested', result) } catch { return }
  1234. if (ownership.correlated !== true) return
  1235. }
  1236. await this.coordinator.enqueueNotification(pending)
  1237. return
  1238. }
  1239. const inflight = record(result.inflight ?? result.active_run)
  1240. const queued = record(result.queued)
  1241. const running = result.running === true || inflight.streaming === true
  1242. || Boolean(string(queued.user ?? queued.text))
  1243. if (running && expectedRunID && !resumedRunID) {
  1244. this.#socket?.close(1012, 'active push run lacks ownership identity')
  1245. return
  1246. }
  1247. if (running) this.#retryAttempt = 0
  1248. if (!running) {
  1249. const resumeError = string(result.error ?? inflight.error)
  1250. if (resumeError) {
  1251. await this.#finishTerminal('run.failed', { ...result, error: resumeError })
  1252. return
  1253. }
  1254. let resolved: ChatTerminalContent = {}
  1255. try { resolved = await this.resolver.resolveTerminal(this.#job, 'run.completed', result) } catch { /* retry below */ }
  1256. if (resolved.confirmed !== true) {
  1257. this.#socket?.close(1012, 'awaiting authoritative terminal message')
  1258. return
  1259. }
  1260. await this.#finishTerminal(
  1261. resolved.failed ? 'run.failed' : 'run.completed',
  1262. resolved.failed ? { ...result, error: resolved.error ?? resolved.body } : result,
  1263. resolved,
  1264. )
  1265. }
  1266. }).catch(() => {
  1267. if (!this.#running || generation !== this.#generation) return
  1268. this.#socket?.close(1011, 'chat push watcher frame failed')
  1269. this.#scheduleReconnect(generation)
  1270. })
  1271. }
  1272. async #finishTerminal(eventType: string, payload: JsonRecord, resolved?: ChatTerminalContent): Promise<void> {
  1273. if (!this.#running) return
  1274. if (!await this.#ownershipStillAllowed()) {
  1275. this.stop()
  1276. this.#notifyFinished()
  1277. return
  1278. }
  1279. let ownership = resolved
  1280. if (!ownership) {
  1281. try { ownership = await this.resolver.resolveTerminal(this.#job, eventType, payload) } catch { ownership = {} }
  1282. }
  1283. const expectedRunID = string(this.#job.metadata?.runID)
  1284. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  1285. const ownedByRun = Boolean(expectedRunID && eventRunID === expectedRunID)
  1286. const failed = eventType === 'run.failed' || eventType === 'error' || Boolean(payload.error)
  1287. if (ownership.confirmed !== true && !ownedByRun && !(failed && ownership.correlated === true)) {
  1288. this.#socket?.close(1012, 'terminal push ownership is not confirmed')
  1289. return
  1290. }
  1291. const candidate = await chatTerminalCandidate(this.#job, eventType, payload, this.resolver, ownership)
  1292. await this.coordinator.enqueueNotification(candidate)
  1293. await this.coordinator.completeChatJob(this.#job.id)
  1294. this.stop()
  1295. this.#notifyFinished()
  1296. }
  1297. #scheduleReconnect(generation: number): void {
  1298. if (!this.#running || generation !== this.#generation || this.#retryTimer) return
  1299. this.#socket = undefined
  1300. const delay = Math.min(60_000, 1_000 * 2 ** Math.min(this.#retryAttempt++, 6))
  1301. this.#retryTimer = setTimeout(() => {
  1302. this.#retryTimer = undefined
  1303. if (this.#running && generation === this.#generation) void this.#connect(generation)
  1304. }, delay)
  1305. this.#retryTimer.unref()
  1306. }
  1307. async #ownershipStillAllowed(): Promise<boolean> {
  1308. if (!this.coordinator.canRecoverChatJob) return true
  1309. try { return await this.coordinator.canRecoverChatJob(this.#job) } catch { return false }
  1310. }
  1311. #notifyFinished(): void {
  1312. if (this.#finished) return
  1313. this.#finished = true
  1314. this.onFinished?.(this.#job.id)
  1315. }
  1316. }
  1317. export interface ChatPushJobManagerOptions {
  1318. transportFactory?: ChatPushTransportFactory
  1319. maximumConcurrent?: number
  1320. reconcileMilliseconds?: number
  1321. now?: () => number
  1322. resolver?: ChatNotificationResolver
  1323. }
  1324. /** Starts and bounds the server-owned recovery sockets for persisted jobs. */
  1325. export class ChatPushJobManager {
  1326. private transportFactory?: ChatPushTransportFactory
  1327. private brokerOwnsJob?: (job: ChatPushJob) => boolean
  1328. readonly #watchers = new Map<string, HermesChatPushJobWatcher>()
  1329. readonly #maximumConcurrent: number
  1330. readonly #reconcileMilliseconds: number
  1331. readonly #now: () => number
  1332. readonly #resolver: ChatNotificationResolver
  1333. #running = false
  1334. #timer?: ReturnType<typeof setInterval>
  1335. #activeReconcile?: Promise<void>
  1336. constructor(
  1337. readonly config: ServerConfig,
  1338. readonly upstreamSession: UpstreamServiceSession,
  1339. readonly coordinator: PushEventCoordinator,
  1340. options: ChatPushJobManagerOptions = {},
  1341. ) {
  1342. this.#maximumConcurrent = Math.max(1, Math.min(64, Math.trunc(options.maximumConcurrent ?? 16)))
  1343. this.#reconcileMilliseconds = Math.max(250, Math.trunc(options.reconcileMilliseconds ?? 1_000))
  1344. this.#now = options.now ?? Date.now
  1345. this.#resolver = options.resolver ?? new HermesChatNotificationResolver(upstreamSession)
  1346. this.transportFactory = options.transportFactory
  1347. }
  1348. setTransportFactory(factory: ChatPushTransportFactory, ownsJob?: (job: ChatPushJob) => boolean): void {
  1349. this.transportFactory = factory; this.brokerOwnsJob = ownsJob
  1350. }
  1351. start(): void {
  1352. if (this.#running) return
  1353. this.#running = true
  1354. void this.reconcile().catch(() => undefined)
  1355. this.#timer = setInterval(() => {
  1356. void this.reconcile().catch(() => undefined)
  1357. }, this.#reconcileMilliseconds)
  1358. this.#timer.unref()
  1359. }
  1360. stop(): void {
  1361. this.#running = false
  1362. if (this.#timer) clearInterval(this.#timer)
  1363. this.#timer = undefined
  1364. for (const watcher of this.#watchers.values()) watcher.stop()
  1365. this.#watchers.clear()
  1366. }
  1367. reconcile(): Promise<void> {
  1368. if (this.#activeReconcile) return this.#activeReconcile
  1369. const operation = this.#reconcile()
  1370. const tracked = operation.finally(() => {
  1371. if (this.#activeReconcile === tracked) this.#activeReconcile = undefined
  1372. })
  1373. this.#activeReconcile = tracked
  1374. return tracked
  1375. }
  1376. async #reconcile(): Promise<void> {
  1377. if (!this.#running) return
  1378. const pending = [...await this.coordinator.pendingChatJobs()]
  1379. const now = this.#now()
  1380. for (const job of pending) {
  1381. if (job.expiresAt <= now) await this.coordinator.completeChatJob(job.id)
  1382. }
  1383. const jobs: ChatPushJob[] = []
  1384. for (const job of pending) {
  1385. const structurallyEligible = job.expiresAt > now
  1386. && job.metadata?.baselineCaptured === true
  1387. && (Boolean(this.transportFactory) || Number.isFinite(Number(job.metadata?.disconnectedAt)))
  1388. && Boolean(string(job.metadata?.runID) || string(job.metadata?.promptDigest))
  1389. && (job.phase === 'submitted' || job.phase === 'accepted' || job.phase === 'watching')
  1390. if (!structurallyEligible) continue
  1391. if (this.brokerOwnsJob?.(job)) continue
  1392. if (this.coordinator.canRecoverChatJob
  1393. && !await this.coordinator.canRecoverChatJob(job)) continue
  1394. jobs.push(job)
  1395. }
  1396. jobs.sort((left, right) => left.submittedAt - right.submittedAt)
  1397. if (!this.#running) return
  1398. const pendingIDs = new Set(jobs.map(job => job.id))
  1399. for (const [jobID, watcher] of this.#watchers) {
  1400. if (pendingIDs.has(jobID)) continue
  1401. watcher.stop()
  1402. this.#watchers.delete(jobID)
  1403. }
  1404. for (const job of jobs) {
  1405. if (!this.#running || this.#watchers.size >= this.#maximumConcurrent) break
  1406. if (this.#watchers.has(job.id)) continue
  1407. const watcher = new HermesChatPushJobWatcher(
  1408. job,
  1409. this.config,
  1410. this.upstreamSession,
  1411. this.coordinator,
  1412. this.#resolver,
  1413. this.#now,
  1414. jobID => {
  1415. this.#watchers.delete(jobID)
  1416. if (this.#running) void this.reconcile()
  1417. },
  1418. this.transportFactory,
  1419. )
  1420. this.#watchers.set(job.id, watcher)
  1421. watcher.start()
  1422. }
  1423. }
  1424. }
  1425. export class HermesGroupEventSource implements GroupEventSource {
  1426. constructor(
  1427. readonly config: ServerConfig,
  1428. readonly upstreamSession: UpstreamServiceSession,
  1429. ) {}
  1430. async currentAnchor(): Promise<GroupWatchAnchor> {
  1431. const response = await this.upstreamSession.request('/api/plugins/yaoyao/v1/capabilities')
  1432. if (response.status < 200 || response.status >= 300) throw new Error(`Yaoyao capabilities failed (${response.status})`)
  1433. const payload = parseUpstreamJSON(response.body)
  1434. const protocolVersion = Number(payload.protocolVersion ?? payload.protocol_version)
  1435. if (!isSupportedGroupProtocolVersion(protocolVersion)) throw new Error('Unsupported Yaoyao group protocol')
  1436. return canonicalAnchor({
  1437. epoch: string(payload.journalEpoch ?? payload.journal_epoch),
  1438. cursor: Number(payload.latestCursor ?? payload.latest_cursor),
  1439. })
  1440. }
  1441. async connect(
  1442. anchor: GroupWatchAnchor,
  1443. handlers: { onEnvelope(envelope: GroupEventEnvelope): void; onClose(error?: Error): void },
  1444. ): Promise<GroupEventConnection> {
  1445. const ticketResponse = await this.upstreamSession.request('/api/auth/ws-ticket', { method: 'POST' })
  1446. if (ticketResponse.status < 200 || ticketResponse.status >= 300) {
  1447. throw new Error(`Hermes WebSocket ticket failed (${ticketResponse.status})`)
  1448. }
  1449. const ticket = string(parseUpstreamJSON(ticketResponse.body).ticket)
  1450. if (!ticket) throw new Error('Hermes returned an empty WebSocket ticket')
  1451. const url = new URL(this.config.upstream)
  1452. url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'
  1453. const prefix = this.config.upstream.pathname === '/' ? '' : this.config.upstream.pathname.replace(/\/$/, '')
  1454. url.pathname = `${prefix}/api/plugins/yaoyao/v1/events`
  1455. url.search = ''
  1456. url.searchParams.set('ticket', ticket)
  1457. url.searchParams.set('epoch', anchor.epoch)
  1458. url.searchParams.set('cursor', String(anchor.cursor))
  1459. return await new Promise<GroupEventConnection>((resolve, reject) => {
  1460. const socket = new WebSocket(url, {
  1461. headers: { Origin: this.config.upstream.origin },
  1462. maxPayload: 2 * 1_024 * 1_024,
  1463. handshakeTimeout: 15_000,
  1464. })
  1465. let opened = false
  1466. let closed = false
  1467. socket.once('open', () => {
  1468. opened = true
  1469. resolve({ close: () => socket.close(1000, 'group push watcher close') })
  1470. })
  1471. socket.on('message', (data, isBinary) => {
  1472. if (isBinary) {
  1473. socket.close(1008, 'binary group frame')
  1474. return
  1475. }
  1476. try {
  1477. const value = record(JSON.parse(Buffer.from(data as Uint8Array).toString('utf8')))
  1478. handlers.onEnvelope({
  1479. type: string(value.type) as GroupEventEnvelope['type'],
  1480. epoch: string(value.epoch) || undefined,
  1481. cursor: value.cursor == null ? undefined : Number(value.cursor),
  1482. heartbeatSeconds: value.heartbeatSeconds == null && value.heartbeat_seconds == null
  1483. ? undefined : Number(value.heartbeatSeconds ?? value.heartbeat_seconds),
  1484. roomId: string(value.roomId ?? value.room_id) || undefined,
  1485. event: string(value.event) || undefined,
  1486. payload: value.payload,
  1487. reason: string(value.reason) || undefined,
  1488. })
  1489. } catch {
  1490. socket.close(1008, 'malformed group frame')
  1491. }
  1492. })
  1493. socket.once('unexpected-response', (_request, response) => {
  1494. const error = new Error(`Hermes rejected group WebSocket (${response.statusCode ?? 0})`)
  1495. response.resume()
  1496. if (!opened) reject(error)
  1497. })
  1498. socket.once('error', (error) => {
  1499. if (!opened) reject(error)
  1500. })
  1501. socket.once('close', (code, reason) => {
  1502. if (closed) return
  1503. closed = true
  1504. const error = code === 1000 ? undefined : new Error(reason.toString() || `Group WebSocket closed (${code})`)
  1505. if (!opened) reject(error ?? new Error('Group WebSocket closed before opening'))
  1506. else handlers.onClose(error)
  1507. })
  1508. })
  1509. }
  1510. }
  1511. export class GroupPushEventWatcher {
  1512. #running = false
  1513. #generation = 0
  1514. #retryAttempt = 0
  1515. #retryTimer?: ReturnType<typeof setTimeout>
  1516. #connection?: GroupEventConnection
  1517. #processor?: GroupPushEventProcessor
  1518. #frames = Promise.resolve()
  1519. constructor(
  1520. readonly source: GroupEventSource,
  1521. readonly coordinator: PushEventCoordinator,
  1522. ) {}
  1523. start(): void {
  1524. if (this.#running) return
  1525. this.#running = true
  1526. this.#generation += 1
  1527. void this.#open(this.#generation)
  1528. }
  1529. stop(): void {
  1530. this.#running = false
  1531. this.#generation += 1
  1532. if (this.#retryTimer) clearTimeout(this.#retryTimer)
  1533. this.#retryTimer = undefined
  1534. this.#connection?.close()
  1535. this.#connection = undefined
  1536. }
  1537. async #open(generation: number): Promise<void> {
  1538. try {
  1539. let anchor = await this.coordinator.groupWatchAnchor()
  1540. if (!anchor) {
  1541. anchor = await this.source.currentAnchor()
  1542. const restored = await this.coordinator.resetGroupCursor({ ...anchor, reason: 'initial' })
  1543. anchor = restored ?? anchor
  1544. }
  1545. if (!this.#running || generation !== this.#generation) return
  1546. this.#processor = new GroupPushEventProcessor(anchor, this.coordinator)
  1547. const connection = await this.source.connect(anchor, {
  1548. onEnvelope: envelope => this.#queueEnvelope(generation, envelope),
  1549. onClose: () => this.#scheduleReconnect(generation),
  1550. })
  1551. if (!this.#running || generation !== this.#generation || this.#connection) {
  1552. connection.close()
  1553. return
  1554. }
  1555. this.#connection = connection
  1556. this.#retryAttempt = 0
  1557. } catch {
  1558. this.#scheduleReconnect(generation)
  1559. }
  1560. }
  1561. #queueEnvelope(generation: number, envelope: GroupEventEnvelope): void {
  1562. this.#frames = this.#frames.then(async () => {
  1563. if (!this.#running || generation !== this.#generation || !this.#processor) return
  1564. const result = await this.#processor.accept(envelope)
  1565. if (result === 'reset') {
  1566. this.#connection?.close()
  1567. this.#connection = undefined
  1568. this.#scheduleReconnect(generation, 0)
  1569. }
  1570. }).catch(() => {
  1571. if (!this.#running || generation !== this.#generation) return
  1572. this.#connection?.close()
  1573. this.#connection = undefined
  1574. this.#scheduleReconnect(generation)
  1575. })
  1576. }
  1577. #scheduleReconnect(generation: number, delay?: number): void {
  1578. if (!this.#running || generation !== this.#generation || this.#retryTimer) return
  1579. this.#connection = undefined
  1580. const wait = delay ?? Math.min(15_000, 500 * 2 ** Math.min(this.#retryAttempt++, 5))
  1581. this.#retryTimer = setTimeout(() => {
  1582. this.#retryTimer = undefined
  1583. if (this.#running && generation === this.#generation) void this.#open(generation)
  1584. }, wait)
  1585. this.#retryTimer.unref()
  1586. }
  1587. }