pushEvents.ts 55 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416
  1. import { createHash, randomUUID } from 'node:crypto'
  2. import WebSocket, { type RawData } from 'ws'
  3. import { isSupportedGroupProtocolVersion } from '../shared/types.js'
  4. import type { ServerConfig } from './config.js'
  5. import type { UpstreamServiceSession } from './localAuth.js'
  6. import type { PushCoordinator as PersistentPushCoordinator } from './pushCoordinator.js'
  7. import { notificationPlainText } from './notificationText.js'
  8. type MaybePromise<T> = T | Promise<T>
  9. type JsonRecord = Record<string, unknown>
  10. export interface ChatRelayIdentity {
  11. localUserID: string
  12. connectionID?: string
  13. accountKey?: string
  14. source: 'web' | 'gateway' | 'paired'
  15. }
  16. interface ChatObservationBase {
  17. localUserID: string
  18. connectionID: string
  19. accountKey?: string
  20. source: ChatRelayIdentity['source']
  21. observedAt: number
  22. }
  23. export type ChatPushObservation = ChatObservationBase & (
  24. | {
  25. type: 'chat.session_opened'
  26. action: 'create' | 'resume'
  27. profile: string
  28. runtimeSessionID: string
  29. storedSessionID: string
  30. }
  31. | {
  32. type: 'chat.prompt'
  33. phase: 'submitted' | 'accepted' | 'rejected'
  34. requestID: string
  35. profile?: string
  36. runtimeSessionID: string
  37. storedSessionID?: string
  38. queued: boolean
  39. status?: string
  40. error?: string
  41. }
  42. | {
  43. type: 'chat.rpc_event'
  44. eventType: string
  45. profile?: string
  46. runtimeSessionID?: string
  47. storedSessionID?: string
  48. payload: JsonRecord
  49. }
  50. | {
  51. type: 'chat.disconnected'
  52. activeSessions: Array<{
  53. profile: string
  54. runtimeSessionID: string
  55. storedSessionID: string
  56. }>
  57. }
  58. )
  59. type ChatObservationPayload = ChatPushObservation extends infer Observation
  60. ? Observation extends ChatObservationBase
  61. ? Omit<Observation, keyof ChatObservationBase>
  62. : never
  63. : never
  64. export interface GroupWatchAnchor {
  65. epoch: string
  66. cursor: number
  67. }
  68. export interface GroupWatchReset extends GroupWatchAnchor {
  69. reason: string
  70. }
  71. export interface ChatPushJob {
  72. id: string
  73. localUserID: string
  74. profile?: string
  75. runtimeSessionID: string
  76. storedSessionID?: string
  77. requestID: string
  78. queued: boolean
  79. phase: 'submitted' | 'accepted' | 'watching'
  80. submittedAt: number
  81. expiresAt: number
  82. metadata?: Record<string, unknown>
  83. }
  84. interface NotificationCandidateBase {
  85. eventID: string
  86. localUserID: string
  87. title: string
  88. body: string
  89. collapseID: string
  90. data: Record<string, string>
  91. }
  92. export interface ChatPushCandidate extends NotificationCandidateBase {
  93. kind: 'chat.completed' | 'chat.failed' | 'chat.approval.requested' | 'chat.clarification.requested'
  94. jobID: string
  95. profile?: string
  96. sessionID?: string
  97. requestID?: string
  98. }
  99. export interface GroupPushCandidate {
  100. eventID: NotificationCandidateBase['eventID']
  101. localUserID: NotificationCandidateBase['localUserID']
  102. title: NotificationCandidateBase['title']
  103. body: NotificationCandidateBase['body']
  104. collapseID: NotificationCandidateBase['collapseID']
  105. data: NotificationCandidateBase['data']
  106. kind: 'group.message.completed' | 'group.message.failed' | 'group.interaction.requested'
  107. epoch?: string
  108. cursor?: number
  109. roomID: string
  110. topicID?: string
  111. messageID?: string
  112. messageSequence?: number
  113. interactionID?: string
  114. interactionKind?: 'approval' | 'clarification' | 'unknown'
  115. senderName?: string
  116. content?: string
  117. error?: string
  118. occurredAt?: number
  119. }
  120. export type PushNotificationCandidate = ChatPushCandidate | GroupPushCandidate
  121. /**
  122. * Persistence and delivery boundary used by the relay and the group watcher.
  123. * Implementations must durably deduplicate `enqueueNotification` by
  124. * `localUserID + eventID`. `advanceGroupCursor` is called only after every
  125. * candidate for that cursor has been durably accepted by the outbox.
  126. */
  127. export interface PushEventCoordinator {
  128. observeChat?(observation: ChatPushObservation): MaybePromise<void>
  129. saveChatJob(job: ChatPushJob): MaybePromise<void>
  130. completeChatJob(jobID: string): MaybePromise<void>
  131. pendingChatJobs(): MaybePromise<readonly ChatPushJob[]>
  132. recoverChatJob?(job: ChatPushJob): MaybePromise<void>
  133. promptDigest?(localUserID: string, prompt: string): MaybePromise<string>
  134. canRecoverChatJob?(job: ChatPushJob): MaybePromise<boolean>
  135. enqueueNotification(candidate: PushNotificationCandidate): MaybePromise<'enqueued' | 'duplicate' | 'ignored' | void>
  136. }
  137. /** Keep the event pipeline decoupled from the concrete persistent coordinator. */
  138. export function createPushEventCoordinator(push: PersistentPushCoordinator): PushEventCoordinator {
  139. return {
  140. saveChatJob: job => { push.saveChatJob(job) },
  141. completeChatJob: jobID => { push.completeChatJob(jobID) },
  142. pendingChatJobs: () => push.pendingChatJobs(),
  143. promptDigest: (localUserID, prompt) => push.promptDigest(localUserID, prompt),
  144. canRecoverChatJob: job => push.chatJobRecoveryAllowed(job),
  145. enqueueNotification: candidate => push.enqueueNotification(candidate),
  146. }
  147. }
  148. function record(value: unknown): JsonRecord {
  149. return value && typeof value === 'object' && !Array.isArray(value)
  150. ? value as JsonRecord
  151. : {}
  152. }
  153. function string(value: unknown): string {
  154. return typeof value === 'string' ? value.trim() : ''
  155. }
  156. function requestID(value: unknown): string {
  157. return typeof value === 'string' || typeof value === 'number' ? String(value) : ''
  158. }
  159. function errorMessage(value: unknown): string {
  160. const source = record(value)
  161. return string(source.message ?? source.error) || (value == null ? '' : String(value))
  162. }
  163. function summary(value: unknown, fallback: string): string {
  164. return notificationPlainText(value, { fallback, maximum: 180 })
  165. }
  166. function eventTimestamp(value: unknown): number | undefined {
  167. if (typeof value === 'number' && Number.isFinite(value)) {
  168. return value < 10_000_000_000 ? value * 1_000 : value
  169. }
  170. if (typeof value !== 'string' || !value.trim()) return undefined
  171. const numeric = Number(value)
  172. if (Number.isFinite(numeric)) return numeric < 10_000_000_000 ? numeric * 1_000 : numeric
  173. const parsed = Date.parse(value)
  174. return Number.isNaN(parsed) ? undefined : parsed
  175. }
  176. function collapseID(prefix: string, value: string): string {
  177. return `${prefix}:${createHash('sha256').update(value, 'utf8').digest('base64url').slice(0, 32)}`
  178. }
  179. function notificationData(
  180. eventID: string,
  181. kind: PushNotificationCandidate['kind'],
  182. localUserID: string,
  183. values: Record<string, string | undefined>,
  184. ): Record<string, string> {
  185. return Object.fromEntries(Object.entries({
  186. version: '1', eventId: eventID, kind, clientAccountId: localUserID, ...values,
  187. }).filter((entry): entry is [string, string] => typeof entry[1] === 'string' && Boolean(entry[1])))
  188. }
  189. interface SessionIdentity {
  190. profile: string
  191. runtimeSessionID: string
  192. storedSessionID: string
  193. baseline?: ChatMessageBaseline
  194. baselinePromise?: Promise<ChatMessageBaseline>
  195. }
  196. interface PendingSessionOpen {
  197. action: 'create' | 'resume'
  198. profile: string
  199. requestedStoredSessionID: string
  200. }
  201. interface PendingPrompt {
  202. requestID: string
  203. runtimeSessionID: string
  204. storedSessionID?: string
  205. profile?: string
  206. queued: boolean
  207. job: ChatPushJob
  208. }
  209. export interface ChatTerminalContent {
  210. title?: string
  211. body?: string
  212. messageID?: string
  213. timestamp?: number
  214. confirmed?: boolean
  215. failed?: boolean
  216. error?: string
  217. correlated?: boolean
  218. }
  219. export interface ChatMessageBaseline {
  220. messageID?: string
  221. sequence?: number
  222. total: number
  223. assistantCount: number
  224. lastRowSequence?: number
  225. }
  226. export interface ChatNotificationResolver {
  227. captureBaseline?(job: ChatPushJob): Promise<ChatMessageBaseline>
  228. resolveTerminal(job: ChatPushJob, eventType: string, payload: JsonRecord): Promise<ChatTerminalContent>
  229. }
  230. /** Observes one already-authorized chat relay without changing its protocol. */
  231. export class ChatPushRelayObserver {
  232. readonly #connectionID: string
  233. readonly #sessions = new Map<string, SessionIdentity>()
  234. readonly #pendingSessionOpens = new Map<string, PendingSessionOpen>()
  235. readonly #pendingPrompts = new Map<string, PendingPrompt>()
  236. readonly #jobsByRuntimeSession = new Map<string, ChatPushJob[]>()
  237. readonly #terminalJobIDs = new Set<string>()
  238. #closed = false
  239. #delivery = Promise.resolve()
  240. constructor(
  241. readonly coordinator: PushEventCoordinator,
  242. readonly identity: ChatRelayIdentity,
  243. readonly now: () => number = Date.now,
  244. readonly notificationResolver?: ChatNotificationResolver,
  245. ) {
  246. this.#connectionID = identity.connectionID ?? randomUUID().toLowerCase()
  247. }
  248. observeClientFrame(frameText: string): void {
  249. if (this.#closed) return
  250. let frame: JsonRecord
  251. try { frame = record(JSON.parse(frameText)) } catch { return }
  252. const method = string(frame.method)
  253. const id = requestID(frame.id)
  254. const params = record(frame.params)
  255. if ((method === 'session.create' || method === 'session.resume') && id) {
  256. this.#pendingSessionOpens.set(id, {
  257. action: method === 'session.create' ? 'create' : 'resume',
  258. profile: string(params.profile),
  259. requestedStoredSessionID: method === 'session.resume' ? string(params.session_id) : '',
  260. })
  261. return
  262. }
  263. if (method !== 'prompt.submit' || !id) return
  264. const runtimeSessionID = string(params.session_id)
  265. if (!runtimeSessionID) return
  266. const session = this.#sessions.get(runtimeSessionID)
  267. let job: ChatPushJob = {
  268. id: randomUUID().toLowerCase(),
  269. localUserID: this.identity.localUserID,
  270. profile: session?.profile,
  271. runtimeSessionID,
  272. storedSessionID: session?.storedSessionID,
  273. requestID: id,
  274. queued: params.queued === true,
  275. phase: 'submitted',
  276. submittedAt: this.now(),
  277. expiresAt: this.now() + 24 * 60 * 60 * 1_000,
  278. }
  279. if (this.coordinator.promptDigest) {
  280. try {
  281. const digest = this.coordinator.promptDigest(
  282. this.identity.localUserID,
  283. string(params.text),
  284. )
  285. if (typeof digest === 'string' && digest) {
  286. job = { ...job, metadata: { ...job.metadata, promptDigest: digest } }
  287. }
  288. } catch { /* Missing correlation fails closed without blocking chat. */ }
  289. }
  290. if (session?.baseline) {
  291. const baseline = session.baseline
  292. job = {
  293. ...job,
  294. metadata: {
  295. ...job.metadata,
  296. baselineCaptured: true,
  297. baselineMessageID: baseline.messageID,
  298. baselineSequence: baseline.sequence,
  299. baselineTotal: baseline.total,
  300. baselineAssistantCount: baseline.assistantCount,
  301. baselineRowSequence: baseline.lastRowSequence,
  302. },
  303. }
  304. } else {
  305. job = { ...job, metadata: { ...job.metadata, baselineCaptured: false } }
  306. }
  307. const prompt: PendingPrompt = {
  308. requestID: id,
  309. runtimeSessionID,
  310. storedSessionID: session?.storedSessionID,
  311. profile: session?.profile,
  312. queued: params.queued === true,
  313. job,
  314. }
  315. this.#pendingPrompts.set(id, prompt)
  316. this.#jobsByRuntimeSession.set(runtimeSessionID, [
  317. ...(this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []),
  318. prompt.job,
  319. ])
  320. this.#enqueue(() => this.coordinator.saveChatJob(prompt.job))
  321. if (!session?.baseline && session?.baselinePromise) {
  322. void session.baselinePromise.then(baseline => {
  323. this.#applyBaseline(runtimeSessionID, prompt.job.id, baseline)
  324. }).catch(() => undefined)
  325. }
  326. this.#emit({
  327. type: 'chat.prompt',
  328. phase: 'submitted',
  329. requestID: prompt.requestID,
  330. runtimeSessionID: prompt.runtimeSessionID,
  331. storedSessionID: prompt.storedSessionID,
  332. profile: prompt.profile,
  333. queued: prompt.queued,
  334. })
  335. }
  336. ownsJob(jobID: string): boolean {
  337. return !this.#closed && [...this.#jobsByRuntimeSession.values()].some(jobs => jobs.some(job => job.id === jobID))
  338. }
  339. observeUpstreamFrame(data: RawData, isBinary: boolean): void {
  340. if (this.#closed || isBinary) return
  341. let frame: JsonRecord
  342. try { frame = record(JSON.parse(Buffer.from(data as Uint8Array).toString('utf8'))) } catch { return }
  343. const id = requestID(frame.id)
  344. if (id) {
  345. const pendingOpen = this.#pendingSessionOpens.get(id)
  346. if (pendingOpen) {
  347. this.#pendingSessionOpens.delete(id)
  348. if (!frame.error) {
  349. const result = record(frame.result)
  350. const runtimeSessionID = string(result.session_id ?? result.sessionId)
  351. const storedSessionID = string(
  352. result.stored_session_id ?? result.storedSessionId ?? result.session_key,
  353. ) || pendingOpen.requestedStoredSessionID
  354. if (runtimeSessionID && storedSessionID && pendingOpen.profile) {
  355. const session: SessionIdentity = {
  356. profile: pendingOpen.profile,
  357. runtimeSessionID,
  358. storedSessionID,
  359. ...(pendingOpen.action === 'create'
  360. ? { baseline: { total: 0, assistantCount: 0, lastRowSequence: 0 } }
  361. : {}),
  362. }
  363. this.#sessions.set(runtimeSessionID, session)
  364. if (pendingOpen.action === 'resume' && this.notificationResolver?.captureBaseline) {
  365. const baselineJob: ChatPushJob = {
  366. id: `baseline:${this.#connectionID}:${runtimeSessionID}`,
  367. localUserID: this.identity.localUserID,
  368. profile: session.profile,
  369. runtimeSessionID,
  370. storedSessionID,
  371. requestID: 'baseline',
  372. queued: false,
  373. phase: 'submitted',
  374. submittedAt: this.now(),
  375. expiresAt: this.now() + 60_000,
  376. }
  377. const baselinePromise = this.notificationResolver.captureBaseline(baselineJob)
  378. session.baselinePromise = baselinePromise
  379. void baselinePromise.then(baseline => {
  380. if (this.#sessions.get(runtimeSessionID) === session) session.baseline = baseline
  381. }).catch(() => undefined)
  382. }
  383. this.#emit({
  384. type: 'chat.session_opened',
  385. profile: session.profile,
  386. runtimeSessionID,
  387. storedSessionID,
  388. action: pendingOpen.action,
  389. })
  390. }
  391. }
  392. }
  393. const pendingPrompt = this.#pendingPrompts.get(id)
  394. if (pendingPrompt) {
  395. this.#pendingPrompts.delete(id)
  396. const result = record(frame.result)
  397. const status = string(result.status) || (frame.error ? 'rejected' : 'accepted')
  398. this.#emit({
  399. type: 'chat.prompt',
  400. phase: frame.error ? 'rejected' : 'accepted',
  401. requestID: pendingPrompt.requestID,
  402. runtimeSessionID: pendingPrompt.runtimeSessionID,
  403. storedSessionID: pendingPrompt.storedSessionID,
  404. profile: pendingPrompt.profile,
  405. queued: pendingPrompt.queued,
  406. status,
  407. ...(frame.error ? { error: errorMessage(frame.error) || 'Hermes rejected the prompt' } : {}),
  408. })
  409. if (this.#terminalJobIDs.has(pendingPrompt.job.id)) {
  410. // A terminal event can legally overtake its JSON-RPC receipt.
  411. // Never recreate a job that is already being completed.
  412. } else if (frame.error) {
  413. this.#removeJob(pendingPrompt.job)
  414. this.#enqueue(() => this.coordinator.completeChatJob(pendingPrompt.job.id))
  415. } else {
  416. const runID = string(result.run_id ?? result.runId ?? result.turn_id ?? result.turnId)
  417. const accepted: ChatPushJob = {
  418. ...pendingPrompt.job,
  419. phase: 'accepted',
  420. ...(runID ? { metadata: { ...pendingPrompt.job.metadata, runID } } : {}),
  421. }
  422. this.#replaceJob(accepted)
  423. this.#enqueue(() => this.coordinator.saveChatJob(accepted))
  424. }
  425. }
  426. }
  427. if (string(frame.method) !== 'event') return
  428. const params = record(frame.params)
  429. const eventType = string(params.type)
  430. if (!eventType || eventType === 'gateway.ready') return
  431. const runtimeSessionID = string(params.session_id)
  432. const session = runtimeSessionID ? this.#sessions.get(runtimeSessionID) : undefined
  433. this.#emit({
  434. type: 'chat.rpc_event',
  435. eventType,
  436. profile: string(params.profile) || session?.profile || undefined,
  437. runtimeSessionID: runtimeSessionID || undefined,
  438. storedSessionID: session?.storedSessionID,
  439. payload: record(params.payload),
  440. })
  441. if (runtimeSessionID) this.#observeRPCEvent(
  442. eventType,
  443. runtimeSessionID,
  444. record(params.payload),
  445. )
  446. }
  447. disconnected(): void {
  448. if (this.#closed) return
  449. this.#closed = true
  450. this.#emit({
  451. type: 'chat.disconnected',
  452. activeSessions: [...this.#sessions.values()],
  453. })
  454. for (const jobs of this.#jobsByRuntimeSession.values()) {
  455. for (const job of [...jobs]) {
  456. if (this.#terminalJobIDs.has(job.id)) continue
  457. const disconnectedJob: ChatPushJob = {
  458. ...job,
  459. metadata: { ...job.metadata, disconnectedAt: this.now() },
  460. }
  461. this.#replaceJob(disconnectedJob)
  462. this.#enqueue(() => this.coordinator.saveChatJob(disconnectedJob))
  463. if (this.coordinator.recoverChatJob) {
  464. this.#enqueue(() => this.coordinator.recoverChatJob!(disconnectedJob))
  465. }
  466. }
  467. }
  468. }
  469. async flush(): Promise<void> {
  470. await Promise.allSettled(
  471. [...this.#sessions.values()].flatMap(session => session.baselinePromise ? [session.baselinePromise] : []),
  472. )
  473. await Promise.resolve()
  474. await this.#delivery
  475. }
  476. #emit(observation: ChatObservationPayload): void {
  477. const event = {
  478. ...observation,
  479. localUserID: this.identity.localUserID,
  480. connectionID: this.#connectionID,
  481. accountKey: this.identity.accountKey,
  482. source: this.identity.source,
  483. observedAt: this.now(),
  484. } as ChatPushObservation
  485. if (this.coordinator.observeChat) this.#enqueue(() => this.coordinator.observeChat!(event))
  486. }
  487. #enqueue(operation: () => MaybePromise<void>): void {
  488. this.#delivery = this.#delivery.then(operation).catch(() => undefined)
  489. }
  490. #replaceJob(job: ChatPushJob): void {
  491. const jobs = this.#jobsByRuntimeSession.get(job.runtimeSessionID) ?? []
  492. const index = jobs.findIndex(candidate => candidate.id === job.id)
  493. if (index >= 0) jobs[index] = job
  494. else jobs.push(job)
  495. this.#jobsByRuntimeSession.set(job.runtimeSessionID, jobs)
  496. }
  497. #applyBaseline(runtimeSessionID: string, jobID: string, baseline: ChatMessageBaseline): void {
  498. const jobs = this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []
  499. const index = jobs.findIndex(job => job.id === jobID)
  500. if (index < 0 || this.#terminalJobIDs.has(jobID)) return
  501. const current = jobs[index]!
  502. const updated: ChatPushJob = {
  503. ...current,
  504. metadata: {
  505. ...current.metadata,
  506. baselineCaptured: true,
  507. baselineMessageID: baseline.messageID,
  508. baselineSequence: baseline.sequence,
  509. baselineTotal: baseline.total,
  510. baselineAssistantCount: baseline.assistantCount,
  511. baselineRowSequence: baseline.lastRowSequence,
  512. },
  513. }
  514. jobs[index] = updated
  515. for (const pending of this.#pendingPrompts.values()) {
  516. if (pending.job.id === jobID) pending.job = updated
  517. }
  518. this.#enqueue(() => this.coordinator.saveChatJob(updated))
  519. }
  520. #removeJob(job: ChatPushJob): void {
  521. const jobs = (this.#jobsByRuntimeSession.get(job.runtimeSessionID) ?? [])
  522. .filter(candidate => candidate.id !== job.id)
  523. if (jobs.length) this.#jobsByRuntimeSession.set(job.runtimeSessionID, jobs)
  524. else this.#jobsByRuntimeSession.delete(job.runtimeSessionID)
  525. }
  526. #observeRPCEvent(eventType: string, runtimeSessionID: string, payload: JsonRecord): void {
  527. const jobs = this.#jobsByRuntimeSession.get(runtimeSessionID) ?? []
  528. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  529. let job = eventRunID
  530. ? jobs.find(candidate => string(candidate.metadata?.runID) === eventRunID)
  531. ?? jobs.find(candidate => !string(candidate.metadata?.runID))
  532. : jobs[0]
  533. if (!job) return
  534. if (eventRunID && !string(job.metadata?.runID)) {
  535. job = { ...job, metadata: { ...job.metadata, runID: eventRunID } }
  536. this.#replaceJob(job)
  537. const boundJob = job
  538. this.#enqueue(() => this.coordinator.saveChatJob(boundJob))
  539. }
  540. if (['approval.request', 'approval.requested', 'clarify.request', 'clarify.requested'].includes(eventType)) {
  541. const requestIDValue = string(payload.request_id ?? payload.requestId ?? payload.id)
  542. if (!requestIDValue) return
  543. const clarification = eventType.startsWith('clarify.')
  544. const kind: ChatPushCandidate['kind'] = clarification
  545. ? 'chat.clarification.requested' : 'chat.approval.requested'
  546. const eventID = `chat-interaction:${job.id}:${requestIDValue}`
  547. const candidate: ChatPushCandidate = {
  548. eventID,
  549. localUserID: job.localUserID,
  550. kind,
  551. jobID: job.id,
  552. profile: job.profile,
  553. sessionID: job.storedSessionID,
  554. requestID: requestIDValue,
  555. title: clarification ? '机器人需要补充信息' : '机器人请求审批',
  556. body: summary(payload.question ?? payload.message ?? payload.prompt, clarification ? '请打开夭夭继续处理' : '请打开夭夭审批'),
  557. collapseID: collapseID('chat-interaction', `${job.storedSessionID ?? runtimeSessionID}:${requestIDValue}`),
  558. data: notificationData(eventID, kind, job.localUserID, {
  559. profile: job.profile, sessionId: job.storedSessionID, requestId: requestIDValue,
  560. }),
  561. }
  562. this.#enqueue(async () => {
  563. if (!await this.#jobOwnershipAllowed(job)) return
  564. if (!eventRunID) {
  565. let ownership: ChatTerminalContent = {}
  566. try {
  567. ownership = await this.notificationResolver?.resolveTerminal(job, eventType, payload) ?? {}
  568. } catch { return }
  569. if (ownership.correlated !== true) return
  570. }
  571. await this.coordinator.enqueueNotification(candidate)
  572. })
  573. return
  574. }
  575. const terminal = ['message.complete', 'run.completed', 'run.failed', 'error'].includes(eventType)
  576. if (!terminal) return
  577. if (this.#terminalJobIDs.has(job.id)) return
  578. this.#terminalJobIDs.add(job.id)
  579. this.#enqueue(async () => {
  580. if (!await this.#jobOwnershipAllowed(job)) return
  581. let resolved: ChatTerminalContent | undefined
  582. if (!eventRunID) {
  583. try { resolved = await this.notificationResolver?.resolveTerminal(job, eventType, payload) ?? {} } catch { resolved = {} }
  584. const failed = eventType === 'run.failed' || eventType === 'error' || Boolean(payload.error)
  585. if (resolved.confirmed !== true && !(failed && resolved.correlated === true)) {
  586. this.#terminalJobIDs.delete(job.id)
  587. return
  588. }
  589. }
  590. const candidate = await chatTerminalCandidate(job, eventType, payload, this.notificationResolver, resolved)
  591. await this.coordinator.enqueueNotification(candidate)
  592. await this.coordinator.completeChatJob(job.id)
  593. this.#removeJob(job)
  594. })
  595. }
  596. async #jobOwnershipAllowed(job: ChatPushJob): Promise<boolean> {
  597. if (!this.coordinator.canRecoverChatJob) return true
  598. try { return await this.coordinator.canRecoverChatJob(job) } catch { return false }
  599. }
  600. }
  601. export interface GroupEventEnvelope {
  602. type: 'group.ready' | 'group.event' | 'group.heartbeat' | 'group.reset_required'
  603. epoch?: string
  604. cursor?: number
  605. heartbeatSeconds?: number
  606. roomId?: string
  607. event?: string
  608. payload?: unknown
  609. reason?: string
  610. }
  611. function canonicalAnchor(value: GroupWatchAnchor): GroupWatchAnchor {
  612. if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value.epoch)
  613. || !Number.isSafeInteger(value.cursor) || value.cursor < 0) {
  614. throw new Error('Invalid Yaoyao group event anchor')
  615. }
  616. return { epoch: value.epoch, cursor: value.cursor }
  617. }
  618. function eventRoomID(envelope: GroupEventEnvelope, payload: JsonRecord): string {
  619. return string(envelope.roomId ?? payload.roomId ?? payload.room_id)
  620. }
  621. function terminalGroupCandidate(
  622. envelope: GroupEventEnvelope,
  623. anchor: GroupWatchAnchor,
  624. ): Omit<GroupPushCandidate, 'localUserID' | 'data'> | undefined {
  625. if (envelope.type !== 'group.event') return undefined
  626. const payload = record(envelope.payload)
  627. const roomID = eventRoomID(envelope, payload)
  628. if (!roomID) return undefined
  629. if (envelope.event === 'message.upsert') {
  630. const senderKind = string(payload.senderKind ?? payload.sender_kind).toLowerCase()
  631. const status = string(payload.status).toLowerCase()
  632. const messageID = string(payload.id)
  633. if (senderKind !== 'agent' || !messageID || (status !== 'completed' && status !== 'failed')) return undefined
  634. const kind: GroupPushCandidate['kind'] = status === 'completed'
  635. ? 'group.message.completed' : 'group.message.failed'
  636. const eventID = `group-message:${messageID}:${status}`
  637. const content = string(payload.content)
  638. const error = string(payload.error)
  639. return {
  640. eventID,
  641. kind,
  642. epoch: anchor.epoch,
  643. cursor: anchor.cursor + 1,
  644. roomID,
  645. topicID: string(payload.topicId ?? payload.topic_id) || undefined,
  646. messageID,
  647. messageSequence: Number.isSafeInteger(Number(payload.seq ?? payload.sequence))
  648. ? Number(payload.seq ?? payload.sequence) : undefined,
  649. senderName: string(payload.senderName ?? payload.sender_name) || undefined,
  650. content: content || undefined,
  651. error: error || undefined,
  652. occurredAt: eventTimestamp(payload.updatedAt ?? payload.updated_at ?? payload.createdAt ?? payload.created_at),
  653. title: status === 'completed'
  654. ? `${string(payload.senderName ?? payload.sender_name) || '机器人'} 已回复`
  655. : `${string(payload.senderName ?? payload.sender_name) || '机器人'} 执行失败`,
  656. body: summary(error || content, status === 'completed' ? '打开夭夭查看团队消息' : '打开夭夭查看失败详情'),
  657. collapseID: collapseID('group-message', messageID),
  658. }
  659. }
  660. if (envelope.event !== 'interaction.requested') return undefined
  661. const interactionID = string(payload.id)
  662. const status = string(payload.status).toLowerCase()
  663. if (!interactionID || (status && status !== 'pending')) return undefined
  664. const rawKind = string(payload.kind).toLowerCase()
  665. const interactionKind = rawKind === 'approval' || rawKind === 'clarification' ? rawKind : 'unknown'
  666. const interactionPayload = record(payload.payload)
  667. const eventID = `group-interaction:${interactionID}`
  668. const content = string(interactionPayload.question ?? interactionPayload.message ?? interactionPayload.prompt)
  669. return {
  670. eventID,
  671. kind: 'group.interaction.requested',
  672. epoch: anchor.epoch,
  673. cursor: anchor.cursor + 1,
  674. roomID,
  675. topicID: string(payload.topicId ?? payload.topic_id) || undefined,
  676. interactionID,
  677. interactionKind,
  678. content: content || undefined,
  679. occurredAt: eventTimestamp(payload.updatedAt ?? payload.updated_at ?? payload.createdAt ?? payload.created_at),
  680. title: interactionKind === 'clarification' ? '团队机器人需要补充信息' : '团队机器人请求审批',
  681. body: summary(content, '请打开夭夭继续处理'),
  682. collapseID: collapseID('group-interaction', interactionID),
  683. }
  684. }
  685. export type GroupEnvelopeResult = 'ready' | 'heartbeat' | 'advanced' | 'ignored' | 'reset'
  686. export interface GroupEventConnection {
  687. close(): void
  688. }
  689. export interface GroupEventSource {
  690. currentAnchor(): Promise<GroupWatchAnchor>
  691. connect(
  692. anchor: GroupWatchAnchor,
  693. handlers: {
  694. onEnvelope(envelope: GroupEventEnvelope): void
  695. onClose(error?: Error): void
  696. },
  697. ): Promise<GroupEventConnection>
  698. }
  699. function parseUpstreamJSON(body: Buffer): JsonRecord {
  700. try { return record(JSON.parse(body.toString('utf8'))) } catch { throw new Error('Hermes returned invalid JSON') }
  701. }
  702. function values(value: unknown): unknown[] {
  703. if (Array.isArray(value)) return value
  704. const source = record(value)
  705. const data = source.data
  706. return Array.isArray(data) ? data : []
  707. }
  708. function messageContent(value: unknown): string {
  709. if (typeof value === 'string') return value.trim()
  710. if (!Array.isArray(value)) return ''
  711. return value.map(item => {
  712. if (typeof item === 'string') return item
  713. const part = record(item)
  714. return string(part.text ?? part.content)
  715. }).filter(Boolean).join('\n').trim()
  716. }
  717. function messageTimestamp(value: unknown): number | undefined {
  718. if (typeof value === 'number' && Number.isFinite(value)) return value < 1_000_000_000_000 ? value * 1_000 : value
  719. if (typeof value === 'string' && value.trim()) {
  720. const numeric = Number(value)
  721. if (Number.isFinite(numeric)) return numeric < 1_000_000_000_000 ? numeric * 1_000 : numeric
  722. const parsed = Date.parse(value)
  723. if (!Number.isNaN(parsed)) return parsed
  724. }
  725. return undefined
  726. }
  727. interface ChatHistoryRow {
  728. role: string
  729. content: string
  730. messageID?: string
  731. runID?: string
  732. sequence: number
  733. timestamp?: number
  734. failed: boolean
  735. error?: string
  736. }
  737. export class HermesChatNotificationResolver implements ChatNotificationResolver {
  738. private readonly titleCache = new Map<string, { value: string; expiresAt: number }>()
  739. constructor(
  740. readonly upstreamSession: UpstreamServiceSession,
  741. readonly digestPrompt?: (localUserID: string, prompt: string) => MaybePromise<string>,
  742. ) {}
  743. private async snapshot(job: ChatPushJob): Promise<{
  744. title?: string
  745. total: number
  746. rows: ChatHistoryRow[]
  747. assistants: ChatHistoryRow[]
  748. latest?: ChatHistoryRow
  749. }> {
  750. if (!job.storedSessionID) return { total: 0, rows: [], assistants: [] }
  751. const search = new URLSearchParams({
  752. offset: '0', limit: '500', order: 'latest', include_compacted: 'true',
  753. })
  754. if (job.profile) search.set('profile', job.profile)
  755. const response = await this.upstreamSession.request(
  756. `/api/sessions/${encodeURIComponent(job.storedSessionID)}/messages`,
  757. { search },
  758. )
  759. if (response.status < 200 || response.status >= 300) throw new Error(`Hermes history failed (${response.status})`)
  760. const source = parseUpstreamJSON(response.body)
  761. const session = record(source.session)
  762. const messages = values(source.messages ?? source.items ?? source.data ?? source)
  763. const rows = messages.map((raw, index): ChatHistoryRow => {
  764. const message = record(raw)
  765. const role = string(message.role ?? message.senderKind ?? message.sender_kind).toLowerCase()
  766. const content = messageContent(message.content ?? message.text ?? message.message)
  767. const error = string(message.error)
  768. const status = string(message.status).toLowerCase()
  769. const rowID = Number(message.id)
  770. const explicitSequence = Number(message.sequence ?? message.seq ?? message.index)
  771. const rawSequence = Number.isFinite(rowID)
  772. ? rowID : Number.isFinite(explicitSequence) ? explicitSequence : index
  773. return {
  774. role,
  775. content: content || error,
  776. messageID: requestID(message.id ?? message.message_id ?? message.messageId) || undefined,
  777. runID: string(message.run_id ?? message.runId ?? message.turn_id ?? message.turnId) || undefined,
  778. sequence: rawSequence,
  779. timestamp: messageTimestamp(message.timestamp ?? message.createdAt ?? message.created_at),
  780. failed: status === 'failed' || status === 'error' || Boolean(error),
  781. ...(error ? { error } : {}),
  782. }
  783. }).sort((left, right) => left.sequence - right.sequence || (left.timestamp ?? 0) - (right.timestamp ?? 0))
  784. const assistants = rows.filter(row => (row.role === 'assistant' || row.role === 'agent') && Boolean(row.content))
  785. const pagination = record(source.pagination)
  786. const totalValue = Number(pagination.total ?? source.total ?? messages.length)
  787. let title = string(session.title ?? source.title) || undefined
  788. if (!title) title = await this.sessionTitle(job)
  789. return {
  790. title,
  791. total: Number.isSafeInteger(totalValue) && totalValue >= 0 ? totalValue : messages.length,
  792. rows,
  793. assistants,
  794. latest: assistants.at(-1),
  795. }
  796. }
  797. private async sessionTitle(job: ChatPushJob): Promise<string | undefined> {
  798. if (!job.storedSessionID) return undefined
  799. const key = `${job.profile ?? 'default'}:${job.storedSessionID}`
  800. const cached = this.titleCache.get(key)
  801. if (cached && cached.expiresAt > Date.now()) return cached.value
  802. const search = new URLSearchParams()
  803. if (job.profile) search.set('profile', job.profile)
  804. try {
  805. const response = await this.upstreamSession.request(
  806. `/api/sessions/${encodeURIComponent(job.storedSessionID)}`,
  807. { search },
  808. )
  809. if (response.status < 200 || response.status >= 300) return undefined
  810. const payload = parseUpstreamJSON(response.body)
  811. const data = record(payload.data)
  812. const session = record(payload.session ?? data.session ?? payload.data ?? payload)
  813. const title = string(session.title)
  814. if (!title) return undefined
  815. this.titleCache.set(key, { value: title, expiresAt: Date.now() + 5 * 60 * 1_000 })
  816. return title
  817. } catch {
  818. return undefined
  819. }
  820. }
  821. async captureBaseline(job: ChatPushJob): Promise<ChatMessageBaseline> {
  822. if (!job.storedSessionID) {
  823. return { total: 0, assistantCount: 0, lastRowSequence: 0 }
  824. }
  825. const search = new URLSearchParams({
  826. offset: '0', limit: '1', order: 'latest', include_compacted: 'true',
  827. })
  828. if (job.profile) search.set('profile', job.profile)
  829. const response = await this.upstreamSession.request(
  830. `/api/sessions/${encodeURIComponent(job.storedSessionID)}/messages`,
  831. { search },
  832. )
  833. if (response.status < 200 || response.status >= 300) throw new Error(`Hermes history failed (${response.status})`)
  834. const source = parseUpstreamJSON(response.body)
  835. const messages = values(source.messages ?? source.items ?? source.data ?? source)
  836. const raw = messages.at(-1)
  837. const message = record(raw)
  838. const role = string(message.role ?? message.senderKind ?? message.sender_kind).toLowerCase()
  839. const rowID = Number(message.id)
  840. const explicitSequence = Number(message.sequence ?? message.seq ?? message.index)
  841. const sequence = Number.isFinite(rowID)
  842. ? rowID : Number.isFinite(explicitSequence) ? explicitSequence : 0
  843. const pagination = record(source.pagination)
  844. const totalValue = Number(pagination.total ?? source.total ?? messages.length)
  845. return {
  846. messageID: role === 'assistant' || role === 'agent'
  847. ? requestID(message.id ?? message.message_id ?? message.messageId) || undefined
  848. : undefined,
  849. sequence: role === 'assistant' || role === 'agent' ? sequence : undefined,
  850. total: Number.isSafeInteger(totalValue) && totalValue >= 0 ? totalValue : messages.length,
  851. assistantCount: role === 'assistant' || role === 'agent' ? 1 : 0,
  852. lastRowSequence: sequence,
  853. }
  854. }
  855. async resolveTerminal(job: ChatPushJob, _eventType: string, _payload: JsonRecord): Promise<ChatTerminalContent> {
  856. const snapshot = await this.snapshot(job)
  857. const expectedRunID = string(job.metadata?.runID)
  858. if (expectedRunID) {
  859. const latest = snapshot.assistants.filter(candidate => candidate.runID === expectedRunID).at(-1)
  860. const baselineSequence = Number(job.metadata?.baselineSequence)
  861. const baselineRowSequence = Number(job.metadata?.baselineRowSequence)
  862. const baselineMessageID = string(job.metadata?.baselineMessageID)
  863. const confirmed = job.metadata?.baselineCaptured === true && Boolean(latest) && (
  864. (Number.isFinite(baselineSequence) && latest!.sequence > baselineSequence)
  865. || (Number.isFinite(baselineRowSequence) && latest!.sequence > baselineRowSequence)
  866. || Boolean(baselineMessageID && latest!.messageID && latest!.messageID !== baselineMessageID)
  867. )
  868. return this.terminalContent(snapshot.title, latest, confirmed, confirmed)
  869. }
  870. const promptDigest = string(job.metadata?.promptDigest)
  871. const baselineRowSequence = Number(job.metadata?.baselineRowSequence)
  872. if (!this.digestPrompt || !promptDigest || !Number.isFinite(baselineRowSequence)) {
  873. return { title: snapshot.title, confirmed: false, correlated: false }
  874. }
  875. const matchingUsers: ChatHistoryRow[] = []
  876. for (const row of snapshot.rows) {
  877. if (row.role !== 'user' || row.sequence <= baselineRowSequence || !row.content) continue
  878. let digest = ''
  879. try { digest = await this.digestPrompt(job.localUserID, row.content) } catch { continue }
  880. if (digest === promptDigest) matchingUsers.push(row)
  881. }
  882. if (matchingUsers.length !== 1) {
  883. return { title: snapshot.title, confirmed: false, correlated: false }
  884. }
  885. const userRow = matchingUsers[0]!
  886. const nextUser = snapshot.rows.find(row => row.role === 'user' && row.sequence > userRow.sequence)
  887. const latest = snapshot.assistants
  888. .filter(row => row.sequence > userRow.sequence && (!nextUser || row.sequence < nextUser.sequence))
  889. .at(-1)
  890. return this.terminalContent(snapshot.title, latest, Boolean(latest), true)
  891. }
  892. private terminalContent(
  893. title: string | undefined,
  894. latest: ChatHistoryRow | undefined,
  895. confirmed: boolean,
  896. correlated: boolean,
  897. ): ChatTerminalContent {
  898. return {
  899. title,
  900. body: latest?.content,
  901. messageID: latest?.messageID,
  902. timestamp: latest?.timestamp,
  903. confirmed,
  904. correlated,
  905. ...(latest?.failed ? { failed: true } : {}),
  906. ...(latest?.error ? { error: latest.error } : {}),
  907. }
  908. }
  909. }
  910. function chatInteractionCandidate(
  911. job: ChatPushJob,
  912. eventType: string,
  913. payload: JsonRecord,
  914. ): ChatPushCandidate | undefined {
  915. const requestIDValue = string(payload.request_id ?? payload.requestId ?? payload.id)
  916. if (!requestIDValue) return undefined
  917. const clarification = eventType.startsWith('clarify.')
  918. || string(payload.kind).toLowerCase() === 'clarification'
  919. const kind: ChatPushCandidate['kind'] = clarification
  920. ? 'chat.clarification.requested' : 'chat.approval.requested'
  921. const eventID = `chat-interaction:${job.id}:${requestIDValue}`
  922. return {
  923. eventID,
  924. localUserID: job.localUserID,
  925. kind,
  926. jobID: job.id,
  927. profile: job.profile,
  928. sessionID: job.storedSessionID,
  929. requestID: requestIDValue,
  930. title: clarification ? '机器人需要补充信息' : '机器人请求审批',
  931. body: summary(payload.question ?? payload.message ?? payload.prompt, clarification ? '请打开夭夭继续处理' : '请打开夭夭审批'),
  932. collapseID: collapseID('chat-interaction', `${job.storedSessionID ?? job.runtimeSessionID}:${requestIDValue}`),
  933. data: notificationData(eventID, kind, job.localUserID, {
  934. profile: job.profile, sessionId: job.storedSessionID, requestId: requestIDValue,
  935. }),
  936. }
  937. }
  938. async function chatTerminalCandidate(
  939. job: ChatPushJob,
  940. eventType: string,
  941. payload: JsonRecord,
  942. resolver?: ChatNotificationResolver,
  943. suppliedContent?: ChatTerminalContent,
  944. ): Promise<ChatPushCandidate> {
  945. const terminalStatus = string(payload.status).toLowerCase()
  946. const failed = eventType === 'run.failed' || eventType === 'error'
  947. || terminalStatus === 'failed' || terminalStatus === 'error' || Boolean(payload.error)
  948. let resolved: ChatTerminalContent = suppliedContent ?? {}
  949. if (!suppliedContent) {
  950. try { resolved = await resolver?.resolveTerminal(job, eventType, payload) ?? {} } catch { /* payload fallback */ }
  951. }
  952. const kind: ChatPushCandidate['kind'] = failed ? 'chat.failed' : 'chat.completed'
  953. const eventID = `chat-terminal:${job.id}:${failed ? 'failed' : 'completed'}`
  954. const authoritativeBody = resolved.confirmed === true ? resolved.body : undefined
  955. return {
  956. eventID,
  957. localUserID: job.localUserID,
  958. kind,
  959. jobID: job.id,
  960. profile: job.profile,
  961. sessionID: job.storedSessionID,
  962. title: resolved.title || (failed ? '任务执行失败' : '任务已完成'),
  963. body: failed
  964. ? summary(payload.error ?? payload.message ?? authoritativeBody, '打开夭夭查看失败详情')
  965. : summary(authoritativeBody ?? payload.text ?? payload.message, '打开夭夭查看结果'),
  966. collapseID: collapseID('chat', job.storedSessionID ?? job.runtimeSessionID),
  967. data: notificationData(eventID, kind, job.localUserID, {
  968. profile: job.profile, sessionId: job.storedSessionID,
  969. }),
  970. }
  971. }
  972. /**
  973. * Server-owned recovery connection for one persisted chat job. The caller
  974. * controls the number of concurrently started watchers.
  975. */
  976. export interface ChatPushRecoveryTransport {
  977. start(): void
  978. send(data: string): void
  979. close(code?: number, reason?: string): void
  980. }
  981. export type ChatPushTransportFactory = (job: ChatPushJob, onFrame: (frame: string) => void, onClose: () => void) => Promise<ChatPushRecoveryTransport>
  982. export class HermesChatPushJobWatcher {
  983. #job: ChatPushJob
  984. #running = false
  985. #generation = 0
  986. #retryAttempt = 0
  987. #retryTimer?: ReturnType<typeof setTimeout>
  988. #socket?: WebSocket | ChatPushRecoveryTransport
  989. #resumeRequestID = ''
  990. #frames = Promise.resolve()
  991. #finished = false
  992. constructor(
  993. job: ChatPushJob,
  994. readonly config: ServerConfig,
  995. readonly upstreamSession: UpstreamServiceSession,
  996. readonly coordinator: PushEventCoordinator,
  997. readonly resolver: ChatNotificationResolver = new HermesChatNotificationResolver(upstreamSession),
  998. readonly now: () => number = Date.now,
  999. readonly onFinished?: (jobID: string) => void,
  1000. readonly transportFactory?: ChatPushTransportFactory,
  1001. ) {
  1002. this.#job = { ...job }
  1003. }
  1004. start(): void {
  1005. if (this.#running) return
  1006. this.#running = true
  1007. this.#generation += 1
  1008. void this.#connect(this.#generation)
  1009. }
  1010. stop(): void {
  1011. this.#running = false
  1012. this.#generation += 1
  1013. if (this.#retryTimer) clearTimeout(this.#retryTimer)
  1014. this.#retryTimer = undefined
  1015. this.#socket?.close(1000, 'chat push watcher close')
  1016. this.#socket = undefined
  1017. }
  1018. async #connect(generation: number): Promise<void> {
  1019. if (!this.#job.storedSessionID || this.#job.expiresAt <= this.now()) {
  1020. await this.coordinator.completeChatJob(this.#job.id)
  1021. this.stop()
  1022. this.#notifyFinished()
  1023. return
  1024. }
  1025. try {
  1026. if (this.transportFactory) {
  1027. const transport = await this.transportFactory(this.#job,
  1028. frame => this.#queueFrame(generation, Buffer.from(frame), false),
  1029. () => this.#scheduleReconnect(generation))
  1030. if (!this.#running || generation !== this.#generation) { transport.close(); return }
  1031. this.#socket = transport
  1032. transport.start()
  1033. return
  1034. }
  1035. const credential = await this.upstreamSession.webSocketCredential()
  1036. if (!this.#running || generation !== this.#generation) return
  1037. const url = new URL(this.config.upstream)
  1038. url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'
  1039. const prefix = this.config.upstream.pathname === '/' ? '' : this.config.upstream.pathname.replace(/\/$/, '')
  1040. url.pathname = `${prefix}/api/ws`
  1041. url.search = ''
  1042. url.searchParams.set(credential.name, credential.value)
  1043. const socket = new WebSocket(url, {
  1044. agent: this.upstreamSession.client.directAgent,
  1045. headers: { Origin: this.config.upstream.origin },
  1046. maxPayload: 36 * 1_024 * 1_024,
  1047. handshakeTimeout: 15_000,
  1048. })
  1049. this.#socket = socket
  1050. socket.on('message', (data, isBinary) => this.#queueFrame(generation, data, isBinary))
  1051. socket.once('error', () => {
  1052. if (socket.readyState !== WebSocket.OPEN) this.#scheduleReconnect(generation)
  1053. })
  1054. socket.once('close', () => this.#scheduleReconnect(generation))
  1055. } catch {
  1056. this.#scheduleReconnect(generation)
  1057. }
  1058. }
  1059. #queueFrame(generation: number, data: RawData, isBinary: boolean): void {
  1060. this.#frames = this.#frames.then(async () => {
  1061. if (!this.#running || generation !== this.#generation || isBinary) return
  1062. let frame: JsonRecord
  1063. try { frame = record(JSON.parse(Buffer.from(data as Uint8Array).toString('utf8'))) } catch { return }
  1064. if (string(frame.method) === 'event') {
  1065. const params = record(frame.params)
  1066. const eventType = string(params.type)
  1067. if (eventType === 'gateway.ready') {
  1068. this.#resumeRequestID = `push-resume:${this.#job.id}:${generation}`
  1069. this.#socket?.send(JSON.stringify({
  1070. jsonrpc: '2.0',
  1071. id: this.#resumeRequestID,
  1072. method: 'session.resume',
  1073. params: {
  1074. session_id: this.#job.storedSessionID,
  1075. profile: this.#job.profile,
  1076. source: 'web',
  1077. close_on_disconnect: false,
  1078. omit_messages: true,
  1079. cols: 80,
  1080. },
  1081. }))
  1082. return
  1083. }
  1084. const runtimeSessionID = string(params.session_id)
  1085. if (runtimeSessionID && this.#job.runtimeSessionID && runtimeSessionID !== this.#job.runtimeSessionID) return
  1086. const payload = record(params.payload)
  1087. const expectedRunID = string(this.#job.metadata?.runID)
  1088. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  1089. if (expectedRunID && eventRunID !== expectedRunID) return
  1090. if (['approval.request', 'approval.requested', 'clarify.request', 'clarify.requested'].includes(eventType)) {
  1091. if (!await this.#ownershipStillAllowed()) return
  1092. if (!expectedRunID) {
  1093. let ownership: ChatTerminalContent = {}
  1094. try { ownership = await this.resolver.resolveTerminal(this.#job, eventType, payload) } catch { return }
  1095. if (ownership.correlated !== true) return
  1096. }
  1097. const candidate = chatInteractionCandidate(this.#job, eventType, payload)
  1098. if (candidate) await this.coordinator.enqueueNotification(candidate)
  1099. return
  1100. }
  1101. if (['message.complete', 'run.completed', 'run.failed', 'error'].includes(eventType)) {
  1102. await this.#finishTerminal(eventType, payload)
  1103. }
  1104. return
  1105. }
  1106. if (requestID(frame.id) !== this.#resumeRequestID) return
  1107. if (frame.error) throw new Error(errorMessage(frame.error) || 'Unable to resume chat push job')
  1108. const result = record(frame.result)
  1109. const runtimeSessionID = string(result.session_id ?? result.sessionId)
  1110. const storedSessionID = string(result.stored_session_id ?? result.storedSessionId ?? result.session_key)
  1111. || this.#job.storedSessionID
  1112. const resumedRunID = string(result.run_id ?? result.runId ?? result.turn_id ?? result.turnId)
  1113. const expectedRunID = string(this.#job.metadata?.runID)
  1114. if (!runtimeSessionID) throw new Error('Hermes resume did not return a runtime session ID')
  1115. const promptDigest = string(this.#job.metadata?.promptDigest)
  1116. if ((!expectedRunID && !promptDigest) || (expectedRunID && resumedRunID && resumedRunID !== expectedRunID)) {
  1117. this.#socket?.close(1012, 'push run ownership is not confirmed')
  1118. return
  1119. }
  1120. this.#job = {
  1121. ...this.#job,
  1122. runtimeSessionID,
  1123. storedSessionID,
  1124. phase: 'watching',
  1125. metadata: {
  1126. ...this.#job.metadata,
  1127. ...(expectedRunID ? { runID: expectedRunID } : {}),
  1128. },
  1129. }
  1130. await this.coordinator.saveChatJob(this.#job)
  1131. const approval = record(result.pending_approval)
  1132. const clarification = record(result.pending_clarify)
  1133. const pending = string(approval.request_id ?? approval.requestId ?? approval.id)
  1134. ? chatInteractionCandidate(this.#job, 'approval.requested', approval)
  1135. : string(clarification.request_id ?? clarification.requestId ?? clarification.id)
  1136. ? chatInteractionCandidate(this.#job, 'clarify.requested', clarification)
  1137. : undefined
  1138. if (pending) {
  1139. if (!await this.#ownershipStillAllowed()) return
  1140. if (!expectedRunID) {
  1141. let ownership: ChatTerminalContent = {}
  1142. try { ownership = await this.resolver.resolveTerminal(this.#job, 'interaction.requested', result) } catch { return }
  1143. if (ownership.correlated !== true) return
  1144. }
  1145. await this.coordinator.enqueueNotification(pending)
  1146. return
  1147. }
  1148. const inflight = record(result.inflight ?? result.active_run)
  1149. const queued = record(result.queued)
  1150. const running = result.running === true || inflight.streaming === true
  1151. || Boolean(string(queued.user ?? queued.text))
  1152. if (running && expectedRunID && !resumedRunID) {
  1153. this.#socket?.close(1012, 'active push run lacks ownership identity')
  1154. return
  1155. }
  1156. if (running) this.#retryAttempt = 0
  1157. if (!running) {
  1158. const resumeError = string(result.error ?? inflight.error)
  1159. if (resumeError) {
  1160. await this.#finishTerminal('run.failed', { ...result, error: resumeError })
  1161. return
  1162. }
  1163. let resolved: ChatTerminalContent = {}
  1164. try { resolved = await this.resolver.resolveTerminal(this.#job, 'run.completed', result) } catch { /* retry below */ }
  1165. if (resolved.confirmed !== true) {
  1166. this.#socket?.close(1012, 'awaiting authoritative terminal message')
  1167. return
  1168. }
  1169. await this.#finishTerminal(
  1170. resolved.failed ? 'run.failed' : 'run.completed',
  1171. resolved.failed ? { ...result, error: resolved.error ?? resolved.body } : result,
  1172. resolved,
  1173. )
  1174. }
  1175. }).catch(() => {
  1176. if (!this.#running || generation !== this.#generation) return
  1177. this.#socket?.close(1011, 'chat push watcher frame failed')
  1178. this.#scheduleReconnect(generation)
  1179. })
  1180. }
  1181. async #finishTerminal(eventType: string, payload: JsonRecord, resolved?: ChatTerminalContent): Promise<void> {
  1182. if (!this.#running) return
  1183. if (!await this.#ownershipStillAllowed()) {
  1184. this.stop()
  1185. this.#notifyFinished()
  1186. return
  1187. }
  1188. let ownership = resolved
  1189. if (!ownership) {
  1190. try { ownership = await this.resolver.resolveTerminal(this.#job, eventType, payload) } catch { ownership = {} }
  1191. }
  1192. const expectedRunID = string(this.#job.metadata?.runID)
  1193. const eventRunID = string(payload.run_id ?? payload.runId ?? payload.turn_id ?? payload.turnId)
  1194. const ownedByRun = Boolean(expectedRunID && eventRunID === expectedRunID)
  1195. const failed = eventType === 'run.failed' || eventType === 'error' || Boolean(payload.error)
  1196. if (ownership.confirmed !== true && !ownedByRun && !(failed && ownership.correlated === true)) {
  1197. this.#socket?.close(1012, 'terminal push ownership is not confirmed')
  1198. return
  1199. }
  1200. const candidate = await chatTerminalCandidate(this.#job, eventType, payload, this.resolver, ownership)
  1201. await this.coordinator.enqueueNotification(candidate)
  1202. await this.coordinator.completeChatJob(this.#job.id)
  1203. this.stop()
  1204. this.#notifyFinished()
  1205. }
  1206. #scheduleReconnect(generation: number): void {
  1207. if (!this.#running || generation !== this.#generation || this.#retryTimer) return
  1208. this.#socket = undefined
  1209. const delay = Math.min(60_000, 1_000 * 2 ** Math.min(this.#retryAttempt++, 6))
  1210. this.#retryTimer = setTimeout(() => {
  1211. this.#retryTimer = undefined
  1212. if (this.#running && generation === this.#generation) void this.#connect(generation)
  1213. }, delay)
  1214. this.#retryTimer.unref()
  1215. }
  1216. async #ownershipStillAllowed(): Promise<boolean> {
  1217. if (!this.coordinator.canRecoverChatJob) return true
  1218. try { return await this.coordinator.canRecoverChatJob(this.#job) } catch { return false }
  1219. }
  1220. #notifyFinished(): void {
  1221. if (this.#finished) return
  1222. this.#finished = true
  1223. this.onFinished?.(this.#job.id)
  1224. }
  1225. }
  1226. export interface ChatPushJobManagerOptions {
  1227. transportFactory?: ChatPushTransportFactory
  1228. maximumConcurrent?: number
  1229. reconcileMilliseconds?: number
  1230. now?: () => number
  1231. resolver?: ChatNotificationResolver
  1232. }
  1233. /** Starts and bounds the server-owned recovery sockets for persisted jobs. */
  1234. export class ChatPushJobManager {
  1235. private transportFactory?: ChatPushTransportFactory
  1236. private brokerOwnsJob?: (job: ChatPushJob) => boolean
  1237. readonly #watchers = new Map<string, HermesChatPushJobWatcher>()
  1238. readonly #maximumConcurrent: number
  1239. readonly #reconcileMilliseconds: number
  1240. readonly #now: () => number
  1241. readonly #resolver: ChatNotificationResolver
  1242. #running = false
  1243. #timer?: ReturnType<typeof setInterval>
  1244. #activeReconcile?: Promise<void>
  1245. constructor(
  1246. readonly config: ServerConfig,
  1247. readonly upstreamSession: UpstreamServiceSession,
  1248. readonly coordinator: PushEventCoordinator,
  1249. options: ChatPushJobManagerOptions = {},
  1250. ) {
  1251. this.#maximumConcurrent = Math.max(1, Math.min(64, Math.trunc(options.maximumConcurrent ?? 16)))
  1252. this.#reconcileMilliseconds = Math.max(250, Math.trunc(options.reconcileMilliseconds ?? 1_000))
  1253. this.#now = options.now ?? Date.now
  1254. this.#resolver = options.resolver ?? new HermesChatNotificationResolver(upstreamSession)
  1255. this.transportFactory = options.transportFactory
  1256. }
  1257. setTransportFactory(factory: ChatPushTransportFactory, ownsJob?: (job: ChatPushJob) => boolean): void {
  1258. this.transportFactory = factory; this.brokerOwnsJob = ownsJob
  1259. }
  1260. start(): void {
  1261. if (this.#running) return
  1262. this.#running = true
  1263. void this.reconcile().catch(() => undefined)
  1264. this.#timer = setInterval(() => {
  1265. void this.reconcile().catch(() => undefined)
  1266. }, this.#reconcileMilliseconds)
  1267. this.#timer.unref()
  1268. }
  1269. stop(): void {
  1270. this.#running = false
  1271. if (this.#timer) clearInterval(this.#timer)
  1272. this.#timer = undefined
  1273. for (const watcher of this.#watchers.values()) watcher.stop()
  1274. this.#watchers.clear()
  1275. }
  1276. reconcile(): Promise<void> {
  1277. if (this.#activeReconcile) return this.#activeReconcile
  1278. const operation = this.#reconcile()
  1279. const tracked = operation.finally(() => {
  1280. if (this.#activeReconcile === tracked) this.#activeReconcile = undefined
  1281. })
  1282. this.#activeReconcile = tracked
  1283. return tracked
  1284. }
  1285. async #reconcile(): Promise<void> {
  1286. if (!this.#running) return
  1287. const pending = [...await this.coordinator.pendingChatJobs()]
  1288. const now = this.#now()
  1289. for (const job of pending) {
  1290. if (job.expiresAt <= now) await this.coordinator.completeChatJob(job.id)
  1291. }
  1292. const jobs: ChatPushJob[] = []
  1293. for (const job of pending) {
  1294. const structurallyEligible = job.expiresAt > now
  1295. && job.metadata?.baselineCaptured === true
  1296. && (Boolean(this.transportFactory) || Number.isFinite(Number(job.metadata?.disconnectedAt)))
  1297. && Boolean(string(job.metadata?.runID) || string(job.metadata?.promptDigest))
  1298. && (job.phase === 'submitted' || job.phase === 'accepted' || job.phase === 'watching')
  1299. if (!structurallyEligible) continue
  1300. if (this.brokerOwnsJob?.(job)) continue
  1301. if (this.coordinator.canRecoverChatJob
  1302. && !await this.coordinator.canRecoverChatJob(job)) continue
  1303. jobs.push(job)
  1304. }
  1305. jobs.sort((left, right) => left.submittedAt - right.submittedAt)
  1306. if (!this.#running) return
  1307. const pendingIDs = new Set(jobs.map(job => job.id))
  1308. for (const [jobID, watcher] of this.#watchers) {
  1309. if (pendingIDs.has(jobID)) continue
  1310. watcher.stop()
  1311. this.#watchers.delete(jobID)
  1312. }
  1313. for (const job of jobs) {
  1314. if (!this.#running || this.#watchers.size >= this.#maximumConcurrent) break
  1315. if (this.#watchers.has(job.id)) continue
  1316. const watcher = new HermesChatPushJobWatcher(
  1317. job,
  1318. this.config,
  1319. this.upstreamSession,
  1320. this.coordinator,
  1321. this.#resolver,
  1322. this.#now,
  1323. jobID => {
  1324. this.#watchers.delete(jobID)
  1325. if (this.#running) void this.reconcile()
  1326. },
  1327. this.transportFactory,
  1328. )
  1329. this.#watchers.set(job.id, watcher)
  1330. watcher.start()
  1331. }
  1332. }
  1333. }