workspaceScheduler.ts 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. import { randomUUID } from 'node:crypto'
  2. import { WorkspaceStore } from './workspaceStore.js'
  3. import { HttpError } from './errors.js'
  4. import { mentionedAgents } from './workspaceMentions.js'
  5. import type { WorkspaceAgent as Agent, WorkspaceConversation as Conversation, WorkspaceMessage as Message, WorkspaceRun as Run, WorkspaceInteraction } from '../shared/workspace.js'
  6. export const NO_REPLY = '[[YAOYAO_NO_REPLY_V1]]'
  7. export const HOST_FALLBACK = '我还不能确定你希望我处理什么,请补充具体目标、范围,或明确需要我协调的 Agent。'
  8. const terminal = (status: string) => ['complete', 'failed', 'interrupted'].includes(status)
  9. export interface Work {
  10. id: string
  11. runId: string
  12. conversationId: string
  13. conversationTaskId?: string
  14. agentId: string
  15. messageId: string
  16. triggerSeq: number
  17. depth: number
  18. batchId: string
  19. reviewOf?: string
  20. replyMode: 'mentioned' | 'automatic'
  21. requiredReply: boolean
  22. status: Run['status']
  23. currentMessageId?: string
  24. submitted?: boolean
  25. cancelRequested?: boolean
  26. planned?: boolean
  27. silent?: boolean
  28. hadInteraction?: boolean
  29. error?: string
  30. contextThroughSeq?: number
  31. turnConfiguration?: { mode: 'host' | 'free'; administratorId: string; members: Array<Pick<Agent, 'id' | 'name'>> }
  32. createdAt: number
  33. }
  34. /** Durable per-member work and cascade ledger; Hermes transport lives in WorkspaceRuntime. */
  35. export abstract class WorkspaceScheduler {
  36. protected closing = false
  37. protected executing = new Set<string>()
  38. private wakePending = false
  39. private pumping = false
  40. private retryTimer?: ReturnType<typeof setTimeout>
  41. private recoverAfter = new Map<string, number>()
  42. private attempts = new Map<string, number>()
  43. private interrupting = new Set<string>()
  44. onMessage: (owner: string, message: Message) => Promise<void> = async () => {}
  45. onNotify: (owner: string, c: Conversation, run: Run, message?: Message, interaction?: WorkspaceInteraction) => void = () => {}
  46. constructor(readonly store: WorkspaceStore, readonly userActive: (owner: string) => boolean = () => true) {
  47. for (const owner of store.owners()) {
  48. // Forward the already-admitted v0.3.x turn without replaying its prompt.
  49. for (const root of store.list<Run & { queue?: string[]; next?: string[]; hostReturn?: boolean; round: number; currentMessageId?: string; turnConfiguration?: Work['turnConfiguration'] }>(owner, 'run')) {
  50. if (terminal(root.status) || this.works(owner, root.id).length) continue
  51. const c = store.require<Conversation>(owner, 'conversation', root.conversationId)
  52. const trigger = store.require<Message>(owner, 'message', root.messageId)
  53. const continuation = root.queue?.length === 0
  54. const ids = continuation ? root.hostReturn ? [c.administratorId] : root.next ?? [] : root.queue ?? [root.activeAgentId ?? c.administratorId]
  55. if (!ids.length && root.currentMessageId) {
  56. const agentId = store.get<Message>(owner, 'message', root.currentMessageId)?.agentId ?? root.activeAgentId
  57. if (agentId) ids.push(agentId)
  58. }
  59. if (!ids.length) { root.status = root.stopRequested ? 'interrupted' : 'complete'; store.saveRun(owner, root); continue }
  60. const last = store.messages(owner, c.id).filter(m => m.runId === root.id).at(-1)
  61. for (const [index, id] of ids.entries()) {
  62. const work = this.enqueue(owner, root, id, root.round + (continuation && !root.hostReturn ? 1 : 0), root.id, last?.seq ?? trigger.seq, 'mentioned', id === c.administratorId)
  63. if (index === 0 && root.currentMessageId) {
  64. Object.assign(work, { currentMessageId: root.currentMessageId, submitted: true, status: 'uncertain', turnConfiguration: root.turnConfiguration })
  65. store.put(owner, 'turn', work.id, work)
  66. const key = this.bindingKey(c.id, root.conversationTaskId, id)
  67. const binding = store.get<{ runId: string; messageId: string; taskId?: string }>(owner, 'binding', key)
  68. if (binding?.runId === root.id && binding.messageId === root.currentMessageId) {
  69. binding.taskId = work.id; store.put(owner, 'binding', key, binding)
  70. }
  71. }
  72. }
  73. }
  74. for (const work of this.works(owner)) {
  75. if (['running', 'waiting'].includes(work.status)) {
  76. work.status = work.submitted ? 'uncertain' : 'queued'
  77. work.error = work.submitted ? '服务已重启,正在核对原执行' : undefined
  78. store.put(owner, 'turn', work.id, work)
  79. }
  80. }
  81. }
  82. }
  83. protected notify(owner: string, c: Conversation, run: Run, message?: Message, interaction?: WorkspaceInteraction): void {
  84. try { this.onNotify(owner, c, run, message, interaction) } catch { /* Push cannot change execution. */ }
  85. }
  86. protected abstract performTurn(owner: string, c: Conversation, agent: Agent, work: Work, recovering: boolean): Promise<Message>
  87. protected abstract interruptTurn(owner: string, work: Work): Promise<void>
  88. protected bindingKey(conversationId: string, conversationTaskId: string | undefined, agentId: string): string {
  89. return conversationTaskId ? `${conversationId}:${conversationTaskId}:${agentId}` : `${conversationId}:${agentId}`
  90. }
  91. protected works(owner: string, runId?: string): Work[] {
  92. return this.store.db.prepare(`SELECT data FROM workspace_entities WHERE owner=? AND kind='turn' ${runId ? "AND json_extract(data,'$.runId')=?" : ''}`).all(...(runId ? [owner, runId] : [owner]))
  93. .map(row => JSON.parse(String(row.data)) as Work)
  94. }
  95. private pendingWorks(owner: string): Work[] {
  96. return this.store.db.prepare("SELECT data FROM workspace_entities WHERE owner=? AND kind='turn' AND (json_extract(data,'$.status') NOT IN ('complete','failed','interrupted') OR coalesce(json_extract(data,'$.planned'),0)=0)").all(owner)
  97. .map(row => JSON.parse(String(row.data)) as Work)
  98. }
  99. protected getWork(owner: string, id: string): Work { return this.store.require(owner, 'turn', id) }
  100. protected saveWork(owner: string, work: Work): void {
  101. this.store.atomic(() => { this.store.put(owner, 'turn', work.id, work); this.updateRoot(owner, work.runId) })
  102. }
  103. protected enqueue(owner: string, root: Run, agentId: string, depth: number, batchId: string, triggerSeq: number, replyMode: Work['replyMode'], requiredReply: boolean, reviewOf?: string): Work {
  104. const existing = this.works(owner, root.id).find(w => reviewOf ? w.reviewOf === reviewOf : w.agentId === agentId && w.depth === depth && !w.reviewOf)
  105. if (existing) return existing
  106. const work: Work = { id: randomUUID(), runId: root.id, conversationId: root.conversationId, conversationTaskId: root.conversationTaskId, agentId, messageId: root.messageId, depth, batchId, triggerSeq, replyMode, requiredReply, reviewOf, status: 'queued', createdAt: Date.now() }
  107. this.store.put(owner, 'turn', work.id, work)
  108. return work
  109. }
  110. protected admit(owner: string, root: Run, c: Conversation, trigger: Message): void {
  111. this.store.put(owner, 'run', root.id, root)
  112. const ids = c.kind === 'direct' || c.mode === 'host' ? [c.administratorId]
  113. : root.mentionIds.length ? root.mentionIds : [...new Set([c.administratorId, ...c.autoReplyIds])]
  114. for (const id of ids) this.enqueue(owner, root, id, 0, root.id, trigger.seq,
  115. c.kind === 'direct' ? 'mentioned' : c.mode === 'host' || !root.mentionIds.length ? 'automatic' : 'mentioned', c.kind === 'group' && id === c.administratorId && (c.mode === 'host' || !root.mentionIds.length))
  116. this.updateRoot(owner, root.id)
  117. }
  118. start(): void { this.wake() }
  119. wake(): void {
  120. if (this.closing || this.wakePending) return
  121. this.wakePending = true
  122. queueMicrotask(() => { this.wakePending = false; if (!this.closing) this.pump() })
  123. }
  124. private retrySoon(): void {
  125. if (this.closing || this.retryTimer) return
  126. this.retryTimer = setTimeout(() => { this.retryTimer = undefined; this.wake() }, 500)
  127. this.retryTimer.unref()
  128. }
  129. private updateRoot(owner: string, id: string): void {
  130. const root = this.store.require<Run>(owner, 'run', id), tasks = this.works(owner, id)
  131. const active = tasks.filter(t => !terminal(t.status))
  132. const done = !active.length && tasks.every(t => t.planned)
  133. const wasTerminal = terminal(root.status)
  134. root.round = Math.max(0, ...tasks.map(t => t.depth))
  135. root.activeAgentId = active.find(t => t.status !== 'queued')?.agentId ?? active[0]?.agentId
  136. if (done) {
  137. const interrupted = root.stopRequested || tasks.some(t => t.requiredReply && t.status === 'interrupted') || (tasks.length > 0 && tasks.every(t => t.status === 'interrupted'))
  138. const failed = tasks.some(t => t.status === 'failed' && (t.requiredReply || this.store.require<Conversation>(owner, 'conversation', root.conversationId).kind === 'direct'))
  139. root.status = interrupted ? 'interrupted' : failed ? 'failed' : 'complete'
  140. } else {
  141. root.status = active.some(t => t.status === 'uncertain') ? 'uncertain' : active.some(t => t.status === 'running') ? 'running' : active.some(t => t.status === 'waiting') ? 'waiting' : tasks.some(t => t.status !== 'queued') ? 'running' : 'queued'
  142. }
  143. root.error = tasks.find(t => t.status === 'uncertain' || t.status === 'failed')?.error
  144. this.store.saveRun(owner, root)
  145. if (done && !wasTerminal && root.status !== 'interrupted') {
  146. const c = this.store.require<Conversation>(owner, 'conversation', root.conversationId)
  147. if (root.status === 'failed' && c.kind === 'direct') this.store.saveMessage(owner, { id: randomUUID(), conversationId: c.id, conversationTaskId: root.conversationTaskId, seq: 0, role: 'system', content: `执行失败:${root.error ?? '运行失败'}`, reasoning: '', status: 'failed', runId: root.id, attachments: [], tools: [], createdAt: Date.now() })
  148. this.notify(owner, c, root, this.store.messages(owner, c.id, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, false, root.conversationTaskId).filter(m => m.runId === id && m.visible !== false).at(-1))
  149. }
  150. }
  151. private plan(owner: string, id: string): void {
  152. this.store.atomic(() => {
  153. const work = this.getWork(owner, id)
  154. if (!terminal(work.status) || work.planned) return
  155. const root = this.store.require<Run>(owner, 'run', work.runId), c = this.store.require<Conversation>(owner, 'conversation', work.conversationId)
  156. work.planned = true
  157. this.store.put(owner, 'turn', work.id, work)
  158. if (root.stopRequested || c.archived || c.kind === 'direct') { this.updateRoot(owner, root.id); return }
  159. const config = work.turnConfiguration ?? { mode: c.mode, administratorId: c.administratorId, members: c.memberIds.map(id => this.store.require<Agent>(owner, 'agent', id)) }
  160. const message = work.currentMessageId ? this.store.get<Message>(owner, 'message', work.currentMessageId) : undefined
  161. if (config.mode === 'host' && work.agentId !== config.administratorId) {
  162. const batch = this.works(owner, root.id).filter(t => t.batchId === work.batchId && !t.reviewOf)
  163. if (batch.every(t => terminal(t.status))) {
  164. const through = Math.max(work.triggerSeq, ...batch.map(t => t.currentMessageId ? this.store.get<Message>(owner, 'message', t.currentMessageId)?.seq ?? 0 : 0))
  165. this.enqueue(owner, root, c.administratorId, work.depth, `review:${work.batchId}`, through, 'automatic', true, work.batchId)
  166. }
  167. } else if (work.status === 'complete' && !work.silent && message) {
  168. const nextDepth = work.depth + 1
  169. const mentions = mentionedAgents(message.content, config.members).filter(id => id !== work.agentId && c.memberIds.includes(id))
  170. const targets = mentions.length ? mentions : config.mode === 'free' ? c.autoReplyIds.filter(id => id !== work.agentId && c.memberIds.includes(id)) : []
  171. if (c.maxReplyRounds === -1 || nextDepth < c.maxReplyRounds) {
  172. for (const target of targets) this.enqueue(owner, root, target, nextDepth, work.id, message.seq, mentions.includes(target) ? 'mentioned' : 'automatic', false)
  173. } else if (targets.length && !this.store.get(owner, 'limit-notice', root.id)) {
  174. this.store.put(owner, 'limit-notice', root.id, { id: root.id })
  175. this.store.saveMessage(owner, { id: randomUUID(), conversationId: c.id, conversationTaskId: root.conversationTaskId, seq: 0, role: 'system', content: '已达到自动协作轮数上限,本轮不再自动分派。', reasoning: '', status: 'complete', runId: root.id, attachments: [], tools: [], createdAt: Date.now() })
  176. }
  177. }
  178. this.updateRoot(owner, root.id)
  179. })
  180. }
  181. private pump(): void {
  182. if (this.pumping || this.closing) return
  183. this.pumping = true
  184. try {
  185. let all = this.store.owners().flatMap(owner => this.pendingWorks(owner).map(work => ({ owner, work })))
  186. for (const { owner, work } of all) {
  187. if (['running', 'waiting'].includes(work.status) && !this.executing.has(work.id)) {
  188. work.status = work.submitted ? 'uncertain' : 'queued'
  189. this.saveWork(owner, work)
  190. }
  191. }
  192. for (const { owner, work } of all) if (terminal(work.status) && !work.planned && !this.executing.has(work.id)) this.plan(owner, work.id)
  193. all = this.store.owners().flatMap(owner => this.pendingWorks(owner).map(work => ({ owner, work })))
  194. all.sort((a, b) => {
  195. const am = this.store.require<Message>(a.owner, 'message', a.work.messageId), bm = this.store.require<Message>(b.owner, 'message', b.work.messageId)
  196. return am.createdAt - bm.createdAt || (a.work.conversationId === b.work.conversationId && a.work.conversationTaskId === b.work.conversationTaskId ? am.seq - bm.seq : 0) || a.work.createdAt - b.work.createdAt || a.work.id.localeCompare(b.work.id)
  197. })
  198. for (const { owner, work: candidate } of all) {
  199. const work = this.getWork(owner, candidate.id)
  200. if (work.cancelRequested && work.status === 'uncertain' && this.executing.has(work.id)) {
  201. if ((this.recoverAfter.get(work.id) ?? 0) > Date.now()) { this.retrySoon(); continue }
  202. if (!this.interrupting.has(work.id)) {
  203. this.interrupting.add(work.id)
  204. void this.interruptTurn(owner, work).catch(error => {
  205. if (this.closing) return
  206. const current = this.getWork(owner, work.id)
  207. if (terminal(current.status)) return
  208. current.status = 'uncertain'; current.error = `停止状态待确认:${error instanceof Error ? error.message : '连接中断'}`
  209. this.saveWork(owner, current)
  210. const attempt = this.attempts.get(work.id) ?? 0
  211. this.attempts.set(work.id, attempt + 1)
  212. this.recoverAfter.set(work.id, Date.now() + Math.min(30_000, 500 * 2 ** Math.min(attempt, 6)))
  213. this.retrySoon()
  214. }).finally(() => { this.interrupting.delete(work.id); this.wake() }).catch(() => this.retrySoon())
  215. }
  216. continue
  217. }
  218. if (this.executing.size >= 4 || !['queued', 'uncertain'].includes(work.status) || this.executing.has(work.id)) continue
  219. if (!this.userActive(owner)) { this.retrySoon(); continue }
  220. const c = this.store.require<Conversation>(owner, 'conversation', work.conversationId), root = this.store.require<Run>(owner, 'run', work.runId)
  221. if (work.status === 'queued' && c.mode === 'host' && work.depth === 0 && work.batchId === root.id && work.requiredReply && !work.currentMessageId) {
  222. work.agentId = c.administratorId
  223. }
  224. if (work.status === 'queued' && (c.archived || root.stopRequested || !c.memberIds.includes(work.agentId))) {
  225. work.status = 'interrupted'; work.error = '执行前成员已移除或聊天已停止'; this.saveWork(owner, work); this.wake(); continue
  226. }
  227. const occupied = all.map(entry => this.getWork(entry.owner, entry.work.id)).filter(t => (['running', 'waiting', 'uncertain'].includes(t.status) || this.executing.has(t.id)) && t.id !== work.id)
  228. if (occupied.some(t => t.conversationId === c.id && t.conversationTaskId === work.conversationTaskId && t.agentId === work.agentId)) continue
  229. const inTask = occupied.filter(t => t.conversationId === c.id && t.conversationTaskId === work.conversationTaskId)
  230. if (work.status === 'queued' && (occupied.length >= 4 || inTask.length >= 3)) continue
  231. if (work.status === 'uncertain' && (this.recoverAfter.get(work.id) ?? 0) > Date.now()) { this.retrySoon(); continue }
  232. if (c.mode === 'host' || c.kind === 'direct') {
  233. if (inTask.some(t => c.kind === 'direct' || t.runId !== work.runId || t.batchId !== work.batchId || t.agentId === c.administratorId || work.agentId === c.administratorId)) continue
  234. }
  235. this.executing.add(work.id)
  236. const recovering = work.status === 'uncertain'
  237. if (!recovering) {
  238. work.status = 'running'
  239. if (c.kind === 'group' && c.mode === 'host' && work.agentId === c.administratorId) { work.requiredReply = true; work.replyMode = 'automatic' }
  240. work.turnConfiguration = { mode: c.mode, administratorId: c.administratorId, members: c.memberIds.map(id => { const a = this.store.require<Agent>(owner, 'agent', id); return { id, name: a.name } }) }
  241. this.saveWork(owner, work)
  242. }
  243. void this.executeWork(owner, work, recovering).catch(() => this.retrySoon())
  244. }
  245. } catch { this.retrySoon() } finally { this.pumping = false }
  246. }
  247. private async executeWork(owner: string, work: Work, recovering: boolean): Promise<void> {
  248. try {
  249. if (work.cancelRequested) await this.interruptTurn(owner, work)
  250. else {
  251. const c = this.store.require<Conversation>(owner, 'conversation', work.conversationId)
  252. const agent = this.store.require<Agent>(owner, 'agent', work.agentId)
  253. const message = await this.performTurn(owner, c, agent, work, recovering)
  254. if (message.visible !== false) await this.onMessage(owner, message).catch(() => {})
  255. }
  256. } catch (error) {
  257. if (this.closing) return
  258. const current = this.getWork(owner, work.id)
  259. if (!terminal(current.status) && current.status !== 'uncertain') {
  260. current.status = current.submitted ? 'uncertain' : 'failed'
  261. current.error = error instanceof Error ? error.message : '执行失败'
  262. this.saveWork(owner, current)
  263. }
  264. if (current.status === 'failed' && !current.currentMessageId) {
  265. current.currentMessageId = randomUUID()
  266. this.store.saveMessage(owner, { id: current.currentMessageId, conversationId: current.conversationId, conversationTaskId: current.conversationTaskId, seq: 0, role: 'assistant', agentId: current.agentId, agentName: this.store.get<Agent>(owner, 'agent', current.agentId)?.name, content: `执行失败:${current.error}`, reasoning: '', status: 'failed', error: current.error, taskId: current.id, runId: current.runId, attachments: [], tools: [], createdAt: Date.now() })
  267. this.saveWork(owner, current)
  268. }
  269. } finally {
  270. this.executing.delete(work.id)
  271. if (!this.closing) {
  272. const current = this.getWork(owner, work.id)
  273. if (current.status === 'uncertain') {
  274. const attempt = this.attempts.get(work.id) ?? 0
  275. this.attempts.set(work.id, attempt + 1)
  276. this.recoverAfter.set(work.id, Date.now() + Math.min(30_000, 500 * 2 ** Math.min(attempt, 6)))
  277. this.retrySoon()
  278. } else { this.attempts.delete(work.id); this.recoverAfter.delete(work.id) }
  279. this.wake()
  280. }
  281. }
  282. }
  283. async reconcile(owner: string, id: string): Promise<void> {
  284. this.store.require<Run>(owner, 'run', id)
  285. for (const w of this.works(owner, id)) this.recoverAfter.delete(w.id)
  286. this.wake()
  287. }
  288. async stop(owner: string, id: string): Promise<void> {
  289. const root = this.store.require<Run>(owner, 'run', id)
  290. if (terminal(root.status)) return
  291. root.stopRequested = true
  292. this.store.saveRun(owner, root)
  293. await this.cancelWorks(owner, this.works(owner, id))
  294. }
  295. async stopAgent(owner: string, conversationId: string, agentId: string, conversationTaskId?: string): Promise<void> {
  296. const c = this.store.require<Conversation>(owner, 'conversation', conversationId)
  297. if (!c.memberIds.includes(agentId)) throw new HttpError(400, '只能停止群内成员', 'invalid_members')
  298. const scopedTaskId = c.kind === 'group' ? this.store.resolveTask(owner, conversationId, conversationTaskId)!.id : undefined
  299. await this.cancelWorks(owner, this.works(owner).filter(w => w.conversationId === conversationId && w.agentId === agentId && w.conversationTaskId === scopedTaskId))
  300. }
  301. async stopTask(owner: string, conversationId: string, conversationTaskId: string): Promise<void> {
  302. this.store.requireTask(owner, conversationId, conversationTaskId)
  303. const roots = this.store.list<Run>(owner, 'run').filter(r => r.conversationId === conversationId && r.conversationTaskId === conversationTaskId && !terminal(r.status))
  304. this.store.atomic(() => { for (const root of roots) { root.stopRequested = true; this.store.saveRun(owner, root) } })
  305. await this.cancelWorks(owner, this.works(owner).filter(w => w.conversationId === conversationId && w.conversationTaskId === conversationTaskId))
  306. }
  307. async stopConversation(owner: string, id: string): Promise<void> {
  308. this.store.require<Conversation>(owner, 'conversation', id)
  309. const roots = this.store.list<Run>(owner, 'run').filter(r => r.conversationId === id && !terminal(r.status))
  310. this.store.atomic(() => { for (const root of roots) { root.stopRequested = true; this.store.saveRun(owner, root) } })
  311. await this.cancelWorks(owner, this.works(owner).filter(w => roots.some(r => r.id === w.runId)))
  312. }
  313. private async cancelWorks(owner: string, works: Work[]): Promise<void> {
  314. await Promise.all(works.filter(w => !terminal(w.status)).map(async w => {
  315. const current = this.getWork(owner, w.id)
  316. current.cancelRequested = true
  317. if (current.status === 'queued') { current.status = 'interrupted'; this.saveWork(owner, current) }
  318. else {
  319. this.saveWork(owner, current)
  320. try { await this.interruptTurn(owner, current) }
  321. catch (error) {
  322. const latest = this.getWork(owner, current.id)
  323. if (terminal(latest.status)) return
  324. latest.status = 'uncertain'; latest.error = `停止状态待确认:${error instanceof Error ? error.message : '连接中断'}`
  325. this.saveWork(owner, latest); this.recoverAfter.delete(latest.id); this.retrySoon()
  326. }
  327. }
  328. }))
  329. this.wake()
  330. }
  331. protected closeScheduler(): void {
  332. this.closing = true
  333. if (this.retryTimer) clearTimeout(this.retryTimer)
  334. this.recoverAfter.clear()
  335. this.attempts.clear()
  336. }
  337. }