workspaceScheduler.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. import { randomUUID } from 'node:crypto'
  2. import { WorkspaceStore } from './workspaceStore.js'
  3. import { HttpError } from './errors.js'
  4. import { botRelayIntent, relayFingerprint, relayMessageValue, REPEATED_RELAY_NOTICE } from './workspaceRelay.js'
  5. import type { WorkspaceAgent as Agent, WorkspaceConversation as Conversation, WorkspaceMessage as Message, WorkspaceRun as Run, WorkspaceInteraction } from '../shared/workspace.js'
  6. export const NO_REPLY = '[[YAOYAO_NO_REPLY_V1]]'
  7. export const HOST_FALLBACK = '我还不能确定你希望我处理什么,请补充具体目标、范围,或明确需要我协调的机器人。'
  8. const terminal = (status: string) => ['complete', 'failed', 'interrupted'].includes(status)
  9. export interface Work {
  10. resourceWait?:boolean
  11. id: string
  12. runId: string
  13. conversationId: string
  14. conversationTaskId?: string
  15. agentId: string
  16. messageId: string
  17. triggerSeq: number
  18. depth: number
  19. batchId: string
  20. reviewOf?: string
  21. replyMode: 'mentioned' | 'automatic'
  22. requiredReply: boolean
  23. status: Run['status']
  24. currentMessageId?: string
  25. submitted?: boolean
  26. /** Revision captured when this turn receives a team-management grant. */
  27. teamManagementRevision?: number
  28. cancelRequested?: boolean
  29. planned?: boolean
  30. silent?: boolean
  31. hadInteraction?: boolean
  32. /** Durable assistant-to-assistant routing decisions, scoped to this run. */
  33. relay?: {
  34. fingerprints: Record<string, string>
  35. suppressed: Record<string, 'acknowledgement' | 'repeated'>
  36. notice?: boolean
  37. }
  38. error?: string
  39. contextThroughSeq?: number
  40. turnConfiguration?: { mode: 'host' | 'free'; administratorId: string; members: Array<Pick<Agent, 'id' | 'name'>> }
  41. createdAt: number
  42. }
  43. /** Durable per-member work and cascade ledger; Hermes transport lives in WorkspaceRuntime. */
  44. export abstract class WorkspaceScheduler {
  45. protected closing = false
  46. protected executing = new Set<string>()
  47. private wakePending = false
  48. private pumping = false
  49. private retryTimer?: ReturnType<typeof setTimeout>
  50. private recoverAfter = new Map<string, number>()
  51. private attempts = new Map<string, number>()
  52. private interrupting = new Set<string>()
  53. onMessage: (owner: string, message: Message) => Promise<void> = async () => {}
  54. onNotify: (owner: string, c: Conversation, run: Run, message?: Message, interaction?: WorkspaceInteraction) => void = () => {}
  55. constructor(readonly store: WorkspaceStore, readonly userActive: (owner: string) => boolean = () => true) {
  56. for (const owner of store.owners()) {
  57. // Forward the already-admitted v0.3.x turn without replaying its prompt.
  58. for (const root of store.list<Run & { queue?: string[]; next?: string[]; hostReturn?: boolean; round: number; currentMessageId?: string; turnConfiguration?: Work['turnConfiguration'] }>(owner, 'run')) {
  59. if (terminal(root.status) || this.works(owner, root.id).length) continue
  60. const c = store.require<Conversation>(owner, 'conversation', root.conversationId)
  61. const trigger = store.require<Message>(owner, 'message', root.messageId)
  62. const continuation = root.queue?.length === 0
  63. const ids = continuation ? root.hostReturn ? [c.administratorId] : root.next ?? [] : root.queue ?? [root.activeAgentId ?? c.administratorId]
  64. if (!ids.length && root.currentMessageId) {
  65. const agentId = store.get<Message>(owner, 'message', root.currentMessageId)?.agentId ?? root.activeAgentId
  66. if (agentId) ids.push(agentId)
  67. }
  68. if (!ids.length) { root.status = root.stopRequested ? 'interrupted' : 'complete'; store.saveRun(owner, root); continue }
  69. const last = store.messages(owner, c.id).filter(m => m.runId === root.id).at(-1)
  70. for (const [index, id] of ids.entries()) {
  71. const work = this.enqueue(owner, root, id, root.round + (continuation && !root.hostReturn ? 1 : 0), root.id, last?.seq ?? trigger.seq, 'mentioned', id === c.administratorId)
  72. if (index === 0 && root.currentMessageId) {
  73. Object.assign(work, { currentMessageId: root.currentMessageId, submitted: true, status: 'uncertain', turnConfiguration: root.turnConfiguration })
  74. store.put(owner, 'turn', work.id, work)
  75. const key = this.bindingKey(c.id, root.conversationTaskId, id)
  76. const binding = store.get<{ runId: string; messageId: string; taskId?: string }>(owner, 'binding', key)
  77. if (binding?.runId === root.id && binding.messageId === root.currentMessageId) {
  78. binding.taskId = work.id; store.put(owner, 'binding', key, binding)
  79. }
  80. }
  81. }
  82. }
  83. for (const work of this.works(owner)) {
  84. if (['running', 'waiting'].includes(work.status)) {
  85. work.status = work.submitted ? 'uncertain' : 'queued'
  86. work.error = work.submitted ? '服务已重启,正在核对原执行' : undefined
  87. store.put(owner, 'turn', work.id, work)
  88. }
  89. }
  90. }
  91. }
  92. protected notify(owner: string, c: Conversation, run: Run, message?: Message, interaction?: WorkspaceInteraction): void {
  93. try { this.onNotify(owner, c, run, message, interaction) } catch { /* Push cannot change execution. */ }
  94. }
  95. protected abstract performTurn(owner: string, c: Conversation, agent: Agent, work: Work, recovering: boolean): Promise<Message>
  96. protected abstract interruptTurn(owner: string, work: Work): Promise<void>
  97. protected bindingKey(conversationId: string, conversationTaskId: string | undefined, agentId: string): string {
  98. return conversationTaskId ? `${conversationId}:${conversationTaskId}:${agentId}` : `${conversationId}:${agentId}`
  99. }
  100. protected works(owner: string, runId?: string): Work[] {
  101. return this.store.db.prepare(`SELECT data FROM workspace_entities WHERE owner=? AND kind='turn' ${runId ? "AND json_extract(data,'$.runId')=?" : ''}`).all(...(runId ? [owner, runId] : [owner]))
  102. .map(row => JSON.parse(String(row.data)) as Work)
  103. }
  104. private pendingWorks(owner: string): Work[] {
  105. return this.store.db.prepare("SELECT data FROM workspace_entities WHERE owner=? AND kind='turn' AND (json_extract(data,'$.status') NOT IN ('complete','failed','interrupted') OR coalesce(json_extract(data,'$.planned'),0)=0)").all(owner)
  106. .map(row => JSON.parse(String(row.data)) as Work)
  107. }
  108. protected getWork(owner: string, id: string): Work { return this.store.require(owner, 'turn', id) }
  109. protected saveWork(owner: string, work: Work): void {
  110. this.store.atomic(() => { this.store.put(owner, 'turn', work.id, work); this.updateRoot(owner, work.runId) })
  111. }
  112. protected enqueue(owner: string, root: Run, agentId: string, depth: number, batchId: string, triggerSeq: number, replyMode: Work['replyMode'], requiredReply: boolean, reviewOf?: string): Work {
  113. const existing = this.works(owner, root.id).find(w => reviewOf ? w.reviewOf === reviewOf : w.agentId === agentId && w.depth === depth && !w.reviewOf)
  114. if (existing) return existing
  115. const work: Work = { id: randomUUID(), runId: root.id, conversationId: root.conversationId, conversationTaskId: root.conversationTaskId, agentId, messageId: root.messageId, depth, batchId, triggerSeq, replyMode, requiredReply, reviewOf, status: 'queued', createdAt: Date.now() }
  116. this.store.put(owner, 'turn', work.id, work)
  117. return work
  118. }
  119. protected admit(owner: string, root: Run, c: Conversation, trigger: Message): void {
  120. this.store.put(owner, 'run', root.id, root)
  121. const ids = root.assignmentId && root.targetAgentId ? [root.targetAgentId] : c.kind === 'direct' || c.mode === 'host' ? [c.administratorId]
  122. : root.mentionIds.length ? root.mentionIds : [...new Set([c.administratorId, ...c.autoReplyIds])]
  123. for (const id of ids) this.enqueue(owner, root, id, 0, root.id, trigger.seq,
  124. c.kind === 'direct' || root.assignmentId ? 'mentioned' : c.mode === 'host' || !root.mentionIds.length ? 'automatic' : 'mentioned', !!root.assignmentId || c.kind === 'group' && id === c.administratorId && (c.mode === 'host' || !root.mentionIds.length))
  125. this.updateRoot(owner, root.id)
  126. }
  127. start(): void { this.wake() }
  128. wake(): void {
  129. if (this.closing || this.wakePending) return
  130. this.wakePending = true
  131. queueMicrotask(() => { this.wakePending = false; if (!this.closing) this.pump() })
  132. }
  133. private retrySoon(): void {
  134. if (this.closing || this.retryTimer) return
  135. this.retryTimer = setTimeout(() => { this.retryTimer = undefined; this.wake() }, 500)
  136. this.retryTimer.unref()
  137. }
  138. private updateRoot(owner: string, id: string): void {
  139. const root = this.store.require<Run>(owner, 'run', id), tasks = this.works(owner, id)
  140. const active = tasks.filter(t => !terminal(t.status))
  141. const done = !active.length && tasks.every(t => t.planned)
  142. const wasTerminal = terminal(root.status)
  143. root.round = Math.max(0, ...tasks.map(t => t.depth))
  144. root.activeAgentId = active.find(t => t.status !== 'queued')?.agentId ?? active[0]?.agentId
  145. if (done) {
  146. const interrupted = root.stopRequested || tasks.some(t => t.requiredReply && t.status === 'interrupted') || (tasks.length > 0 && tasks.every(t => t.status === 'interrupted'))
  147. const failed = tasks.some(t => t.status === 'failed' && (t.requiredReply || this.store.require<Conversation>(owner, 'conversation', root.conversationId).kind === 'direct'))
  148. root.status = interrupted ? 'interrupted' : failed ? 'failed' : 'complete'
  149. } else {
  150. root.status = active.some(t => t.status === 'uncertain') ? 'uncertain' : active.some(t => t.status === 'running') ? 'running' : active.some(t => t.status === 'waiting') ? 'waiting' : tasks.some(t => t.status !== 'queued') ? 'running' : 'queued'
  151. }
  152. root.error = tasks.find(t => t.status === 'uncertain' || t.status === 'failed')?.error
  153. this.store.saveRun(owner, root)
  154. if (done && !wasTerminal && root.status !== 'interrupted') {
  155. const c = this.store.require<Conversation>(owner, 'conversation', root.conversationId)
  156. if (root.status === 'failed' && c.kind === 'direct') this.store.saveMessage(owner, { id: randomUUID(), conversationId: c.id, conversationTaskId: root.conversationTaskId, seq: 0, role: 'system', content: `执行失败:${root.error ?? '运行失败'}`, reasoning: '', status: 'failed', runId: root.id, attachments: [], tools: [], createdAt: Date.now() })
  157. this.notify(owner, c, root, this.store.messages(owner, c.id, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, false, root.conversationTaskId).filter(m => m.runId === id && m.visible !== false).at(-1))
  158. }
  159. }
  160. private plan(owner: string, id: string): void {
  161. this.store.atomic(() => {
  162. const work = this.getWork(owner, id)
  163. if (!terminal(work.status) || work.planned) return
  164. const root = this.store.require<Run>(owner, 'run', work.runId), c = this.store.require<Conversation>(owner, 'conversation', work.conversationId)
  165. work.planned = true
  166. this.store.put(owner, 'turn', work.id, work)
  167. if (root.stopRequested || c.archived || c.kind === 'direct' || root.assignmentId) { this.updateRoot(owner, root.id); return }
  168. const config = work.turnConfiguration ?? { mode: c.mode, administratorId: c.administratorId, members: c.memberIds.map(id => this.store.require<Agent>(owner, 'agent', id)) }
  169. const message = work.currentMessageId ? this.store.get<Message>(owner, 'message', work.currentMessageId) : undefined
  170. if (config.mode === 'host' && work.agentId !== config.administratorId) {
  171. const batch = this.works(owner, root.id).filter(t => t.batchId === work.batchId && !t.reviewOf)
  172. if (batch.every(t => terminal(t.status))) {
  173. const through = Math.max(work.triggerSeq, ...batch.map(t => t.currentMessageId ? this.store.get<Message>(owner, 'message', t.currentMessageId)?.seq ?? 0 : 0))
  174. this.enqueue(owner, root, c.administratorId, work.depth, `review:${work.batchId}`, through, 'automatic', true, work.batchId)
  175. }
  176. } else if (work.status === 'complete' && !work.silent && message) {
  177. const nextDepth = work.depth + 1
  178. const intent = botRelayIntent(message.content, config.members)
  179. const eligible = (id: string) => id !== work.agentId && this.store.taskMemberIds(owner, c, work.conversationTaskId).includes(id)
  180. const mentions = intent.targetIds.filter(eligible)
  181. // A filtered mention is still an explicit address. Never turn a closing
  182. // receipt back into a broadcast through the automatic participant list.
  183. const automatic = config.mode === 'free' ? c.autoReplyIds.filter(eligible) : []
  184. const targets = intent.mentionedIds.length ? mentions : intent.acknowledgementOnly ? [] : automatic
  185. work.relay = { fingerprints: {}, suppressed: {} }
  186. for (const target of intent.mentionedIds.filter(eligible)) {
  187. if (!mentions.includes(target)) work.relay.suppressed[target] = 'acknowledgement'
  188. }
  189. if (intent.acknowledgementOnly && !intent.mentionedIds.length)
  190. for (const target of automatic) work.relay.suppressed[target] = 'acknowledgement'
  191. if (targets.length && (c.maxReplyRounds === -1 || nextDepth < c.maxReplyRounds)) {
  192. const works = this.works(owner, root.id).filter(t => t.conversationId === c.id && t.conversationTaskId === work.conversationTaskId)
  193. const evidence = this.relayEvidence(owner, work, works)
  194. const previous = works.filter(t => t.id !== work.id && t.agentId === work.agentId && t.status === 'complete' && t.planned)
  195. const legacyEvidence = new Map<string, unknown>()
  196. for (const target of targets) {
  197. const fingerprint = evidence === undefined ? undefined : relayFingerprint({ target, evidence })
  198. if (fingerprint) work.relay.fingerprints[target] = fingerprint
  199. const repeated = fingerprint && previous.some(t => {
  200. if (t.relay) return t.relay.fingerprints[target] === fingerprint
  201. // Pre-upgrade turns have no receipt. Only a persisted child is
  202. // evidence that this particular edge was actually dispatched.
  203. if (!works.some(child => child.batchId === t.id && child.agentId === target && !child.reviewOf)) return false
  204. if (!legacyEvidence.has(t.id)) legacyEvidence.set(t.id, this.relayEvidence(owner, t, works))
  205. const old = legacyEvidence.get(t.id)
  206. return old !== undefined && relayFingerprint({ target, evidence: old }) === fingerprint
  207. })
  208. if (repeated) { work.relay.suppressed[target] = 'repeated'; continue }
  209. this.enqueue(owner, root, target, nextDepth, work.id, message.seq, mentions.includes(target) ? 'mentioned' : 'automatic', false)
  210. }
  211. if (Object.values(work.relay.suppressed).includes('repeated') && !works.some(t => t.relay?.notice)) {
  212. work.relay.notice = true
  213. this.store.saveMessage(owner, { id: randomUUID(), conversationId: c.id, conversationTaskId: root.conversationTaskId, seq: 0, role: 'system', content: REPEATED_RELAY_NOTICE, reasoning: '', status: 'complete', runId: root.id, attachments: [], tools: [], createdAt: Date.now() })
  214. }
  215. } else if (targets.length && !this.store.get(owner, 'limit-notice', root.id)) {
  216. this.store.put(owner, 'limit-notice', root.id, { id: root.id })
  217. this.store.saveMessage(owner, { id: randomUUID(), conversationId: c.id, conversationTaskId: root.conversationTaskId, seq: 0, role: 'system', content: '已达到自动协作轮数上限,本轮不再自动分派。', reasoning: '', status: 'complete', runId: root.id, attachments: [], tools: [], createdAt: Date.now() })
  218. }
  219. this.store.put(owner, 'turn', work.id, work)
  220. }
  221. this.updateRoot(owner, root.id)
  222. })
  223. }
  224. private relayEvidence(owner: string, work: Work, works: Work[]): unknown | undefined {
  225. const read = (id?: string) => {
  226. const message = id ? this.store.get<Message>(owner, 'message', id) : undefined
  227. return message?.conversationId === work.conversationId && message.conversationTaskId === work.conversationTaskId && message.runId === work.runId ? message : undefined
  228. }
  229. const reply = read(work.currentMessageId)
  230. if (!reply) return undefined
  231. const inputs = work.reviewOf ? works.filter(t => t.batchId === work.reviewOf && !t.reviewOf).sort((a, b) => a.agentId.localeCompare(b.agentId))
  232. : works.filter(t => t.id === work.batchId)
  233. let input: unknown
  234. if (inputs.length) {
  235. input = inputs.map(t => {
  236. const message = read(t.currentMessageId)
  237. return { agentId: t.agentId, status: t.status, error: t.error, message: message ? relayMessageValue(message) : null }
  238. })
  239. } else {
  240. // Initial turns and the legacy admission path have no parent Work.
  241. const row = this.store.db.prepare("SELECT data FROM workspace_entities WHERE owner=? AND kind='message' AND json_extract(data,'$.conversationId')=? AND json_extract(data,'$.seq')=?").get(owner, work.conversationId, work.triggerSeq)
  242. const trigger = row ? read((JSON.parse(String(row.data)) as Message).id) : undefined
  243. if (!trigger) return undefined
  244. input = relayMessageValue(trigger)
  245. }
  246. const relevant = new Set([work.id, ...inputs.map(t => t.id)])
  247. const interactions = this.store.list<WorkspaceInteraction & { answer?: string; responseState?: string }>(owner, 'interaction')
  248. .filter(i => i.runId === work.runId && i.conversationId === work.conversationId && i.conversationTaskId === work.conversationTaskId && i.answer !== undefined && i.responseState === 'sent'
  249. && relevant.has(this.store.get<{ taskId: string }>(owner, 'interaction-binding', i.id)?.taskId ?? ''))
  250. // A fresh human answer is permission to advance, even if the person
  251. // chooses the same answer again. This is not a provider event ID.
  252. .map(i => ({ id: i.id, kind: i.kind, question: i.message, answer: i.answer })).sort((a, b) => a.id.localeCompare(b.id))
  253. return { version: 1, agentId: work.agentId, mode: work.turnConfiguration?.mode, input, reply: relayMessageValue(reply), interactions }
  254. }
  255. private pump(): void {
  256. if (this.pumping || this.closing) return
  257. this.pumping = true
  258. try {
  259. let all = this.store.owners().flatMap(owner => this.pendingWorks(owner).map(work => ({ owner, work })))
  260. for (const { owner, work } of all) {
  261. if (['running', 'waiting'].includes(work.status) && !this.executing.has(work.id)) {
  262. work.status = work.submitted ? 'uncertain' : 'queued'
  263. this.saveWork(owner, work)
  264. }
  265. }
  266. for (const { owner, work } of all) if (terminal(work.status) && !work.planned && !this.executing.has(work.id)) this.plan(owner, work.id)
  267. all = this.store.owners().flatMap(owner => this.pendingWorks(owner).map(work => ({ owner, work })))
  268. all.sort((a, b) => {
  269. const priority = (entry: typeof a) => {
  270. if (Date.now() - entry.work.createdAt > 60_000) return 0
  271. const run = this.store.get<Run>(entry.owner, 'run', entry.work.runId)
  272. return run?.triggerKind === 'assignment' ? 2 : run?.triggerKind ? 1 : 0
  273. }
  274. const difference = priority(a) - priority(b)
  275. if (difference) return difference
  276. const am = this.store.require<Message>(a.owner, 'message', a.work.messageId), bm = this.store.require<Message>(b.owner, 'message', b.work.messageId)
  277. return am.createdAt - bm.createdAt || (a.work.conversationId === b.work.conversationId && a.work.conversationTaskId === b.work.conversationTaskId ? am.seq - bm.seq : 0) || a.work.createdAt - b.work.createdAt || a.work.id.localeCompare(b.work.id)
  278. })
  279. for (const { owner, work: candidate } of all) {
  280. const work = this.getWork(owner, candidate.id)
  281. if (work.cancelRequested && work.status === 'uncertain' && this.executing.has(work.id)) {
  282. if ((this.recoverAfter.get(work.id) ?? 0) > Date.now()) { this.retrySoon(); continue }
  283. if (!this.interrupting.has(work.id)) {
  284. this.interrupting.add(work.id)
  285. void this.interruptTurn(owner, work).catch(error => {
  286. if (this.closing) return
  287. const current = this.getWork(owner, work.id)
  288. if (terminal(current.status)) return
  289. current.status = 'uncertain'; current.error = `停止状态待确认:${error instanceof Error ? error.message : '连接中断'}`
  290. this.saveWork(owner, current)
  291. const attempt = this.attempts.get(work.id) ?? 0
  292. this.attempts.set(work.id, attempt + 1)
  293. this.recoverAfter.set(work.id, Date.now() + Math.min(30_000, 500 * 2 ** Math.min(attempt, 6)))
  294. this.retrySoon()
  295. }).finally(() => { this.interrupting.delete(work.id); this.wake() }).catch(() => this.retrySoon())
  296. }
  297. continue
  298. }
  299. if (this.executing.size >= 4 || !['queued', 'uncertain'].includes(work.status) || this.executing.has(work.id)) continue
  300. if (!this.userActive(owner)) { this.retrySoon(); continue }
  301. const c = this.store.require<Conversation>(owner, 'conversation', work.conversationId), root = this.store.require<Run>(owner, 'run', work.runId)
  302. if (work.status === 'queued' && !root.assignmentId && c.mode === 'host' && work.depth === 0 && work.batchId === root.id && work.requiredReply && !work.currentMessageId) {
  303. work.agentId = c.administratorId
  304. }
  305. if (work.status === 'queued' && (c.archived || root.stopRequested || !this.store.taskMemberIds(owner,c,work.conversationTaskId).includes(work.agentId))) {
  306. work.status = 'interrupted'; work.error = '执行前成员已移除或聊天已停止'; this.saveWork(owner, work); this.wake(); continue
  307. }
  308. const occupied = all.map(entry => this.getWork(entry.owner, entry.work.id)).filter(t => (['running', 'waiting', 'uncertain'].includes(t.status) || this.executing.has(t.id)) && t.id !== work.id)
  309. // A persistent Agent has one mutable execution lane across every chat
  310. // and task. Parallel work belongs to different Agents/resources.
  311. if (occupied.some(t => t.agentId === work.agentId)) continue
  312. if (occupied.some(t => t.conversationId === c.id && t.conversationTaskId === work.conversationTaskId && t.agentId === work.agentId)) continue
  313. const inTask = occupied.filter(t => t.conversationId === c.id && t.conversationTaskId === work.conversationTaskId)
  314. if (work.status === 'queued' && (occupied.length >= 4 || inTask.length >= 3)) continue
  315. if ((this.recoverAfter.get(work.id) ?? 0) > Date.now()) { this.retrySoon(); continue }
  316. if (c.mode === 'host' || c.kind === 'direct') {
  317. if (inTask.some(t => c.kind === 'direct'
  318. || (!root.assignmentId && (t.runId !== work.runId || t.batchId !== work.batchId))
  319. || t.agentId === c.administratorId || work.agentId === c.administratorId)) continue
  320. }
  321. this.executing.add(work.id)
  322. const recovering = work.status === 'uncertain'
  323. if (!recovering) {
  324. work.status = 'running';work.resourceWait=false
  325. if (c.kind === 'group' && c.mode === 'host' && work.agentId === c.administratorId) { work.requiredReply = true; work.replyMode = 'automatic' }
  326. work.turnConfiguration = { mode: c.mode, administratorId: c.administratorId, members: this.store.taskMemberIds(owner,c,work.conversationTaskId).map(id => { const a = this.store.require<Agent>(owner, 'agent', id); return { id, name: a.name } }) }
  327. this.saveWork(owner, work)
  328. }
  329. void this.executeWork(owner, work, recovering).catch(() => this.retrySoon())
  330. }
  331. } catch { this.retrySoon() } finally { this.pumping = false }
  332. }
  333. private async executeWork(owner: string, work: Work, recovering: boolean): Promise<void> {
  334. try {
  335. if (work.cancelRequested) await this.interruptTurn(owner, work)
  336. else {
  337. const c = this.store.require<Conversation>(owner, 'conversation', work.conversationId)
  338. const agent = this.store.require<Agent>(owner, 'agent', work.agentId)
  339. const message = await this.performTurn(owner, c, agent, work, recovering)
  340. if (message.visible !== false) await this.onMessage(owner, message).catch(() => {})
  341. }
  342. } catch (error) {
  343. if (this.closing) return
  344. const current = this.getWork(owner, work.id)
  345. if(!current.submitted&&error instanceof HttpError&&['computer_busy','computer_quota','runner_offline'].includes(error.code??'')&&current.status!=='interrupted'){
  346. current.status='queued';current.resourceWait=true;current.error='等待执行节点或电脑资源';this.saveWork(owner,current)
  347. }else if (!terminal(current.status) && current.status !== 'uncertain') {
  348. current.status = current.submitted ? 'uncertain' : 'failed'
  349. current.error = error instanceof Error ? error.message : '执行失败'
  350. this.saveWork(owner, current)
  351. }
  352. if (current.status === 'failed' && !current.currentMessageId) {
  353. current.currentMessageId = randomUUID()
  354. this.store.saveMessage(owner, { id: current.currentMessageId, conversationId: current.conversationId, conversationTaskId: current.conversationTaskId, seq: 0, role: 'assistant', agentId: current.agentId, agentName: this.store.get<Agent>(owner, 'agent', current.agentId)?.name, content: `执行失败:${current.error}`, reasoning: '', status: 'failed', error: current.error, taskId: current.id, runId: current.runId, attachments: [], tools: [], createdAt: Date.now() })
  355. this.saveWork(owner, current)
  356. }
  357. } finally {
  358. this.executing.delete(work.id)
  359. if (!this.closing) {
  360. const current = this.getWork(owner, work.id)
  361. if(current.status==='queued'&&current.resourceWait){this.recoverAfter.set(work.id,Date.now()+1000);this.retrySoon()}else if (current.status === 'uncertain') {
  362. const attempt = this.attempts.get(work.id) ?? 0
  363. this.attempts.set(work.id, attempt + 1)
  364. this.recoverAfter.set(work.id, Date.now() + Math.min(30_000, 500 * 2 ** Math.min(attempt, 6)))
  365. this.retrySoon()
  366. } else { this.attempts.delete(work.id); this.recoverAfter.delete(work.id) }
  367. this.wake()
  368. }
  369. }
  370. }
  371. async reconcile(owner: string, id: string): Promise<void> {
  372. this.store.require<Run>(owner, 'run', id)
  373. for (const w of this.works(owner, id)) this.recoverAfter.delete(w.id)
  374. this.wake()
  375. }
  376. async stop(owner: string, id: string): Promise<void> {
  377. const root = this.store.require<Run>(owner, 'run', id)
  378. if (terminal(root.status)) return
  379. root.stopRequested = true
  380. this.store.saveRun(owner, root)
  381. await this.cancelWorks(owner, this.works(owner, id))
  382. }
  383. async stopAgent(owner: string, conversationId: string, agentId: string, conversationTaskId?: string): Promise<void> {
  384. const c = this.store.require<Conversation>(owner, 'conversation', conversationId)
  385. const helper=this.store.get<Agent>(owner,'agent',agentId)
  386. const scopedTaskId = c.kind === 'group' ? this.store.resolveTask(owner, conversationId, conversationTaskId??helper?.temporaryGoalId)!.id : undefined
  387. if (!this.store.taskMemberIds(owner,c,scopedTaskId).includes(agentId)) throw new HttpError(400, '只能停止当前任务成员', 'invalid_members')
  388. await this.cancelWorks(owner, this.works(owner).filter(w => w.conversationId === conversationId && w.agentId === agentId && w.conversationTaskId === scopedTaskId))
  389. }
  390. async stopTask(owner: string, conversationId: string, conversationTaskId: string): Promise<void> {
  391. this.store.requireTask(owner, conversationId, conversationTaskId)
  392. const roots = this.store.list<Run>(owner, 'run').filter(r => r.conversationId === conversationId && r.conversationTaskId === conversationTaskId && !terminal(r.status))
  393. this.store.atomic(() => { for (const root of roots) { root.stopRequested = true; this.store.saveRun(owner, root) } })
  394. await this.cancelWorks(owner, this.works(owner).filter(w => w.conversationId === conversationId && w.conversationTaskId === conversationTaskId))
  395. }
  396. async stopConversation(owner: string, id: string): Promise<void> {
  397. this.store.require<Conversation>(owner, 'conversation', id)
  398. const roots = this.store.list<Run>(owner, 'run').filter(r => r.conversationId === id && !terminal(r.status))
  399. this.store.atomic(() => { for (const root of roots) { root.stopRequested = true; this.store.saveRun(owner, root) } })
  400. await this.cancelWorks(owner, this.works(owner).filter(w => roots.some(r => r.id === w.runId)))
  401. }
  402. private async cancelWorks(owner: string, works: Work[]): Promise<void> {
  403. await Promise.all(works.filter(w => !terminal(w.status)).map(async w => {
  404. const current = this.getWork(owner, w.id)
  405. current.cancelRequested = true
  406. if (current.status === 'queued') { current.status = 'interrupted'; this.saveWork(owner, current) }
  407. else {
  408. this.saveWork(owner, current)
  409. try { await this.interruptTurn(owner, current) }
  410. catch (error) {
  411. const latest = this.getWork(owner, current.id)
  412. if (terminal(latest.status)) return
  413. latest.status = 'uncertain'; latest.error = `停止状态待确认:${error instanceof Error ? error.message : '连接中断'}`
  414. this.saveWork(owner, latest); this.recoverAfter.delete(latest.id); this.retrySoon()
  415. }
  416. }
  417. }))
  418. this.wake()
  419. }
  420. protected closeScheduler(): void {
  421. this.closing = true
  422. if (this.retryTimer) clearTimeout(this.retryTimer)
  423. this.recoverAfter.clear()
  424. this.attempts.clear()
  425. }
  426. }