openVikingMemoryProvider.ts 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. import { createHash } from 'node:crypto'
  2. import { OpenVikingError } from '@openviking/sdk'
  3. import type { MemoryScope, WorkspaceMemory, WorkspaceMemoryRevision } from '../shared/workspaceKnowledge.js'
  4. import type { WorkspaceMessage } from '../shared/workspace.js'
  5. import { HttpError } from './errors.js'
  6. import type { OpenVikingService } from './openVikingService.js'
  7. import type { WorkspaceKnowledge, MemoryActor, MemoryScopeInput, MemoryWriteInput, MemoryWriteResult } from './workspaceKnowledge.js'
  8. type Revision = WorkspaceMemoryRevision
  9. interface StoredRecord {
  10. schemaVersion: 1
  11. memory: WorkspaceMemory
  12. revisions: Revision[]
  13. requestFingerprint: string
  14. }
  15. interface Tombstone { at: number }
  16. const sha = (value: string): string => createHash('sha256').update(value).digest('hex')
  17. const normalize = (value: string): string => value.normalize('NFC').replace(/\s+/gu, ' ').trim()
  18. const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`
  19. const isUuid = (value: string): boolean => /^[0-9a-f]{32}$/.test(value)
  20. const memoryId = (requestId: string): string => requestId.replaceAll('-', '')
  21. const MEMORY_TAG = 'yaoyao_memory=true'
  22. const MEMORY_READ_CONCURRENCY = 8
  23. const activeUri = (scope: MemoryScopeInput, id: string): string => `viking://~/memories/yaoyao/${scope.scope}/${id}.json`
  24. const revisionsUri = (scope: MemoryScopeInput, id: string, revision: number): string => `viking://resources/yaoyao/${scope.scope}/${id}/revisions/${revision}.json`
  25. const tombstoneUri = (scope: MemoryScopeInput, fingerprint: string): string => `viking://resources/yaoyao/${scope.scope}/tombstones/${fingerprint}.json`
  26. const requestUri = (requestId: string): string => `viking://resources/yaoyao/requests/${requestId.replaceAll('-', '')}.json`
  27. const fingerprintUri = (scope: MemoryScopeInput, fingerprint: string): string => `viking://resources/yaoyao/${scope.scope}/fingerprints/${fingerprint}.json`
  28. const forgottenSourcesUri = (scope: MemoryScopeInput): string => `viking://resources/yaoyao/${scope.scope}/forgotten-sources.json`
  29. function assertScope(scope: MemoryScopeInput): void {
  30. if (scope.scope === 'project' && !/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}$/.test(scope.projectId ?? '')) throw new HttpError(400, '需要选择项目', 'project_required')
  31. }
  32. async function notFoundAsUndefined<T>(operation: () => Promise<T>): Promise<T | undefined> {
  33. try { return await operation() } catch (error) {
  34. if (error instanceof HttpError && error.status === 404) return undefined
  35. if (error instanceof OpenVikingError && (error.statusCode === 404 || error.code === 'NOT_FOUND' || error.code === 'ENOENT')) return undefined
  36. throw error
  37. }
  38. }
  39. function uriName(value: unknown): string | undefined {
  40. if (typeof value === 'string') return value
  41. if (value && typeof value === 'object') {
  42. const record = value as Record<string, unknown>
  43. for (const key of ['uri', 'path', 'name', 'id']) if (typeof record[key] === 'string') return record[key] as string
  44. }
  45. return undefined
  46. }
  47. export class OpenVikingMemoryProvider {
  48. readonly #locks = new Map<string, Promise<unknown>>()
  49. constructor(private readonly service: OpenVikingService, private readonly knowledge: WorkspaceKnowledge) {}
  50. private async withLock(owner: string, agentId: string, operation: () => Promise<void>): Promise<void> {
  51. const key = `${owner}\0${agentId}`
  52. const previous = this.#locks.get(key) ?? Promise.resolve()
  53. const next = previous.catch(() => undefined).then(operation)
  54. this.#locks.set(key, next)
  55. try { await next } finally { if (this.#locks.get(key) === next) this.#locks.delete(key) }
  56. }
  57. private async ensure(owner: string, agentId: string): Promise<void> {
  58. if (!this.service.enabled) return
  59. const agent = this.knowledge.agent(owner, agentId)
  60. await this.service.ensureUser(owner, agent)
  61. }
  62. private async readRecord(owner: string, scope: MemoryScopeInput, id: string): Promise<StoredRecord | undefined> {
  63. assertScope(scope)
  64. const raw = await notFoundAsUndefined(() => this.service.userClient(owner, scope.agentId).read(activeUri(scope, id)))
  65. if (!raw) return undefined
  66. try {
  67. const value = JSON.parse(raw) as StoredRecord
  68. if (value.schemaVersion !== 1 || !value.memory || !Array.isArray(value.revisions)) throw new Error('invalid')
  69. return value
  70. } catch { throw new HttpError(409, 'OpenViking 记忆文件损坏', 'openviking_memory_corrupt') }
  71. }
  72. private async readJson<T>(owner: string, agentId: string, uri: string, fallback: T): Promise<T> {
  73. const raw = await notFoundAsUndefined(() => this.service.userClient(owner, agentId).read(uri))
  74. if (!raw) return fallback
  75. try { return JSON.parse(raw) as T } catch { throw new HttpError(409, 'OpenViking 记忆审计文件损坏', 'openviking_memory_corrupt') }
  76. }
  77. private async listScope(owner: string, scope: MemoryScopeInput): Promise<WorkspaceMemory[]> {
  78. assertScope(scope)
  79. const client = this.service.userClient(owner, scope.agentId)
  80. const directory = `viking://~/memories/yaoyao/${scope.scope}`
  81. const entries = await notFoundAsUndefined(() => client.list(directory, { recursive: true })) ?? []
  82. const ids = entries.flatMap(entry => {
  83. const name = uriName(entry)
  84. const match = /([0-9a-f]{32})\.json$/.exec(name ?? '')
  85. return match ? [match[1]!] : []
  86. })
  87. const records: Array<WorkspaceMemory | undefined> = new Array(ids.length)
  88. let next = 0, failed = false
  89. // Preserve directory order and per-record scope checks without N serial round trips.
  90. const reads = await Promise.allSettled(Array.from({ length: Math.min(ids.length, MEMORY_READ_CONCURRENCY) }, async () => {
  91. while (!failed) {
  92. const index = next++
  93. if (index >= ids.length) return
  94. let record: StoredRecord | undefined
  95. try { record = await this.readRecord(owner, scope, ids[index]!) }
  96. catch (error) { failed = true; throw error }
  97. if (record && record.memory.scope === scope.scope && record.memory.agentId === scope.agentId
  98. && record.memory.projectId === scope.projectId) records[index] = record.memory
  99. }
  100. }))
  101. const failure = reads.find(read => read.status === 'rejected')
  102. if (failure?.status === 'rejected') throw failure.reason
  103. return records.filter((memory): memory is WorkspaceMemory => memory !== undefined)
  104. }
  105. async list(owner: string, scopes: MemoryScopeInput[]): Promise<WorkspaceMemory[]> {
  106. const result: WorkspaceMemory[] = []
  107. for (const scope of scopes) {
  108. await this.ensure(owner, scope.agentId)
  109. result.push(...await this.listScope(owner, scope))
  110. }
  111. return result
  112. }
  113. async writeWithResult(owner: string, input: MemoryWriteInput, actor: MemoryActor = {}): Promise<MemoryWriteResult> {
  114. assertScope(input)
  115. this.knowledge.authorizeMemory(owner, input, actor, true)
  116. if (input.scope === 'project') this.knowledge.requireProjectMember(owner, input.projectId!, input.agentId)
  117. let output: WorkspaceMemory | undefined
  118. let outcome: MemoryWriteResult['outcome'] = 'replayed'
  119. await this.withLock(owner, input.agentId, async () => {
  120. await this.ensure(owner, input.agentId)
  121. const client = this.service.userClient(owner, input.agentId)
  122. const requestFingerprint = sha(JSON.stringify({ operation: 'memory.write', input, actor }))
  123. const savedRequest = await this.readJson<{ fingerprint: string; memory: WorkspaceMemory } | undefined>(owner, input.agentId, requestUri(input.requestId), undefined)
  124. if (savedRequest) {
  125. if (savedRequest.fingerprint !== requestFingerprint) throw new HttpError(409, '请求标识已用于其他内容', 'idempotency_conflict')
  126. output = savedRequest.memory
  127. return
  128. }
  129. const all = await this.listScope(owner, input), old = input.id ? all.find(memory => memory.id === input.id) : undefined
  130. if (input.id && !old) throw new HttpError(404, '记忆不存在', 'memory_not_found')
  131. if (old && old.revision !== input.expectedRevision) throw new HttpError(409, '记忆已更新,请重新读取后保存;草稿已保留', 'memory_revision_conflict')
  132. if (actor.agentId && old?.origin === 'manual') throw new HttpError(403, '用户维护的记忆不能被机器人覆盖', 'memory_manual_protected')
  133. if (input.automatic && old && old.origin !== 'synthesis') throw new HttpError(403, '自动整理不能覆盖明确记忆', 'memory_explicit_protected')
  134. const content = normalize(input.content)
  135. if (!content || content.length > 2000) throw new HttpError(400, '每条记忆需要 1 至 2000 个字符', 'memory_content_invalid')
  136. const duplicate = all.find(memory => memory.id !== old?.id && memory.content.toLowerCase() === content.toLowerCase())
  137. if (duplicate) { output = duplicate; outcome = 'duplicate' }
  138. else {
  139. const fingerprint = sha(content.toLowerCase())
  140. const tombstone = await this.readJson<Tombstone | undefined>(owner, input.agentId, tombstoneUri(input, fingerprint), undefined)
  141. const forgottenSources = await this.readJson<string[]>(owner, input.agentId, forgottenSourcesUri(input), [])
  142. if (input.automatic && (tombstone || input.sources?.some(source => forgottenSources.includes(source.messageId)))) throw new HttpError(409, '此记忆或其旧来源已被遗忘', 'memory_forgotten')
  143. const sources = input.sources ?? old?.sources ?? []
  144. if (actor.agentId && !sources.length) throw new HttpError(400, '保存记忆需要来源消息', 'memory_source_required')
  145. for (const source of sources) {
  146. const message = this.knowledge.store.require<WorkspaceMessage>(owner, 'message', source.messageId)
  147. if (message.conversationId !== source.conversationId || message.visible === false || (source.quote && !message.content.includes(source.quote))) throw new HttpError(400, '记忆来源无效', 'memory_source_invalid')
  148. if (actor.agentId && input.scope === 'user' && (message.role !== 'user' || !source.quote?.trim())) throw new HttpError(400, '用户共享记忆需要用户原话依据', 'memory_user_evidence_required')
  149. }
  150. const now = Date.now()
  151. const memory: WorkspaceMemory = {
  152. id: old?.id ?? memoryId(input.requestId), scope: input.scope, agentId: input.agentId,
  153. ...(input.projectId ? { projectId: input.projectId } : {}), content, tier: input.tier,
  154. origin: actor.agentId ? input.automatic ? 'synthesis' : 'explicit' : 'manual', sources,
  155. revision: (old?.revision ?? 0) + 1, createdAt: old?.createdAt ?? now, updatedAt: now,
  156. }
  157. if (input.topic || old?.topic) memory.topic = normalize(input.topic ?? old!.topic!).slice(0, 100)
  158. const revision: Revision = { id: memoryId(`${input.requestId}-${memory.revision}`), memoryId: memory.id, revision: memory.revision, operation: 'write', actor: actor.agentId ?? 'user', at: now, before: old, after: memory }
  159. const previous = await this.readRecord(owner, input, memory.id)
  160. const record: StoredRecord = { schemaVersion: 1, memory, revisions: [...(previous?.revisions ?? []), revision], requestFingerprint }
  161. const uri = activeUri(input, memory.id)
  162. await client.write(uri, json(record), { wait: true })
  163. await client.setTags(uri, [MEMORY_TAG, `yaoyao_scope=${input.scope}`, `yaoyao_memory_id=${memory.id}`, ...(input.projectId ? [`yaoyao_project=${input.projectId}`] : [])])
  164. await client.write(revisionsUri(input, memory.id, memory.revision), json(revision), { wait: true })
  165. await client.write(requestUri(input.requestId), json({ fingerprint: requestFingerprint, memory }), { wait: true })
  166. await client.write(fingerprintUri(input, fingerprint), json({ id: memory.id, revision: memory.revision }), { wait: true })
  167. if (old && old.content.toLowerCase() !== content.toLowerCase() && memory.origin !== 'synthesis') {
  168. const oldFingerprint = sha(old.content.toLowerCase())
  169. await client.write(tombstoneUri(input, oldFingerprint), json({ at: now }), { wait: true })
  170. await client.write(forgottenSourcesUri(input), json([...new Set([...forgottenSources, ...old.sources.map(source => source.messageId)])]), { wait: true })
  171. }
  172. if (!input.automatic) await client.remove(tombstoneUri(input, fingerprint), { wait: true }).catch(() => undefined)
  173. output = memory
  174. outcome = 'written'
  175. }
  176. await client.write(requestUri(input.requestId), json({ fingerprint: requestFingerprint, memory: output! }), { wait: true })
  177. })
  178. this.knowledge.emitMemoryChanged(owner)
  179. return { memory: output!, outcome }
  180. }
  181. async forget(owner: string, input: MemoryScopeInput & { requestId: string; id: string; expectedRevision: number }, actor: MemoryActor = {}): Promise<{ id: string; scope: MemoryScope; agentId: string; projectId?: string }> {
  182. assertScope(input)
  183. this.knowledge.authorizeMemory(owner, input, actor, true)
  184. let output: { id: string; scope: MemoryScope; agentId: string; projectId?: string } | undefined
  185. const requestFingerprint = sha(JSON.stringify({ operation: 'memory.forget', input, actor }))
  186. await this.withLock(owner, input.agentId, async () => {
  187. await this.ensure(owner, input.agentId)
  188. const savedRequest = await this.readJson<{ fingerprint: string; result?: { id: string; scope: MemoryScope; agentId: string; projectId?: string } } | undefined>(owner, input.agentId, requestUri(input.requestId), undefined)
  189. if (savedRequest?.result) {
  190. if (savedRequest.fingerprint !== requestFingerprint) throw new HttpError(409, '请求标识已用于其他内容', 'idempotency_conflict')
  191. output = savedRequest.result
  192. return
  193. }
  194. const old = await this.readRecord(owner, input, input.id)
  195. if (!old) throw new HttpError(404, '记忆不存在', 'memory_not_found')
  196. if (old.memory.revision !== input.expectedRevision) throw new HttpError(409, '记忆已更新,请重新读取后操作', 'memory_revision_conflict')
  197. if (actor.agentId && old.memory.origin === 'manual') throw new HttpError(403, '用户维护的记忆不能被机器人删除', 'memory_manual_protected')
  198. const client = this.service.userClient(owner, input.agentId), now = Date.now(), fingerprint = sha(old.memory.content.toLowerCase())
  199. const revision: Revision = { id: memoryId(`${input.requestId}-${old.memory.revision + 1}`), memoryId: old.memory.id, revision: old.memory.revision + 1, operation: 'forget', actor: actor.agentId ?? 'user', at: now, before: old.memory }
  200. await client.remove(activeUri(input, old.memory.id), { wait: true })
  201. await client.write(revisionsUri(input, old.memory.id, revision.revision), json(revision), { wait: true })
  202. await client.write(tombstoneUri(input, fingerprint), json({ at: now }), { wait: true })
  203. await client.write(forgottenSourcesUri(input), json([...new Set([...await this.readJson<string[]>(owner, input.agentId, forgottenSourcesUri(input), []), ...old.memory.sources.map(source => source.messageId)])]), { wait: true })
  204. output = { id: old.memory.id, scope: old.memory.scope, agentId: old.memory.agentId, ...(old.memory.projectId ? { projectId: old.memory.projectId } : {}) }
  205. await client.write(requestUri(input.requestId), json({ fingerprint: requestFingerprint, result: output }), { wait: true })
  206. })
  207. this.knowledge.emitMemoryChanged(owner)
  208. return output!
  209. }
  210. async revisions(owner: string, scope: MemoryScopeInput, id: string): Promise<Revision[]> {
  211. assertScope(scope)
  212. this.knowledge.authorizeMemory(owner, scope, {})
  213. await this.ensure(owner, scope.agentId)
  214. if (!isUuid(id)) return []
  215. const directory = `viking://resources/yaoyao/${scope.scope}/${id}/revisions`
  216. const entries = await notFoundAsUndefined(() => this.service.userClient(owner, scope.agentId).list(directory, { recursive: true })) ?? []
  217. const revisions: Revision[] = []
  218. for (const entry of entries) {
  219. const name = uriName(entry), match = /(\d+)\.json$/.exec(name ?? '')
  220. if (match) {
  221. const revision = await this.readJson<Revision>(owner, scope.agentId, revisionsUri(scope, id, Number(match[1])), {} as Revision)
  222. if (revision.id) revisions.push(revision)
  223. }
  224. }
  225. return revisions.sort((left, right) => right.revision - left.revision)
  226. }
  227. async search(owner: string, query: { scope: MemoryScope; agentId?: string; projectId?: string; search?: string }, scopes: MemoryScopeInput[], actor: MemoryActor = {}): Promise<WorkspaceMemory[]> {
  228. const exact = await this.list(owner, scopes.filter(scope => {
  229. this.knowledge.authorizeMemory(owner, scope, actor)
  230. return true
  231. }))
  232. if (!query.search) return exact
  233. const wanted = new Set<string>()
  234. for (const scope of scopes) {
  235. const found = await notFoundAsUndefined(() => this.service.userClient(owner, scope.agentId).find(query.search!, { targetUri: `viking://~/memories/yaoyao/${query.scope}`, limit: 100, tags: [MEMORY_TAG] }))
  236. if (!found) continue
  237. for (const memory of Array.isArray(found.memories) ? found.memories : []) {
  238. const uri = uriName(memory) ?? uriName((memory as Record<string, unknown>).source)
  239. const match = /([0-9a-f]{32})\.json$/.exec(uri ?? '')
  240. if (match) wanted.add(match[1]!)
  241. }
  242. }
  243. return exact.filter(memory => wanted.has(memory.id) || memory.content.toLowerCase().includes(query.search!.toLowerCase()))
  244. }
  245. }