app.ts 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697
  1. import { unreadSnapshot } from './unreadCenter.js'
  2. import { jsonCompression } from './jsonCompression.js'
  3. import {ExecutionSettings} from './executionSettings.js'
  4. import {CredentialVaultCoordinator} from './credentialVault/coordinator.js'
  5. import {createCredentialVaultClient} from './credentialVault/localClient.js'
  6. import {ManagedBrowsers} from './managedBrowsers.js'
  7. import { isWorkspaceAccountPath } from './subaccountAccess.js'
  8. import { chatTranscriptRouter } from './chatTranscriptApi.js'
  9. import {DesktopEnvironments} from './desktopEnvironments.js'
  10. import {DesktopHostHub} from './desktopHosts.js'
  11. import type {GrokAuth} from './grokAuth.js'
  12. import {GrokCloud} from './grokCloud.js'
  13. import {WorkspaceInspector} from './workspaceInspector.js'
  14. import {WorkspaceRoutines} from './workspaceRoutines.js'
  15. import {WorkspaceRoutineTools} from './workspaceRoutineTools.js'
  16. import { WorkspacePlugins } from './botPlugins/workspacePlugins.js'
  17. import { nativeMessageFileText } from '../shared/messageFiles.js'
  18. import { hermesBotRelay } from './hermesBotRelay.js'
  19. import { remoteAgentExports } from './workspaceRemoteAgents.js'
  20. import { randomBytes, createHash } from 'node:crypto'
  21. import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
  22. import { createServer as createHttpServer, type Server as HttpServer } from 'node:http'
  23. import { createServer as createHttpsServer } from 'node:https'
  24. import { join } from 'node:path'
  25. import { bodyParser } from '@koa/bodyparser'
  26. import Koa from 'koa'
  27. import { RealtimeAPI } from './realtimeApi.js'
  28. import type { ServerConfig } from './config.js'
  29. import { loadServerConfig } from './config.js'
  30. import { errorMessage, HttpError } from './errors.js'
  31. import { NodePairingStore } from './pairing.js'
  32. import { LocalAuthStore, UpstreamServiceSession } from './localAuth.js'
  33. import { AccountLoginPairingStore } from './accountPairing.js'
  34. import { UpstreamProfileIdentityService } from './profileIdentities.js'
  35. import { createApiRouter } from './routes.js'
  36. import {
  37. applySecurityHeaders,
  38. CsrfProtection,
  39. isAllowedHostHeader,
  40. isExactOrigin,
  41. } from './security.js'
  42. import { UpstreamHttpError, UpstreamClient } from './upstream.js'
  43. import { createEmbeddedHermes, type EmbeddedHermes } from './hermes/embedded.js'
  44. import { readHostTools } from './hostToolSettings.js'
  45. import { UploadStore } from './uploads.js'
  46. import { WorkspaceStore } from './workspaceStore.js'
  47. import { WorkspaceNodes } from './workspaceGateway.js'
  48. import { WorkspaceRuntime } from './workspaceRuntime.js'
  49. import { WorkspaceMemorySynthesis } from './workspaceMemorySynthesis.js'
  50. import { WorkspaceAssets } from './workspaceAssets.js'
  51. import { workspaceRouter } from './workspaceRoutes.js'
  52. import { SystemUpdateManager } from './updateManager.js'
  53. import { PushCoordinator } from './pushCoordinator.js'
  54. import {
  55. APNsConfigurationManager,
  56. type APNsConfigurationSnapshot,
  57. } from './apnsConfiguration.js'
  58. import {
  59. FCMConfigurationManager,
  60. type FCMConfigurationSnapshot,
  61. } from './fcmConfiguration.js'
  62. import { OpenVikingConfigurationManager } from './openVikingConfiguration.js'
  63. import { OpenVikingService } from './openVikingService.js'
  64. import { OpenVikingSessionSync } from './openVikingSessionSync.js'
  65. import { PushCoordinatorEventAdapter } from './pushEventAdapter.js'
  66. import {
  67. AllowedHostsConfigurationManager,
  68. type AllowedHostsConfigurationSnapshot,
  69. } from './allowedHostsConfiguration.js'
  70. import {
  71. ChatPushJobManager,
  72. HermesChatNotificationResolver,
  73. } from './pushEvents.js'
  74. import { ChatCacheCoordinator, ChatCacheStore } from './chatCache.js'
  75. import {LocalVmService} from './localVm.js'
  76. import {SharedComputers} from './sharedComputers.js'
  77. import {ComputerControlService} from './computerControls.js'
  78. import { RunnerHub } from './runnerHub.js'
  79. export interface ApplicationOptions {
  80. deferBackground?: boolean
  81. config?: ServerConfig
  82. fetchImpl?: typeof fetch
  83. grokFetch?:typeof fetch
  84. csrfSecret?: Buffer
  85. pairings?: NodePairingStore
  86. uploads?: UploadStore
  87. updates?: SystemUpdateManager
  88. auth?: LocalAuthStore
  89. pluginFetch?: typeof fetch
  90. upstreamSession?: UpstreamServiceSession
  91. hermes?: EmbeddedHermes
  92. accountPairings?: AccountLoginPairingStore
  93. profileIdentities?: UpstreamProfileIdentityService
  94. push?: PushCoordinator
  95. apnsConfiguration?: APNsConfigurationManager
  96. fcmConfiguration?: FCMConfigurationManager
  97. openVikingConfiguration?: OpenVikingConfigurationManager
  98. allowedHostsConfiguration?: AllowedHostsConfigurationManager
  99. chatCache?: ChatCacheCoordinator
  100. }
  101. export interface ApplicationRuntime {
  102. credentialVault: CredentialVaultCoordinator
  103. runners: RunnerHub
  104. workspace: WorkspaceStore
  105. workspaceRuntime: WorkspaceRuntime
  106. realtime: RealtimeAPI
  107. app: Koa
  108. config: ServerConfig
  109. csrf: CsrfProtection
  110. pairings: NodePairingStore
  111. upstream: UpstreamClient
  112. uploads: UploadStore
  113. updates: SystemUpdateManager
  114. auth: LocalAuthStore
  115. upstreamSession: UpstreamServiceSession
  116. hermes: EmbeddedHermes
  117. accountPairings: AccountLoginPairingStore
  118. profileIdentities: UpstreamProfileIdentityService
  119. push: PushCoordinator
  120. apnsConfiguration: APNsConfigurationManager
  121. fcmConfiguration: FCMConfigurationManager
  122. openVikingConfiguration: OpenVikingConfigurationManager
  123. openVikingService: OpenVikingService
  124. openVikingSessionSync: OpenVikingSessionSync
  125. allowedHostsConfiguration: AllowedHostsConfigurationManager
  126. pushEventCoordinator: PushCoordinatorEventAdapter
  127. chatPushJobs: ChatPushJobManager
  128. chatCache?: ChatCacheCoordinator
  129. desktopEnvironments:DesktopEnvironments
  130. desktopHosts:DesktopHostHub
  131. grokAuth:GrokAuth
  132. workspaceRoutines:WorkspaceRoutines
  133. workspacePlugins:WorkspacePlugins
  134. localVm: LocalVmService
  135. startBackground(): void
  136. close(): void
  137. }
  138. function isMutation(method: string): boolean {
  139. return !['GET', 'HEAD', 'OPTIONS'].includes(method)
  140. }
  141. function persistentCsrfSecret(home: string, override?: Buffer): Buffer {
  142. if (override) return override
  143. const path = join(home, 'csrf-secret.bin')
  144. try {
  145. const secret = readFileSync(path)
  146. if (secret.byteLength === 32) return secret
  147. throw new Error('CSRF secret has an invalid length')
  148. } catch (error) {
  149. if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
  150. }
  151. mkdirSync(home, { recursive: true, mode: 0o700 })
  152. const generated = randomBytes(32)
  153. try {
  154. writeFileSync(path, generated, { mode: 0o600, flag: 'wx' })
  155. return generated
  156. } catch (error) {
  157. // Another 15300 process can finish first during launchd hand-off. Never
  158. // rotate its secret; use the one it safely wrote instead.
  159. if ((error as NodeJS.ErrnoException).code === 'EEXIST') return readFileSync(path)
  160. throw error
  161. }
  162. }
  163. export function createApplication(options: ApplicationOptions = {}): ApplicationRuntime {
  164. const config = options.config ?? loadServerConfig()
  165. const runtimeAllowedHosts = new Set(config.allowedHosts)
  166. config.allowedHosts = runtimeAllowedHosts
  167. const app = new Koa()
  168. app.proxy = false
  169. const csrf = new CsrfProtection(persistentCsrfSecret(config.home, options.csrfSecret), Boolean(config.tlsCert))
  170. const pairings = options.pairings ?? new NodePairingStore(config.home)
  171. const upstream = new UpstreamClient(config.upstream, options.fetchImpl, Boolean(config.tlsCert))
  172. // Embedded Hermes: the kernel runs as child processes under <data>/hermes; nothing listens on 9119.
  173. const hermes = options.hermes ?? createEmbeddedHermes(config.home, {
  174. defaultCwd: config.mediaRoot,
  175. policy: () => {
  176. // Same global switch the Hermes dashboard used for native approvals.
  177. try { const tools = readHostTools(config.home); return { approval: tools.approvalPolicy, denyHostTools: tools.denyServerTools } }
  178. catch { return { approval: 'ask', denyHostTools: false } }
  179. },
  180. })
  181. // `fetchImpl` is a test-only fake Dashboard transport; production always answers in process.
  182. if (!options.upstreamSession && !options.fetchImpl) upstream.local = hermes.api
  183. const uploads = options.uploads ?? new UploadStore(config.home)
  184. const updates = options.updates ?? new SystemUpdateManager(config)
  185. const auth = options.auth ?? new LocalAuthStore(config.home, Boolean(config.tlsCert))
  186. const configuredUpstreamCredentials = config.upstreamUsername && config.upstreamPassword
  187. ? { username: config.upstreamUsername, password: config.upstreamPassword }
  188. : undefined
  189. const upstreamSession = options.upstreamSession ?? new UpstreamServiceSession(
  190. upstream,
  191. () => auth.upstreamCredentials(configuredUpstreamCredentials),
  192. )
  193. let chatCache = options.chatCache
  194. if (!chatCache) {
  195. chatCache = new ChatCacheCoordinator(
  196. new ChatCacheStore(
  197. config.home,
  198. auth.soleUserIDForLegacyMigration(),
  199. ),
  200. upstreamSession,
  201. config.chatCacheMode ?? 'prefer-local',
  202. )
  203. }
  204. const accountPairings = options.accountPairings ?? new AccountLoginPairingStore()
  205. const profileIdentities = options.profileIdentities
  206. ?? new UpstreamProfileIdentityService(config, upstreamSession)
  207. const initialAPNsSettings: APNsConfigurationSnapshot = config.apnsSettings ?? {
  208. source: config.apns || config.apnsConfigurationError ? 'environment' : 'none',
  209. editable: !(config.apns || config.apnsConfigurationError),
  210. ...(config.apns ? {
  211. input: {
  212. ...config.apns,
  213. environments: [...(config.apns.environments ?? ['development', 'production'])],
  214. },
  215. config: config.apns,
  216. } : {}),
  217. warnings: [],
  218. ...(config.apnsConfigurationError ? { configurationError: config.apnsConfigurationError } : {}),
  219. }
  220. const apnsConfiguration = options.apnsConfiguration
  221. ?? new APNsConfigurationManager(config.home, initialAPNsSettings)
  222. const initialFCMSettings: FCMConfigurationSnapshot = config.fcmSettings ?? {
  223. source: config.fcm || config.fcmConfigurationError ? 'environment' : 'none',
  224. editable: !(config.fcm || config.fcmConfigurationError),
  225. ...(config.fcm ? { input: { ...config.fcm }, config: config.fcm } : {}),
  226. warnings: [],
  227. ...(config.fcmConfigurationError ? { configurationError: config.fcmConfigurationError } : {}),
  228. }
  229. const fcmConfiguration = options.fcmConfiguration
  230. ?? new FCMConfigurationManager(config.home, initialFCMSettings)
  231. const openVikingConfiguration = options.openVikingConfiguration
  232. ?? new OpenVikingConfigurationManager(config.home)
  233. const initialAllowedHostsSettings: AllowedHostsConfigurationSnapshot = config.allowedHostsSettings ?? {
  234. source: runtimeAllowedHosts.size ? 'environment' : 'none',
  235. hosts: [...runtimeAllowedHosts],
  236. editableHosts: [],
  237. environmentHosts: [...runtimeAllowedHosts],
  238. }
  239. const allowedHostsConfiguration = options.allowedHostsConfiguration
  240. ?? new AllowedHostsConfigurationManager(config.home, initialAllowedHostsSettings, runtimeAllowedHosts)
  241. const push = options.push ?? new PushCoordinator({
  242. home: config.home,
  243. apns: config.apns,
  244. apnsConfigurationError: config.apnsConfigurationError,
  245. fcm: config.fcm,
  246. fcmConfigurationError: config.fcmConfigurationError,
  247. isUserActive: userID => auth.isUserActive(userID),
  248. userAuthorizationVersion: userID => auth.pushAuthorizationVersion(userID),
  249. })
  250. const pushEventCoordinator = new PushCoordinatorEventAdapter(push)
  251. const chatPushJobs = new ChatPushJobManager(config, upstreamSession, pushEventCoordinator, {
  252. resolver: new HermesChatNotificationResolver(
  253. upstreamSession,
  254. (localUserID, prompt) => push.promptDigest(localUserID, prompt),
  255. ),
  256. })
  257. // The application owns group events. No Dashboard plugin is queried.
  258. const workspace = new WorkspaceStore(config.home)
  259. push.setUnreadCounter(owner => unreadSnapshot(workspace, chatCache!.store, owner,(node,profile)=>auth.canUseSource(owner,node,profile)).total)
  260. const openVikingService = new OpenVikingService(workspace, openVikingConfiguration)
  261. const workspaceNodes = new WorkspaceNodes(workspace, config, { url: config.upstream, client: upstream, session: upstreamSession, ...(upstream.embedded ? { embedded: hermes.gateway } : {}) }, pairings.nodeID)
  262. workspaceNodes.sourceAllowed = (owner, nodeId, profile) => auth.canUseSource(owner, nodeId, profile)
  263. const runners=new RunnerHub(workspace,auth,workspaceNodes.local)
  264. runners.hermesUpstream=config.upstream.origin
  265. runners.composeDesktops.desktops.push(...(config.composeDesktops??[]))
  266. const sharedComputers=new SharedComputers(workspace,auth,workspaceNodes,runners)
  267. const computerControls=new ComputerControlService(workspace,auth,workspaceNodes,runners)
  268. const localVm=new LocalVmService(workspace,auth,workspaceNodes,runners,sharedComputers,config)
  269. runners.localVmAllowed=(id,runnerId)=>localVm.allowed(id,runnerId)
  270. runners.controlAllowed=(id,runnerId)=>computerControls.allowed(id,runnerId)
  271. workspaceNodes.runnerTarget=(owner,nodeId,computer)=>runners.target(owner,nodeId,computer)
  272. const executionSettings=new ExecutionSettings(workspace,workspaceNodes,auth)
  273. const credentialVault=new CredentialVaultCoordinator(createCredentialVaultClient(config.credentialVault,config.home),workspace,auth,runners)
  274. const workspaceRuntime = new WorkspaceRuntime(workspace, workspaceNodes, uploads, owner => auth.isUserActive(owner), owner => auth.pushAuthorizationVersion(owner) ?? 0, openVikingService)
  275. workspaceRuntime.executionSettings=executionSettings
  276. workspaceRuntime.credentialVault=credentialVault
  277. runners.executionSettings=executionSettings
  278. executionSettings.onProxyChange=()=>runners.configureNetworks()
  279. executionSettings.testProxy=()=>runners.testNetworks()
  280. executionSettings.nodeStatus=()=>runners.networkStatus()
  281. executionSettings.activeTurns=()=>workspaceRuntime.activeExecutionTurns()
  282. const workspaceMemory = new WorkspaceMemorySynthesis(workspaceRuntime)
  283. const openVikingSessionSync = new OpenVikingSessionSync(workspace, openVikingService, owner => auth.isUserActive(owner))
  284. let backgroundStarted = false, closed = false
  285. const startBackground = () => {
  286. if (backgroundStarted || closed) return
  287. backgroundStarted = true
  288. workspaceMemory.start()
  289. openVikingSessionSync.start()
  290. }
  291. if (!options.deferBackground) startBackground()
  292. const grokCloud=new GrokCloud(workspace,auth,workspaceNodes,sharedComputers,options.grokFetch)
  293. const desktopEnvironments=new DesktopEnvironments(workspace,auth,workspaceNodes)
  294. const managedBrowsers=new ManagedBrowsers(workspace,auth,workspaceNodes,runners)
  295. workspaceRuntime.managedBrowsers=managedBrowsers
  296. runners.browserAllowed=(runnerId,scope,grantId)=>managedBrowsers.allowed(runnerId,scope,grantId)
  297. const desktopHosts=new DesktopHostHub(workspace,auth)
  298. desktopHosts.exchange=(id,value)=>desktopEnvironments.remoteExchange(id,value)
  299. desktopHosts.drop=id=>desktopEnvironments.dropHost(id)
  300. workspaceRuntime.desktopEnvironments=desktopEnvironments
  301. grokCloud.preferDesktop=(owner,agent)=>!!desktopEnvironments.selected(owner,agent)
  302. grokCloud.beforeSelection=(owner,id)=>desktopEnvironments.assertIdle(owner,id)
  303. workspaceRuntime.computerAvailable=(owner,agent)=>{try{runners.computerRunner(owner,agent);return true}catch{return false}}
  304. workspaceRuntime.cloud=grokCloud
  305. const workspaceInspector=new WorkspaceInspector(workspace,auth)
  306. workspaceRuntime.inspector=workspaceInspector
  307. const workspaceRoutines=new WorkspaceRoutines(workspace,auth,workspaceNodes,workspaceRuntime)
  308. workspaceRuntime.routineTools=new WorkspaceRoutineTools(workspaceRoutines,workspace,workspaceNodes,workspaceRuntime)
  309. const workspacePlugins=new WorkspacePlugins(workspace,workspaceNodes,auth,workspaceRuntime,config.home,options.pluginFetch)
  310. workspaceRuntime.plugins=workspacePlugins
  311. workspaceRuntime.retireHelper=(owner,helper)=>runners.retireHelper(owner,helper)
  312. workspaceRuntime.onTeamCreated = (owner, team) => {
  313. try { push.setGroupSubscription(owner, team.id, true, team.lastSeq) } catch { /* Optional notifications do not undo a team. */ }
  314. }
  315. const workspaceAssets = new WorkspaceAssets(workspace, workspaceNodes, config.home)
  316. workspaceRuntime.publishArtifact=(owner,message,agent,name,bytes)=>workspaceAssets.publish(owner,message,agent,name,bytes)
  317. workspaceRuntime.onMessage = (owner, message) => workspaceAssets.archive(owner, message)
  318. workspaceRuntime.onNotify = (owner, c, run, message, interaction) => {
  319. if (c.kind === 'group' && !push.isGroupSubscribed(owner,c.id)) return
  320. push.enqueueNotification({ kind: interaction ? interaction.kind === 'approval' ? 'chat.approval.requested' : 'chat.clarification.requested' : run.status === 'failed' ? 'chat.failed' : 'chat.completed', eventID: `workspace:${interaction?.id ?? run.id}`, localUserID: owner, title: c.name, body: (interaction?.message || message?.content || run.error || '回复完成').slice(0,180), collapseID: `workspace:${c.id}:${run.conversationTaskId ?? 'direct'}`, data: { conversationId:c.id, ...(run.conversationTaskId ? { conversationTaskId: run.conversationTaskId } : {}), workspace:'1' }, ...(c.kind === 'group' ? {roomID:c.id} : {}) })
  321. }
  322. const realtime = new RealtimeAPI(config, auth, csrf, pairings, upstream, upstreamSession, {
  323. coordinator: pushEventCoordinator,
  324. resolver: new HermesChatNotificationResolver(upstreamSession, (user, prompt) => push.promptDigest(user, prompt)),
  325. }, (owner, profile, sessionID) =>
  326. chatCache?.store.ownsSession(owner, profile, sessionID) ?? false)
  327. if (upstream.embedded) realtime.gateway = hermes.gateway
  328. realtime.broker.protectedSession = id => workspace.ownsUpstream(id)
  329. realtime.transcriptsAvailable = chatCache.store.transcripts.enabled
  330. realtime.broker.onNativeEvent = (owner, profile, storedId, frame) => {
  331. chatCache?.observe(owner, profile, storedId, frame)
  332. if (owner.startsWith('device:')) return
  333. if (!['message.complete', 'attachment.staged'].includes(frame.type)) return
  334. const data = frame.payload ?? {}, text = frame.type === 'attachment.staged' ? JSON.stringify(data) : nativeMessageFileText(data)
  335. const messageId = String(chatCache.store.projectedMessageID(owner,profile,storedId) ?? data.row_id ?? data.message_id ?? data.id ?? createHash('sha256').update(text).digest('hex'))
  336. void workspaceAssets.archiveText(owner, text, 'local', profile, storedId, messageId, frame.type === 'attachment.staged' ? 'user' : 'agent').catch(() => {})
  337. }
  338. workspace.nativeMessageForFile = (owner, file) => {
  339. const message = chatCache?.store.fileSourceMessage(owner, file.profile ?? 'default', file.conversationId ?? '', file.messageId ?? '')
  340. if (!message) return undefined
  341. return {
  342. id: file.messageId!, conversationId: file.conversationId!, seq: 0,
  343. role: message.role === 'assistant' ? 'assistant' : message.role === 'user' ? 'user' : 'system',
  344. content: nativeMessageFileText(message), reasoning: '', status: 'complete',
  345. attachments: [], tools: [], createdAt: file.createdAt,
  346. }
  347. }
  348. realtime.broker.onNativeGlobalEvent = (owner, type, upstreamInvalidation) => chatCache?.observeGlobal(owner, type, upstreamInvalidation)
  349. chatCache.onSynchronized = (owner, profile, sessionID) => {
  350. realtime.broker.publishOwnerSessionsChanged(owner, 'cache.synced', profile, sessionID)
  351. }
  352. realtime.broker.onNativeCommand = (owner, profile, storedId, method, params) => {
  353. chatCache?.command(owner, profile, storedId, method, params)
  354. }
  355. realtime.broker.nativeSourceCursor = (owner,profile,id,runtime,epoch)=>chatCache.store.transcripts.sourceCursor(owner,profile,id,runtime,epoch)
  356. realtime.broker.onNativeRoute = (owner, profile, storedId, runtimeId, running, snapshot) => {
  357. chatCache?.route(owner, profile, storedId, runtimeId, snapshot?undefined:running)
  358. if(snapshot)chatCache.store.recordEvent(owner,profile,storedId,{type:'route.resumed',payload:snapshot})
  359. }
  360. chatPushJobs.setTransportFactory(realtime.recoveryTransport, job => realtime.ownsPushJob(job.id))
  361. try {
  362. uploads.cleanupUncommitted()
  363. } catch {
  364. // A locked cleanup should not prevent startup; upload operations will
  365. // still surface the underlying storage error when used.
  366. }
  367. app.use(async (ctx, next) => {
  368. try {
  369. await next()
  370. } catch (error) {
  371. if (error instanceof UpstreamHttpError) {
  372. ctx.status = error.response.status
  373. ctx.type = 'application/json; charset=utf-8'
  374. ctx.body = { error: `Hermes returned HTTP ${error.response.status}`, code: error.code }
  375. return
  376. }
  377. const status = error instanceof HttpError
  378. ? error.status
  379. : typeof (error as { status?: unknown })?.status === 'number'
  380. ? Number((error as { status: number }).status)
  381. : 500
  382. const expose = status < 500 || error instanceof HttpError
  383. ctx.status = status
  384. ctx.type = 'application/json; charset=utf-8'
  385. ctx.body = {
  386. error: expose ? errorMessage(error) : 'Internal server error',
  387. code: error instanceof HttpError ? error.code : 'internal_error',
  388. }
  389. if (status >= 500 && !config.production) ctx.app.emit('error', error, ctx)
  390. }
  391. })
  392. app.use(async (ctx, next) => {
  393. applySecurityHeaders(ctx, Boolean(config.tlsCert), config.allowedHosts)
  394. if (!isAllowedHostHeader(ctx.get('host'), config)) {
  395. throw new HttpError(421, 'Request Host is not allowed', 'invalid_host')
  396. }
  397. await next()
  398. })
  399. app.use(async (ctx, next) => {
  400. if (ctx.path.startsWith('/api/app/') && isMutation(ctx.method)) {
  401. const secure = ctx.secure || Boolean(config.tlsCert)
  402. if (!isExactOrigin(ctx.get('origin'), ctx.get('host'), secure, config.allowedHosts)) {
  403. throw new HttpError(403, 'Request Origin is not allowed', 'invalid_origin')
  404. }
  405. if (!csrf.verify(ctx.get('cookie'), ctx.get('x-csrf-token'))) {
  406. throw new HttpError(403, 'CSRF token is invalid or missing', 'invalid_csrf')
  407. }
  408. }
  409. await next()
  410. })
  411. app.use(jsonCompression)
  412. app.use(runners.middleware())
  413. app.use(desktopHosts.middleware())
  414. app.use(async (ctx, next) => {
  415. const user = auth.current(ctx)
  416. if (user && user.role !== 'admin' && (ctx.path.startsWith('/api/') || ctx.path.startsWith('/node/') || ctx.path.startsWith('/ws/') || ctx.path.startsWith('/Users/') || ctx.path.startsWith('/attachments/'))) {
  417. const path = ctx.path
  418. const publicPath = ['/api/status', '/api/auth/providers', '/api/auth/me', '/api/profiles', '/api/app/bootstrap', '/api/app/login', '/api/app/logout'].includes(path)
  419. const accountPath = ['/api/account/credentials', '/api/app/account/credentials', '/api/app/account/avatar'].includes(path)
  420. const workspacePath = /^\/api\/app\/(?:capabilities|agents|conversations|runs|interactions|events|uploads|files|message-files|computers|bot-tools)(?:\/|$)/.test(path)
  421. && path !== '/api/app/agents/remote'
  422. const pushPath = /^\/api\/(?:app\/)?push\/v1\//.test(path)
  423. const identityRead = path === '/api/app/server-identity' && ['GET', 'HEAD'].includes(ctx.method)
  424. if (!publicPath && !accountPath && !workspacePath && !isWorkspaceAccountPath(ctx.method, path) && !pushPath && !identityRead)
  425. throw new HttpError(403, '子账号只能使用 Bot 模式', 'bot_mode_required')
  426. }
  427. await next()
  428. })
  429. app.use(hermesBotRelay(workspaceNodes, auth, csrf))
  430. app.use(remoteAgentExports(workspace,workspaceNodes,pairings,auth))
  431. app.use(realtime.middleware())
  432. const parseBody = (limit: string) => bodyParser({
  433. encoding: 'utf-8',
  434. enableTypes: ['json'],
  435. parsedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
  436. jsonLimit: limit,
  437. onError: (error, ctx) => {
  438. throw new HttpError(ctx.status === 413 || (error as { status?: number }).status === 413 ? 413 : 400, 'Invalid JSON request body', 'invalid_json_body')
  439. },
  440. })
  441. const ordinaryBody = parseBody('2mb'), workspaceBody = parseBody('4mb')
  442. app.use((ctx,next) => (/^\/api\/app\/(agents|conversations)(\/|$)/.test(ctx.path) || ctx.path === '/api/app/account/avatar' ? workspaceBody : ordinaryBody)(ctx,next))
  443. app.use(async (ctx, next) => {
  444. ctx.state.localAuth = auth
  445. ctx.state.upstreamSession = upstreamSession
  446. const anonymous = ctx.path === '/api/app/bootstrap'
  447. || ctx.path === '/api/app/setup'
  448. || ctx.path === '/api/app/login'
  449. || ctx.path === '/api/app/register'
  450. || ctx.path === '/api/status'
  451. || ctx.path === '/api/auth/providers'
  452. || ctx.path === '/auth/password-login'
  453. || ctx.path.startsWith('/api/pair/v1/')
  454. || ctx.path.startsWith('/api/account-pair/v1/')
  455. || ctx.path === '/healthz'
  456. || ctx.path === '/readyz'
  457. if (ctx.path.startsWith('/api/app/') && !anonymous) {
  458. const allowPasswordChange = ctx.path === '/api/app/account/credentials'
  459. || ctx.path === '/api/app/logout'
  460. ctx.state.localUser = auth.require(ctx, allowPasswordChange)
  461. const adminOnly = ctx.path.startsWith('/api/app/admin/')
  462. || ctx.path.startsWith('/api/app/system/')
  463. || ctx.path.startsWith('/api/app/plugins/')
  464. || ctx.path.startsWith('/api/app/pairings')
  465. || ctx.path.startsWith('/api/app/paired-devices')
  466. || ctx.path.startsWith('/api/app/groups/nodes')
  467. if (adminOnly) ctx.state.localUser = auth.requireAdmin(ctx)
  468. }
  469. const gatewayCompatibility = (ctx.path.startsWith('/api/') && !ctx.path.startsWith('/api/app/')
  470. && !ctx.path.startsWith('/api/pair/'))
  471. || ctx.path === '/auth/logout'
  472. if (gatewayCompatibility && !anonymous) {
  473. const allowPasswordChange = ctx.path === '/api/auth/me' || ctx.path === '/api/profiles'
  474. || ctx.path === '/api/account/credentials'
  475. ctx.state.localUser = auth.require(ctx, allowPasswordChange)
  476. if (isMutation(ctx.method)) {
  477. const origin = ctx.get('origin')
  478. if (origin && !isExactOrigin(origin, ctx.get('host'), ctx.secure || Boolean(config.tlsCert), config.allowedHosts)) {
  479. throw new HttpError(403, 'Request Origin is not allowed', 'invalid_origin')
  480. }
  481. }
  482. }
  483. const user = ctx.state.localUser as { id: string } | undefined
  484. if (user) {
  485. await upstream.withReadScope(`user:${user.id}:${auth.pushAuthorizationVersion(user.id)}`, ctx.get('x-yaoyao-cache') === 'bypass', next)
  486. } else await next()
  487. })
  488. app.use(async(ctx,next)=>{
  489. const user=ctx.state.localUser as {id:string}|undefined,cookie=ctx.get('cookie')
  490. if(['/api/app/logout','/auth/logout'].includes(ctx.path)&&ctx.method==='POST'&&user)await credentialVault.logout(user.id,cookie)
  491. await next()
  492. })
  493. for(const router of [credentialVault.router(),executionSettings.router(),managedBrowsers.router(),desktopEnvironments.router(),grokCloud.authorization.router(),grokCloud.router(),workspaceInspector.router(),workspaceRoutines.router(),workspacePlugins.router()]){app.use(router.routes());app.use(router.allowedMethods())}
  494. const sharedComputerRouter=sharedComputers.router();app.use(sharedComputerRouter.routes());app.use(sharedComputerRouter.allowedMethods())
  495. const localVmRouter=localVm.router();app.use(localVmRouter.routes());app.use(localVmRouter.allowedMethods())
  496. const computerRouter=computerControls.router();app.use(computerRouter.routes());app.use(computerRouter.allowedMethods())
  497. const applicationRouter = workspaceRouter(workspace, workspaceRuntime, workspaceNodes, workspaceAssets, uploads, auth, push, csrf)
  498. app.use(runners.adminRouter().routes())
  499. app.use(desktopHosts.adminRouter().routes())
  500. app.use(applicationRouter.routes())
  501. app.use(async (ctx, next) => {
  502. if (ctx.path.includes('/plugins/yaoyao') || ctx.path.startsWith('/api/app/groups')) throw new HttpError(410, '请使用新版聊天接口', 'retired_plugin_api')
  503. const id = /\/sessions\/([^/]+)/.exec(ctx.path)?.[1]
  504. if (id && workspace.ownsUpstream(decodeURIComponent(id))) throw new HttpError(404, '会话不存在', 'session_not_found')
  505. await next()
  506. // Native history is a separate surface, including search and profile projections.
  507. if (ctx.path.includes('/sessions') || ctx.path.endsWith('/profiles')) {
  508. let value: any = ctx.body
  509. const buffered = Buffer.isBuffer(value)
  510. try {
  511. if (buffered) value = JSON.parse(value.toString())
  512. const filter = (v: any): any => {
  513. if (Array.isArray(v)) return v.filter(x => !x || typeof x !== 'object' || (x.source !== 'yaoyao_workspace' && !workspace.ownsUpstream(String(x.id ?? x.session_id ?? x.sessionId ?? '')))).map(filter)
  514. if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k,x]) => [k, filter(x)]))
  515. return v
  516. }
  517. ctx.body = buffered ? Buffer.from(JSON.stringify(filter(value))) : filter(value)
  518. } catch { /* Preserve non-JSON error responses. */ }
  519. }
  520. })
  521. app.use(chatTranscriptRouter(chatCache, auth).routes())
  522. const router = createApiRouter({
  523. onChatListChanged: (owner,profile,id) => realtime.broker.publishOwnerSessionsChanged(owner,'local-state',profile,id),
  524. onServerIdentityChanged: identity => realtime.broker.publishServerIdentity(identity, config.upstream.href),
  525. onUserAccessChanged: async owner => {
  526. for (const c of workspace.list<{ id: string }>(owner, 'conversation')) await workspaceRuntime.stopConversation(owner, c.id)
  527. },
  528. workspace,
  529. config,
  530. csrf,
  531. upstream,
  532. pairings,
  533. uploads,
  534. updates,
  535. auth,
  536. upstreamSession,
  537. hermes,
  538. accountPairings,
  539. profileIdentities,
  540. push,
  541. apnsConfiguration,
  542. fcmConfiguration,
  543. openVikingConfiguration,
  544. openVikingService,
  545. openVikingSessionSync,
  546. allowedHostsConfiguration,
  547. chatCache,
  548. })
  549. app.use(router.routes())
  550. app.use(router.allowedMethods({ throw: false }))
  551. app.use(async (ctx, next) => {
  552. if (ctx.path === '/api' || ctx.path.startsWith('/api/') || ctx.path.startsWith('/ws/')) {
  553. ctx.status = 404
  554. ctx.type = 'application/json; charset=utf-8'
  555. ctx.body = { error: 'Route not found', code: 'not_found' }
  556. return
  557. }
  558. await next()
  559. })
  560. return {
  561. credentialVault,
  562. runners,
  563. localVm,
  564. workspace,
  565. workspaceRuntime,
  566. workspaceRoutines,
  567. workspacePlugins,
  568. desktopEnvironments,
  569. desktopHosts,
  570. grokAuth:grokCloud.authorization,
  571. realtime,
  572. app,
  573. config,
  574. csrf,
  575. pairings,
  576. upstream,
  577. uploads,
  578. updates,
  579. auth,
  580. upstreamSession,
  581. hermes,
  582. accountPairings,
  583. profileIdentities,
  584. push,
  585. apnsConfiguration,
  586. fcmConfiguration,
  587. openVikingConfiguration,
  588. openVikingService,
  589. openVikingSessionSync,
  590. allowedHostsConfiguration,
  591. pushEventCoordinator,
  592. chatPushJobs,
  593. chatCache,
  594. startBackground,
  595. close: () => {
  596. credentialVault.close()
  597. closed = true
  598. openVikingSessionSync.close()
  599. workspaceMemory.close()
  600. runners.close()
  601. workspaceAssets.close()
  602. managedBrowsers.close()
  603. desktopEnvironments.close()
  604. desktopHosts.close()
  605. grokCloud.authorization.close()
  606. executionSettings.close()
  607. workspaceRoutines.close()
  608. workspacePlugins.close()
  609. workspaceRuntime.close()
  610. workspaceNodes.close()
  611. upstream.close()
  612. if (!options.hermes) hermes.close()
  613. realtime.close()
  614. chatPushJobs.stop()
  615. push.close()
  616. uploads.close()
  617. workspace.close()
  618. chatCache?.close()
  619. chatCache?.store.close()
  620. },
  621. }
  622. }
  623. export interface NodeServerRuntime {
  624. server: HttpServer
  625. startBackground(): void
  626. close(closeConnections?: boolean): Promise<void>
  627. }
  628. export function createNodeServer(runtime: ApplicationRuntime, options: { deferBackground?: boolean } = {}): NodeServerRuntime {
  629. const { config } = runtime
  630. const server: HttpServer = config.tlsCert && config.tlsKey
  631. ? createHttpsServer({
  632. cert: readFileSync(config.tlsCert),
  633. key: readFileSync(config.tlsKey),
  634. }, runtime.app.callback())
  635. : createHttpServer(runtime.app.callback())
  636. let backgroundStarted = false, localVmStarted = false, closed = false
  637. const synchronizePushObservers = (enabled = runtime.push.isAnyProviderEnabled()) => {
  638. if (!backgroundStarted || closed) return
  639. if (enabled) {
  640. runtime.chatPushJobs.start()
  641. } else {
  642. runtime.chatPushJobs.stop()
  643. }
  644. }
  645. const startLocalVm = () => {
  646. if (!backgroundStarted || localVmStarted || closed) return
  647. const address = server.address()
  648. if (!address || typeof address === 'string') return
  649. localVmStarted = true
  650. void runtime.localVm.start(`${config.tlsCert ? 'https' : 'http'}://127.0.0.1:${address.port}`)
  651. .catch(error => console.error('本地虚拟机服务启动失败', error.message))
  652. }
  653. const startBackground = () => {
  654. if (backgroundStarted || closed) return
  655. backgroundStarted = true
  656. runtime.startBackground()
  657. runtime.workspaceRuntime.start()
  658. runtime.workspaceRoutines.start()
  659. runtime.grokAuth.start()
  660. synchronizePushObservers()
  661. startLocalVm()
  662. }
  663. server.once('listening', startLocalVm)
  664. if (!options.deferBackground) startBackground()
  665. const removePushConfigurationListener = runtime.push.onEnabledChange(synchronizePushObservers)
  666. const removeWebSockets = runtime.realtime.rejectLegacyUpgrades(server)
  667. return {
  668. server,
  669. startBackground,
  670. close: async (closeConnections = false) => {
  671. closed = true
  672. await runtime.localVm.close()
  673. removePushConfigurationListener()
  674. removeWebSockets()
  675. runtime.realtime.close()
  676. if (closeConnections) server.closeAllConnections()
  677. await new Promise<void>((resolve, reject) => {
  678. if (!server.listening) {
  679. resolve()
  680. return
  681. }
  682. server.close((error) => error ? reject(error) : resolve())
  683. })
  684. runtime.close()
  685. },
  686. }
  687. }