managedBrowsers.ts 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. import {createHash,randomBytes,randomUUID,timingSafeEqual} from 'node:crypto'
  2. import Router from '@koa/router'
  3. import {z} from 'zod'
  4. import {HttpError} from './errors.js'
  5. import {readHostTools} from './hostToolSettings.js'
  6. import {parse,type WorkspaceStore} from './workspaceStore.js'
  7. import type {LocalAuthStore} from './localAuth.js'
  8. import type {WorkspaceNodes} from './workspaceGateway.js'
  9. import type {RunnerHub} from './runnerHub.js'
  10. import type {WorkspaceAgent} from '../shared/workspace.js'
  11. import type {ComputerFrame,ComputerControlStatus} from '../shared/computerControl.js'
  12. import type {BrowserScope,BrowserState,BrowserUpload} from '../runner/browser/types.js'
  13. import type {BrowserInstallation,ManagedBrowserCard,ManagedBrowserState} from '../shared/managedBrowser.js'
  14. import {ToolOperationJournal} from './toolOperationJournal.js'
  15. const digest=(value:string)=>createHash('sha256').update(value).digest('hex')
  16. const credentials=z.object({controlId:z.string().uuid(),token:z.string().min(32).max(256)}).passthrough()
  17. type RuntimeState=BrowserState&{available?:boolean;installation?:BrowserInstallation}
  18. interface Grant {id:string;role:'task'|'human'|'view'|'prepare'|'session';resource:Resource;expiresAt:number;authVersion:number;authorize():void;signal?:AbortSignal;parking?:boolean;requestId?:string;sealedToken?:string;tokenHash?:string}
  19. interface Resource {owner:string;agent:WorkspaceAgent;runnerId:string;runnerEpoch?:string;authVersion:number;scope:BrowserScope;state:RuntimeState;task?:Grant;human?:Grant;session?:Grant;parked?:boolean;nextStatusAt?:number;refreshing?:Promise<void>;preparation?:Grant;notify?:()=>void;paused:boolean;forceClose?:boolean;retired?:boolean;resetRequired?:boolean;resetting?:Promise<void>;transition?:Promise<unknown>;taking?:Promise<void>;closing?:Promise<unknown>;frame?:ComputerFrame;frames?:Map<string,Omit<ComputerFrame,'data'>>;notes?:string}
  20. export interface BrowserTurn {
  21. call(name:string,args:unknown,callId?:string):Promise<unknown>
  22. close(options?:{retain?:boolean}):Promise<void>
  23. }
  24. export interface BrowserTurnOptions {
  25. workId:string;signal:AbortSignal;authorize():void
  26. publish(name:string,bytes:Buffer):Promise<unknown>
  27. upload(fileId:string):Promise<BrowserUpload>
  28. /** Return false when persistence failed so the next observation can retry. */
  29. onCard?(card:ManagedBrowserCard):boolean|void
  30. toVm?(file:BrowserUpload,path:string,overwrite:boolean):Promise<unknown>
  31. fromVm?(path:string):Promise<BrowserUpload>
  32. }
  33. /** Browser grants share Runner authentication, owner/source policy and task lifetime.
  34. * They do not grant host/VM tools or reactivate the retired Electron browser path. */
  35. export class ManagedBrowsers {
  36. private resources=new Map<string,Resource>()
  37. private grants=new Map<string,Grant>()
  38. private journal:ToolOperationJournal
  39. private timer:ReturnType<typeof setInterval>
  40. private closed=false
  41. constructor(readonly store:WorkspaceStore,readonly auth:LocalAuthStore,readonly nodes:WorkspaceNodes,readonly hub:RunnerHub){
  42. this.journal=new ToolOperationJournal(store)
  43. this.timer=setInterval(()=>void this.sweep(),5000);this.timer.unref()
  44. }
  45. private agent(owner:string,id:string){
  46. if(this.closed||!readHostTools(this.store.home).managedBrowser)throw new HttpError(409,'托管浏览器未启用','browser_disabled')
  47. const agent=this.store.require<WorkspaceAgent>(owner,'agent',id)
  48. if(agent.archived||agent.remoteAgentId)throw new HttpError(409,'当前 Bot 没有托管浏览器','browser_unavailable')
  49. this.nodes.requireSource(owner,agent)
  50. return agent
  51. }
  52. available(owner:string,agent:WorkspaceAgent){try{this.agent(owner,agent.id);this.hub.browserRunner(owner,agent);return true}catch{return false}}
  53. private resource(owner:string,id:string){
  54. const agent=this.agent(owner,id),runner=this.hub.browserRunner(owner,agent),key=owner+':'+id
  55. let resource=this.resources.get(key),resetRequired=false
  56. if(resource&&(resource.retired||resource.authVersion!==(this.auth.pushAuthorizationVersion(owner)??0)||resource.runnerId!==runner.id||resource.runnerEpoch!==runner.browserEpoch||resource.agent.profile!==agent.profile||resource.agent.nodeId!==agent.nodeId)){
  57. // Source changes revoke every old holder. The old Runner observes the
  58. // revoked grant; the current Runner must confirm a close before reuse.
  59. this.retire(resource)
  60. resetRequired=true
  61. this.resources.delete(key);resource=undefined
  62. }
  63. if(!resource){
  64. const scope:BrowserScope={ownerKey:digest(owner),environmentId:agent.id,profile:agent.temporaryGoalId?'temporary':'persistent'}
  65. resource={owner,agent,runnerId:runner.id,runnerEpoch:runner.browserEpoch,authVersion:this.auth.pushAuthorizationVersion(owner)??0,scope,state:{open:false,generation:1,profile:scope.profile,tabs:[],downloads:[]},paused:false,resetRequired}
  66. this.resources.set(key,resource)
  67. }
  68. return resource
  69. }
  70. private grant(resource:Resource,role:Grant['role'],authorize:()=>void,signal?:AbortSignal,expiresAt=Infinity){
  71. const grant:Grant={id:randomUUID(),resource,role,authorize,signal,expiresAt,authVersion:this.auth.pushAuthorizationVersion(resource.owner)??0}
  72. this.grants.set(grant.id,grant);return grant
  73. }
  74. private check(grant:Grant){
  75. const r=grant.resource
  76. if(this.resources.get(r.owner+':'+r.agent.id)!==r||this.grants.get(grant.id)!==grant||grant.expiresAt<=Date.now()||grant.signal?.aborted||!this.auth.isUserActive(r.owner)||(this.auth.pushAuthorizationVersion(r.owner)??0)!==grant.authVersion)throw new HttpError(410,'浏览器操作授权已结束','browser_authorization_revoked')
  77. this.checkResource(r)
  78. grant.authorize()
  79. }
  80. private checkResource(r:Resource){
  81. if(r.retired||this.resources.get(r.owner+':'+r.agent.id)!==r||!this.auth.isUserActive(r.owner)||(this.auth.pushAuthorizationVersion(r.owner)??0)!==r.authVersion)throw new HttpError(410,'浏览器操作授权已结束','browser_authorization_revoked')
  82. const agent=this.agent(r.owner,r.agent.id),runner=this.hub.browserRunner(r.owner,agent)
  83. if(agent.nodeId!==r.agent.nodeId||agent.profile!==r.agent.profile||runner.id!==r.runnerId||runner.browserEpoch!==r.runnerEpoch)throw new HttpError(410,'浏览器执行来源已改变','browser_source_changed')
  84. }
  85. private retire(r:Resource){
  86. for(const [id,g] of this.grants)if(g.resource===r)this.grants.delete(id)
  87. r.retired=true;r.task=undefined;r.human=undefined;r.session=undefined;r.preparation=undefined;r.paused=false;r.notify?.()
  88. }
  89. private dropSession(r:Resource){if(r.session)this.grants.delete(r.session.id);r.session=undefined;r.parked=false}
  90. private keepSession(r:Resource){
  91. if(!r.forceClose&&r.scope.profile==='persistent'&&this.hub.supportsBrowserRetention(r.owner,r.agent))r.session??=this.grant(r,'session',()=>{})
  92. }
  93. private transition<T>(r:Resource,action:()=>Promise<T>):Promise<T>{
  94. const pending=(r.transition??Promise.resolve()).catch(()=>{}).then(action)
  95. r.transition=pending
  96. void pending.finally(()=>{if(r.transition===pending)r.transition=undefined}).catch(()=>{})
  97. return pending
  98. }
  99. private async rest(r:Resource,retain:boolean){
  100. this.checkResource(r)
  101. r.frame=undefined;r.frames?.clear();r.paused=false
  102. if(retain&&!r.forceClose&&r.scope.profile==='persistent'&&r.state.open&&this.hub.supportsBrowserRetention(r.owner,r.agent)){
  103. const session=r.session??=this.grant(r,'session',()=>{})
  104. try{
  105. const state=await this.request(session,'park')
  106. if(!state||typeof state.open!=='boolean'||!Number.isInteger(state.generation))throw new HttpError(502,'浏览器未确认页面保留','browser_park_unconfirmed')
  107. if(r.state.open){r.parked=true;r.nextStatusAt=Date.now()+30000}else this.dropSession(r)
  108. return
  109. }
  110. catch{this.dropSession(r)}
  111. }
  112. this.dropSession(r)
  113. try{await this.view(r,g=>this.request(g,'close'));r.forceClose=false}
  114. catch(error){r.resetRequired=true;throw error}
  115. }
  116. private async activate(g:Grant){
  117. while(g.resource.parked){
  118. await this.waitForControl(g)
  119. await this.transition(g.resource,async()=>{
  120. const r=g.resource;this.check(g)
  121. if(r.paused||!r.parked)return
  122. await this.request(g,'resume');r.parked=false;r.frame=undefined;r.frames?.clear()
  123. })
  124. }
  125. }
  126. allowed(runnerId:string,scope:unknown,grantId?:string){
  127. if(!scope||typeof scope!=='object')return false
  128. const s=scope as BrowserScope
  129. const matches=(g:Grant)=>g.resource.runnerId===runnerId&&g.resource.scope.ownerKey===s.ownerKey&&g.resource.scope.environmentId===s.environmentId&&g.resource.scope.profile===s.profile
  130. const valid=(g:Grant)=>{try{this.check(g);return true}catch{return false}}
  131. if(grantId){const g=this.grants.get(grantId);return !!g&&(g.role!=='session'||g.parking===true)&&matches(g)&&valid(g)}
  132. return [...this.grants.values()].some(g=>g.role!=='prepare'&&matches(g)&&valid(g))
  133. }
  134. private async request(grant:Grant,op:string,data:Record<string,unknown>={}){
  135. this.check(grant)
  136. if(grant.role==='session'&&op!=='park')throw new HttpError(403,'保留会话不授权浏览器操作','browser_tool_forbidden')
  137. if(grant.role==='task'&&!['status','close','prepare','resume'].includes(op))await this.waitForControl(grant)
  138. const r=grant.resource
  139. if(r.resetRequired){
  140. r.resetting??=(async()=>{
  141. const result=await this.hub.browser(r.owner,r.agent,{scope:r.scope,grantId:grant.id,op:'close'},()=>this.check(grant))
  142. r.state=result;r.resetRequired=false
  143. })().finally(()=>{r.resetting=undefined})
  144. await r.resetting
  145. this.check(grant)
  146. }
  147. if(grant.role==='session')grant.parking=true
  148. try{
  149. const result=await this.hub.browser(r.owner,r.agent,{scope:r.scope,grantId:grant.id,op,...data},()=>this.check(grant))
  150. if(result&&typeof result.open==='boolean'&&Number.isInteger(result.generation)&&result.generation>=r.state.generation){r.state=result;if(!r.state.open)this.dropSession(r);else if(op==='open'&&(grant.role==='task'||grant.role==='human'))this.keepSession(r);r.notify?.()}
  151. return result
  152. }finally{grant.parking=false}
  153. }
  154. private async waitForControl(g:Grant){
  155. while(g.resource.paused){
  156. this.check(g)
  157. await new Promise<void>((resolve,reject)=>{
  158. const stop=()=>{clearTimeout(timer);reject(new HttpError(410,'浏览器任务已停止','browser_authorization_revoked'))}
  159. const timer=setTimeout(()=>{g.signal?.removeEventListener('abort',stop);resolve()},250)
  160. g.signal?.addEventListener('abort',stop,{once:true})
  161. if(g.signal?.aborted){g.signal.removeEventListener('abort',stop);stop()}
  162. })
  163. }
  164. this.check(g)
  165. }
  166. private async view<T>(r:Resource,action:(g:Grant)=>Promise<T>){
  167. const grant=this.grant(r,'view',()=>{},undefined,Date.now()+30000)
  168. try{return await action(grant)}finally{this.grants.delete(grant.id)}
  169. }
  170. private controlStatus(r:Resource):ComputerControlStatus{
  171. return {backend:'managed-browser',hostName:'托管浏览器',mode:r.taking?'pausing':r.paused?(r.human?'human':'pausing'):r.task?'agent':r.state.open?'idle':'off',generation:r.state.generation,...(r.human?{controlId:r.human.id}:{}),canResume:!!r.task}
  172. }
  173. async state(owner:string,id:string){
  174. const enabled=readHostTools(this.store.home).managedBrowser===true
  175. try{const r=this.resource(owner,id);await this.view(r,g=>this.request(g,'status'));this.finishPreparation(r);return this.publicState(r)}
  176. catch(error){return {enabled,available:false,reason:error instanceof Error?error.message:'浏览器不可用',open:false,generation:0,profile:'persistent',tabs:[],downloads:[]}}
  177. }
  178. private publicState(r:Resource):ManagedBrowserState{return {enabled:true,...r.state,supportsRetention:this.hub.supportsBrowserRetention(r.owner,r.agent),available:r.state.available??true,...(r.state.available===false?{reason:r.state.installation?.message??'浏览器环境尚未就绪'}:{})}}
  179. private finishPreparation(r:Resource){
  180. if(r.preparation&&['ready','failed'].includes(r.state.installation?.status??'')){this.grants.delete(r.preparation.id);r.preparation=undefined}
  181. }
  182. async prepare(owner:string,id:string,retry=false){
  183. const r=this.resource(owner,id)
  184. await this.view(r,g=>this.request(g,'status'))
  185. if(r.state.available!==false){this.finishPreparation(r);return this.publicState(r)}
  186. if(r.preparation){try{this.check(r.preparation)}catch{this.grants.delete(r.preparation.id);r.preparation=undefined}}
  187. const g=r.preparation??=this.grant(r,'prepare',()=>{},undefined,Date.now()+600000)
  188. try{await this.request(g,'prepare',{retry});this.finishPreparation(r);return this.publicState(r)}
  189. catch(error){if(r.preparation===g){this.grants.delete(g.id);r.preparation=undefined}throw error}
  190. }
  191. private async ensureReady(g:Grant){
  192. await this.request(g,'status')
  193. if(g.resource.state.available!==false)return
  194. await this.request(g,'prepare')
  195. const deadline=Date.now()+540000
  196. while(g.resource.state.available===false){
  197. this.check(g)
  198. const installation=g.resource.state.installation
  199. if(installation?.status==='failed')throw new HttpError(409,installation.error||installation.message,'browser_install_failed')
  200. if(Date.now()>=deadline)throw new HttpError(504,'浏览器环境准备超时,请在卡片中检查状态后重试','browser_install_timeout')
  201. await new Promise<void>((resolve,reject)=>{
  202. const stop=()=>{clearTimeout(timer);reject(new HttpError(410,'浏览器任务已停止','browser_authorization_revoked'))}
  203. const timer=setTimeout(()=>{g.signal?.removeEventListener('abort',stop);resolve()},750)
  204. g.signal?.addEventListener('abort',stop,{once:true})
  205. if(g.signal?.aborted){g.signal.removeEventListener('abort',stop);stop()}
  206. })
  207. await this.request(g,'status')
  208. }
  209. }
  210. private token(owner:string,id:string,value:unknown){
  211. const input=parse(credentials,value),grant=this.grants.get(input.controlId)
  212. if(!grant||grant.role!=='human'||grant.resource.owner!==owner||grant.resource.agent.id!==id||!grant.tokenHash||!timingSafeEqual(Buffer.from(grant.tokenHash,'hex'),Buffer.from(digest(input.token),'hex')))throw new HttpError(403,'需要当前浏览器控制凭据','browser_control_forbidden')
  213. this.check(grant);if(grant.resource.human!==grant)throw new HttpError(410,'浏览器控制权已改变','browser_control_expired')
  214. return grant
  215. }
  216. private async download(grant:Grant,id:string):Promise<BrowserUpload>{
  217. const parts:Buffer[]=[];let offset=0,size:number|undefined,sha256='',name='',mimeType='application/octet-stream'
  218. do{
  219. const reply=await this.request(grant,'download',{downloadId:id,offset})
  220. if(!Number.isSafeInteger(reply.size)||reply.size<0||reply.size>25*1024*1024||reply.offset!==offset||typeof reply.data!=='string'||reply.data.length>710000||!/^[a-f0-9]{64}$/.test(reply.sha256))throw new HttpError(502,'下载分块无效','browser_download_invalid')
  221. if(size!==undefined&&(size!==reply.size||sha256!==reply.sha256||name!==reply.name))throw new HttpError(502,'下载文件在传输时改变','browser_download_changed')
  222. size=reply.size;sha256=reply.sha256;name=reply.name;mimeType=reply.mimeType
  223. const chunk=Buffer.from(reply.data,'base64')
  224. if(chunk.toString('base64')!==reply.data||chunk.length!==Math.min(512*1024,size!-offset))throw new HttpError(502,'下载分块不完整','browser_download_invalid')
  225. parts.push(chunk);offset+=chunk.length
  226. }while(offset<size!)
  227. const buffer=Buffer.concat(parts)
  228. if(createHash('sha256').update(buffer).digest('hex')!==sha256)throw new HttpError(502,'下载校验失败','browser_download_invalid')
  229. return {name,mimeType,buffer}
  230. }
  231. openTurn(owner:string,agent:WorkspaceAgent,options:BrowserTurnOptions):BrowserTurn{
  232. let grant:Grant|undefined,finished=false,activated=false,closing:Promise<void>|undefined,failure:string|undefined,lastCard='',lastTitle:string|undefined,lastUrl:string|undefined,listener:(()=>void)|undefined
  233. const notify=(r:Resource)=>{
  234. if(!options.onCard)return
  235. const tab=r.state.tabs.find(tab=>tab.active)
  236. let url:string|undefined
  237. try{const parsed=new URL(tab?.url??'');if(['https:','http:'].includes(parsed.protocol)){parsed.username='';parsed.password='';parsed.search='';parsed.hash='';url=parsed.href.slice(0,2000)}}catch{}
  238. if(tab){lastTitle=tab.title.slice(0,240);lastUrl=url}
  239. if((r.retired||finished&&!r.human&&!r.session)&&r.notify===listener)r.notify=undefined
  240. const status:ManagedBrowserCard['status']=r.retired?'closed':failure?'failed':finished&&!r.human?(r.session&&r.state.open?'idle':'closed'):r.state.available===false?'preparing':r.state.open?'active':'ready'
  241. const {updatedAt:_updated,...installation}=r.state.installation??{updatedAt:0}
  242. const body={id:options.workId,agentId:agent.id,agentName:agent.name,status,title:lastTitle,url:lastUrl,
  243. message:r.retired?'浏览器执行来源已改变,本轮会话已结束':failure??(status==='idle'?'本轮操作已结束,页面已保留':status==='closed'?'本轮浏览器操作已结束':r.human?(finished?(r.session&&!r.forceClose?'人工控制中,交还后保留页面':'人工控制中,交还后结束本轮会话'):'人工控制中,可交还机器人继续'):r.paused?'接管已过期,请重新接管后交还机器人':r.state.installation?.message),installation:r.state.installation}
  244. const signature=JSON.stringify({...body,installation})
  245. if(signature===lastCard)return
  246. // Chat persistence is an observer; a deleted history row or storage error
  247. // must never block control handoff, authorization cleanup, or the sweep.
  248. try{if(options.onCard({...body,updatedAt:Date.now()})!==false)lastCard=signature}catch{}
  249. }
  250. const get=async()=>{
  251. options.authorize();options.signal.throwIfAborted()
  252. if(finished)throw new HttpError(410,'浏览器任务已结束','browser_authorization_revoked')
  253. if(!grant){
  254. const r=this.resource(owner,agent.id)
  255. while(r.transition){
  256. await r.transition.catch(()=>{});options.authorize();options.signal.throwIfAborted()
  257. if(finished)throw new HttpError(410,'浏览器任务已结束','browser_authorization_revoked')
  258. }
  259. this.checkResource(r)
  260. if(r.task)throw new HttpError(409,'当前 Bot 浏览器正由另一个任务使用','browser_busy')
  261. r.notify?.()
  262. if(!r.human)r.forceClose=false
  263. grant=this.grant(r,'task',options.authorize,options.signal);r.task=grant;listener=()=>notify(r);r.notify=listener
  264. }
  265. this.check(grant);return grant
  266. }
  267. const close=(value:{retain?:boolean}={}):Promise<void>=>{
  268. if(finished)return closing??Promise.resolve();finished=true;options.signal.removeEventListener('abort',abort)
  269. if(!grant)return Promise.resolve()
  270. const g=grant,r=g.resource;this.grants.delete(g.id);if(r.task===g)r.task=undefined
  271. const retain=value.retain!==false&&!options.signal.aborted
  272. if(!retain){r.forceClose=true;this.dropSession(r)}
  273. closing=this.transition(r,async()=>{
  274. try{
  275. this.checkResource(r)
  276. if(r.human){try{this.check(r.human);return}catch{this.grants.delete(r.human.id);r.human=undefined}}
  277. // A status-only turn must not refresh the retained session's idle deadline.
  278. if(retain&&!activated&&r.session&&r.parked)return
  279. await this.rest(r,retain)
  280. }catch{ /* Revoked resources are fenced and reclaimed by the Runner. */ }
  281. finally{notify(r)}
  282. }).finally(()=>{if(r.closing===closing)r.closing=undefined})
  283. r.closing=closing;return closing
  284. }
  285. const abort=()=>{void close({retain:false})};options.signal.addEventListener('abort',abort,{once:true})
  286. return {close,call:async(name,raw,callId)=>{
  287. const g=await get(),r=g.resource
  288. notify(r)
  289. try{
  290. // Hold the current tool call while the user operates this exact session.
  291. await this.waitForControl(g)
  292. if(name!=='managed_browser_state'){await this.activate(g);activated=true}
  293. const args=z.record(z.string(),z.unknown()).parse(raw??{})
  294. const result=await this.journal.execute(owner,options.workId,callId,name,args,()=>this.check(g),async()=>{
  295. if(name==='managed_browser_state')return this.request(g,'status')
  296. if(name==='managed_browser_open'){
  297. const body=z.object({url:z.string().optional()}).strict().parse(args)
  298. await this.ensureReady(g)
  299. await this.request(g,'open')
  300. if(body.url)await this.request(g,'execute',{operation:{generation:r.state.generation,operationId:callId??randomUUID(),action:{kind:'navigate',url:body.url}}})
  301. return this.request(g,'status')
  302. }
  303. if(name==='managed_browser_export'||name==='managed_browser_to_vm'){
  304. const body=z.object({downloadId:z.string().uuid(),path:z.string().optional(),overwrite:z.boolean().default(false)}).strict().parse(args)
  305. const file=await this.download(g,body.downloadId)
  306. if(name==='managed_browser_export')return options.publish(file.name,file.buffer)
  307. if(!options.toVm||!body.path)throw new HttpError(403,'本轮未开放虚拟机文件传输','browser_vm_unavailable')
  308. return options.toVm(file,body.path,body.overwrite)
  309. }
  310. let action:unknown,upload:Record<string,unknown>|undefined
  311. if(name==='managed_browser_upload'||name==='managed_browser_upload_vm'){
  312. const body=z.object({fileId:z.string().optional(),path:z.string().optional(),snapshotId:z.string().uuid(),ref:z.string()}).strict().parse(args)
  313. let file:BrowserUpload
  314. if(name==='managed_browser_upload'){if(!body.fileId||body.path)throw new HttpError(400,'请选择附件编号','browser_upload_invalid');file=await options.upload(body.fileId)}
  315. else{if(!body.path||body.fileId||!options.fromVm)throw new HttpError(403,'本轮未开放虚拟机文件传输','browser_vm_unavailable');file=await options.fromVm(body.path)}
  316. const id=randomUUID();upload={id,name:file.name,mimeType:file.mimeType,data:file.buffer.toString('base64')};action={kind:'upload',snapshotId:body.snapshotId,ref:body.ref,fileId:id}
  317. }else if(name==='managed_browser_snapshot')action={kind:'snapshot'}
  318. else if(name==='managed_browser_action'){
  319. action=z.object({action:z.record(z.string(),z.unknown())}).strict().parse(args).action
  320. if((action as {kind?:unknown}).kind==='pointer')throw new HttpError(400,'机器人拖动请使用 drag 动作','browser_input_unsupported')
  321. }
  322. else throw new HttpError(403,'本轮未授权此浏览器工具','browser_tool_forbidden')
  323. const value=await this.request(g,'execute',{operation:{generation:r.state.generation,operationId:callId??randomUUID(),action},...(upload?{upload}:{})})
  324. if(name==='managed_browser_action'&&!['snapshot','screenshot','downloads','state'].includes(String((action as {kind?:unknown})?.kind)))await this.request(g,'status')
  325. if(value?.mimeType==='image/png'&&typeof value.data==='string')return {content:[{type:'image',mimeType:'image/png',data:value.data},{type:'text',text:JSON.stringify({width:value.width,height:value.height,generation:value.generation})}]}
  326. return value
  327. })
  328. failure=undefined;notify(r)
  329. if(r.notes){const notes=r.notes;r.notes=undefined;return {result,humanNote:notes,instruction:'用户刚交还浏览器;重新读取页面后继续。'}}
  330. return result
  331. }catch(error){failure=error instanceof Error?error.message:'浏览器操作失败';notify(r);throw error}
  332. }}
  333. }
  334. router(){
  335. const router=new Router(),matches=(ctx:{query:Record<string,unknown>})=>ctx.query.backend==='managed-browser'
  336. router.get('/api/app/agents/:id/managed-browser',async ctx=>{const owner=this.auth.require(ctx).id;this.store.require(owner,'agent',ctx.params.id);ctx.set('Cache-Control','no-store');ctx.body=await this.state(owner,ctx.params.id)})
  337. router.post('/api/app/agents/:id/managed-browser/prepare',async ctx=>{const owner=this.auth.require(ctx).id,body=parse(z.object({retry:z.boolean().default(false)}).strict(),(ctx.request as any).body??{});ctx.body=await this.prepare(owner,ctx.params.id,body.retry)})
  338. router.get('/api/app/agents/:id/computer',async(ctx,next)=>{if(!matches(ctx))return next();const r=this.resource(this.auth.require(ctx).id,ctx.params.id);await this.view(r,g=>this.request(g,'status'));ctx.body=this.controlStatus(r)})
  339. router.get('/api/app/agents/:id/computer/frame',async(ctx,next)=>{if(!matches(ctx))return next();const r=this.resource(this.auth.require(ctx).id,ctx.params.id);const shot=await this.view(r,g=>this.request(g,'execute',{operation:{generation:r.state.generation,operationId:randomUUID(),action:{kind:'screenshot'}}}));r.frame={id:shot.frameId,generation:shot.generation,data:shot.data,width:shot.width,height:shot.height,capturedAt:shot.createdAt};r.frames??=new Map();const {data:_data,...metadata}=r.frame;r.frames.set(metadata.id,metadata);for(const [id,frame] of r.frames)if(frame.capturedAt<Date.now()-30000||r.frames.size>32)r.frames.delete(id);ctx.set('Cache-Control','no-store');ctx.body=r.frame})
  340. router.post('/api/app/agents/:id/computer/take',async(ctx,next)=>{
  341. if(!matches(ctx))return next()
  342. const r=this.resource(this.auth.require(ctx).id,ctx.params.id),body=parse(z.object({requestId:z.string().uuid()}).strict(),(ctx.request as any).body)
  343. ctx.body=await this.transition(r,async()=>{
  344. await this.view(r,g=>this.request(g,'status'))
  345. if(r.state.available===false)throw new HttpError(409,'浏览器环境尚未就绪,请先等待自动准备完成','browser_preparing')
  346. if(r.human&&r.human.expiresAt>Date.now()){
  347. this.check(r.human)
  348. if(r.human.requestId!==body.requestId)throw new HttpError(409,'浏览器已由另一页面接管','browser_control_busy')
  349. if(r.taking)await r.taking
  350. if(!r.human)throw new HttpError(409,'接管未完成,请重新接管','browser_control_expired')
  351. return {...this.controlStatus(r),controlId:r.human.id,token:this.nodes.open(r.human.sealedToken!),expiresAt:r.human.expiresAt}
  352. }
  353. if(r.human)this.grants.delete(r.human.id)
  354. const token=randomBytes(32).toString('base64url'),g=this.grant(r,'human',()=>{},undefined,Date.now()+30000)
  355. Object.assign(g,{requestId:body.requestId,tokenHash:digest(token),sealedToken:this.nodes.seal(token)});r.human=g;r.paused=true
  356. r.taking=(async()=>{await this.request(g,'pause');await this.request(g,'open');r.frame=undefined;r.frames?.clear();r.notify?.()})()
  357. try{await r.taking;r.parked=false;r.taking=undefined;return {...this.controlStatus(r),controlId:g.id,token,expiresAt:g.expiresAt}}
  358. catch(error){this.grants.delete(g.id);if(r.human===g)r.human=undefined;r.notify?.();throw error}
  359. finally{r.taking=undefined}
  360. })
  361. })
  362. router.post('/api/app/agents/:id/computer/renew',(ctx,next)=>{if(!matches(ctx))return next();const g=this.token(this.auth.require(ctx).id,ctx.params.id,(ctx.request as any).body);g.expiresAt=Date.now()+30000;ctx.body={expiresAt:g.expiresAt}})
  363. const input=async(owner:string,id:string,raw:unknown,navigation:boolean)=>{
  364. const body=parse(credentials.extend({requestId:z.string().uuid(),generation:z.number().int().positive(),frameId:z.string().uuid().optional(),action:z.record(z.string(),z.unknown())}).strict(),raw),g=this.token(owner,id,body),r=g.resource
  365. return this.journal.execute(owner,g.id,body.requestId,'managed_browser_human_input',{generation:body.generation,frameId:body.frameId,action:body.action,navigation},()=>{this.token(owner,id,body)},async()=>{
  366. if(r.taking||r.transition)throw new HttpError(409,'请等待当前操作结束后接管','browser_control_pending')
  367. if(body.generation!==r.state.generation)throw new HttpError(409,'控制代次已改变,请刷新','browser_generation_expired')
  368. let action=body.action
  369. if(navigation&&action.kind==='pointer')throw new HttpError(400,'连续拖动需要画面与人工控制凭据','browser_input_unsupported')
  370. if(!navigation){
  371. const frame=r.frames?.get(body.frameId??'')
  372. const cancelling=action.kind==='pointer'&&action.phase==='cancel'
  373. if(!cancelling&&(!frame||frame.generation!==body.generation||frame.capturedAt<Date.now()-30000))throw new HttpError(409,'画面已改变,请刷新后操作','browser_frame_stale')
  374. const a=body.action
  375. const x=z.number().finite().min(0).max(Math.min(1279,(frame?.width??1280)-1)),y=z.number().finite().min(0).max(Math.min(799,(frame?.height??800)-1))
  376. if(a.kind==='click')action={kind:'coordinate',x:a.x,y:a.y,...(a.button?{button:a.button}:{}),...(a.count?{clickCount:a.count}:{})}
  377. else if(a.kind==='drag')action=parse(z.object({kind:z.literal('drag'),fromX:x,fromY:y,toX:x,toY:y}).strict(),a)
  378. else if(a.kind==='pointer')action=parse(z.object({kind:z.literal('pointer'),gestureId:z.string().uuid(),phase:z.enum(['start','move','end','cancel']),x,y}).strict(),a)
  379. else if(a.kind==='key'){
  380. const names:Record<string,string>={Return:'Enter',BackSpace:'Backspace',Up:'ArrowUp',Down:'ArrowDown',Left:'ArrowLeft',Right:'ArrowRight',space:'Space'}
  381. const modifiers=z.array(z.enum(['ctrl','alt','shift','super'])).max(4).parse(a.modifiers??[]),key=z.string().max(40).parse(a.key)
  382. action={kind:'key',key:[...modifiers.map(m=>({ctrl:'Control',alt:'Alt',shift:'Shift',super:'Meta'})[m]),names[key]??key].join('+')}
  383. }else if(a.kind==='scroll'){
  384. const direction=z.enum(['up','down','left','right']).parse(a.direction),amount=z.number().int().min(1).max(50).parse(a.amount??3)*80
  385. action={kind:'scroll',deltaX:direction==='left'?-amount:direction==='right'?amount:0,deltaY:direction==='up'?-amount:direction==='down'?amount:0}
  386. }else if(a.kind!=='text')throw new HttpError(422,'托管浏览器暂不支持此输入','browser_input_unsupported')
  387. }
  388. const value=await this.request(g,'execute',{operation:{generation:body.generation,operationId:body.requestId,action}})
  389. // Recent frames remain usable between phases of the same held gesture.
  390. if(action.kind!=='pointer'||!['start','move'].includes(String(action.phase))){r.frame=undefined;r.frames?.clear()}
  391. return value
  392. })
  393. }
  394. router.post('/api/app/agents/:id/computer/input',async(ctx,next)=>{if(!matches(ctx))return next();ctx.body=await input(this.auth.require(ctx).id,ctx.params.id,(ctx.request as any).body,false)})
  395. router.post('/api/app/agents/:id/managed-browser/action',async ctx=>{ctx.body=await input(this.auth.require(ctx).id,ctx.params.id,(ctx.request as any).body,true)})
  396. router.post('/api/app/agents/:id/computer/giveback',async(ctx,next)=>{
  397. if(!matches(ctx))return next()
  398. const owner=this.auth.require(ctx).id,body=parse(credentials.extend({notes:z.string().max(4000).default('')}).strict(),(ctx.request as any).body),g=this.token(owner,ctx.params.id,body),r=g.resource
  399. ctx.body=await this.transition(r,async()=>{
  400. this.token(owner,ctx.params.id,body)
  401. // Keep the manual grant alive until the runner confirms the generation handoff.
  402. if(r.task)await this.request(r.task,'resume')
  403. else await this.rest(r,true)
  404. r.notes=body.notes;r.human=undefined;r.paused=false;r.frame=undefined;r.frames?.clear();this.grants.delete(g.id)
  405. r.notify?.()
  406. return this.controlStatus(r)
  407. })
  408. })
  409. router.post('/api/app/agents/:id/managed-browser/close',async ctx=>{
  410. const owner=this.auth.require(ctx).id,body=parse(credentials,(ctx.request as any).body),g=this.token(owner,ctx.params.id,body),r=g.resource
  411. ctx.body=await this.transition(r,async()=>{
  412. this.token(owner,ctx.params.id,body)
  413. if(r.task)throw new HttpError(409,'机器人任务仍在使用浏览器,请先停止任务','browser_busy')
  414. await this.rest(r,false)
  415. this.grants.delete(g.id);r.human=undefined;r.paused=false;r.notes=undefined;r.notify?.()
  416. return this.controlStatus(r)
  417. })
  418. })
  419. return router
  420. }
  421. private async sweep(){
  422. for(const r of this.resources.values()){
  423. try{this.checkResource(r)}catch{this.retire(r);continue}
  424. if(r.task){try{this.check(r.task)}catch{this.grants.delete(r.task.id);r.task=undefined;r.forceClose=true;this.dropSession(r)}}
  425. if(r.human){let valid=true;try{this.check(r.human)}catch{valid=false}if(!valid){
  426. this.grants.delete(r.human.id);r.human=undefined;r.paused=!!r.task;r.frame=undefined;r.frames?.clear()
  427. if(!r.task)await this.transition(r,async()=>{if(!r.task&&!r.human)await this.rest(r,true)}).catch(()=>{})
  428. }}
  429. if(r.preparation){try{this.check(r.preparation)}catch{this.grants.delete(r.preparation.id);r.preparation=undefined}}
  430. if(r.session&&!r.task&&!r.human&&!r.transition&&!r.refreshing&&Date.now()>=(r.nextStatusAt??0)){
  431. r.nextStatusAt=Date.now()+30000
  432. r.refreshing=this.transition(r,async()=>{if(!r.task&&!r.human&&r.session)await this.view(r,g=>this.request(g,'status'))})
  433. .then(()=>{},()=>this.retire(r)).finally(()=>{r.refreshing=undefined})
  434. }
  435. r.notify?.()
  436. }
  437. }
  438. close(){this.closed=true;clearInterval(this.timer);this.grants.clear();this.resources.clear()}
  439. }