|
|
@@ -24,8 +24,14 @@ const browserSchema=z.discriminatedUnion('kind',[
|
|
|
z.object({kind:z.enum(['back','forward','reload','snapshot'])}).strict(),
|
|
|
z.object({kind:z.enum(['click','fill']),ref:z.string().regex(/^e[0-9]+$/),snapshotId:z.string().uuid(),text:z.string().max(16000).optional()}).strict(),
|
|
|
])
|
|
|
+const fileSchema=z.discriminatedUnion('op',[
|
|
|
+ z.object({op:z.literal('list'),path:z.string().min(1).max(2048)}).strict(),
|
|
|
+ z.object({op:z.literal('read'),path:z.string().min(1).max(2048)}).strict(),
|
|
|
+ z.object({op:z.literal('write'),path:z.string().min(1).max(2048),data:z.string().max(14*1024*1024).refine(value=>Buffer.from(value,'base64').toString('base64')===value,'文件内容必须是 base64')}).strict(),
|
|
|
+])
|
|
|
+const shellSchema=z.object({command:z.string().min(1).max(8000),cwd:z.string().max(2048).optional(),timeoutMs:z.number().int().min(1000).max(120000).optional()}).strict()
|
|
|
type Mode='local'|'browser'
|
|
|
-interface NativeHost {id:string;name:string;platform:string;screen:boolean;accessibility:boolean;approved:string[]}
|
|
|
+interface NativeHost {id:string;name:string;platform:string;screen:boolean;accessibility:boolean;approved:string[];full?:string[]}
|
|
|
interface Pending {id:string;epoch:string;deadline:number;command:Record<string,unknown>;check:()=>void;resolve:(v:any)=>void;reject:(e:Error)=>void;sent:boolean;timer:ReturnType<typeof setTimeout>}
|
|
|
interface Manual {ready?:boolean;id:string;token:string;requestId:string;owner:string;agentId:string;version:number;expiresAt:number;generation:number;epoch:string;pending?:Promise<unknown>;releasing?:boolean;actions:Map<string,{fingerprint:string;value:Promise<unknown>}>}
|
|
|
/** All per-host state. The loopback App owns key 'local'; remote Macs each
|
|
|
@@ -49,9 +55,14 @@ const props={kind:{type:'string'},url:{type:'string'},tabId:{type:'string'},ref:
|
|
|
export const DESKTOP_ENVIRONMENT_TOOLS=[
|
|
|
{id:'desktop_environment_view',name:'desktop_environment_view',description:'查看当前已选电脑的真实截图,返回截图尺寸、环境类型和连接的电脑名称。',inputSchema:{type:'object',properties:{},additionalProperties:false}},
|
|
|
{id:'desktop_environment_action',name:'desktop_environment_action',description:'操作当前已选电脑。必须先查看新截图;kind 为 click、drag、text、key 或 scroll,坐标使用截图像素。',inputSchema:{type:'object',properties:{kind:{enum:['click','drag','text','key','scroll']},x:{type:'integer'},y:{type:'integer'},fromX:{type:'integer'},fromY:{type:'integer'},toX:{type:'integer'},toY:{type:'integer'},button:{enum:['left','right','middle']},count:{type:'integer'},text:{type:'string'},key:{type:'string'},modifiers:{type:'array',items:{enum:['ctrl','alt','shift','super']}},direction:{enum:['up','down','left','right']},amount:{type:'integer'}},required:['kind'],additionalProperties:false}},
|
|
|
+ {id:'desktop_file_list',name:'desktop_file_list',description:'列出当前电脑允许范围内的目录内容,返回名称、类型、大小与修改时间。路径相对于用户主目录,可用 ~。',inputSchema:{type:'object',properties:{path:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
|
+ {id:'desktop_file_read',name:'desktop_file_read',description:'读取当前电脑上的一个文件,返回 base64 内容与大小。路径相对于用户主目录;超出允许范围会被拒绝。',inputSchema:{type:'object',properties:{path:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
|
+ {id:'desktop_file_write',name:'desktop_file_write',description:'把 base64 内容写入当前电脑上的文件(覆盖已存在文件)。路径相对于用户主目录;超出允许范围会被拒绝。',inputSchema:{type:'object',properties:{path:{type:'string'},data:{type:'string'}},required:['path','data'],additionalProperties:false}},
|
|
|
+ {id:'desktop_shell',name:'desktop_shell',description:'在这台电脑的用户主目录执行一条 shell 命令,返回 stdout、stderr 与退出码。长任务会超时终止;不要用它代替文件工具读写大文件。',inputSchema:{type:'object',properties:{command:{type:'string'},cwd:{type:'string'},timeoutMs:{type:'integer'}},required:['command'],additionalProperties:false}},
|
|
|
{id:'desktop_browser',name:'desktop_browser',description:'操作机器人的独立浏览器。kind: navigate/new-tab(url)、select-tab/close-tab(tabId)、back/forward/reload、snapshot(读取页面元素与 ref)、click/fill(ref,snapshotId,text)。只在仅浏览器环境可用。',inputSchema:{type:'object',properties:props,required:['kind'],additionalProperties:false}},
|
|
|
]
|
|
|
-export const DESKTOP_ENVIRONMENT_RULES='当前环境由用户连接的夭夭桌面端提供。使用 desktop_environment_view/action 查看和操作该环境;本机模式会影响这台真实电脑,遵循用户授权范围。仅浏览器模式使用独立浏览器,desktop_browser 可导航、查看页面及操作元素。先查看再操作,页面变化后重新获取截图或 snapshot;不得猜测旧坐标或 ref。人工接管时等待交还,按交还说明继续。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
+export const DESKTOP_FILE_TOOL_IDS=new Set(['desktop_file_list','desktop_file_read','desktop_file_write','desktop_shell'])
|
|
|
+export const DESKTOP_ENVIRONMENT_RULES='当前环境由用户连接的夭夭桌面端提供。使用 desktop_environment_view/action 查看和操作该环境;本机模式会影响这台真实电脑,遵循用户授权范围。仅浏览器模式使用独立浏览器,desktop_browser 可导航、查看页面及操作元素。先查看再操作,页面变化后重新获取截图或 snapshot;不得猜测旧坐标或 ref。desktop_file_list/read/write 在用户主目录范围内读写文件,desktop_shell 在主目录执行命令;文件与命令是真实生效的操作,写文件和执行命令前确认目标路径,不要批量猜测写入。人工接管时等待交还,按交还说明继续。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
|
|
|
/** The loopback native transport is admitted only by serviceInstance's
|
|
|
* private capability; remote hosts are admitted by DesktopHostHub's bearer
|
|
|
@@ -67,6 +78,9 @@ export class DesktopEnvironments {
|
|
|
get idleForUpdate(){for(const s of this.sessions.values()){for(const key of s.manual.keys())this.current(s,key);if(s.pending.size||s.queue.size||s.manual.size)return false}return true}
|
|
|
private ownerKey(owner:string){return digest((this.nodes.localNodeID??this.nodes.local.url.toString())+':'+owner)}
|
|
|
private ready(s:HostState,owner:string){return this.hostOnline(s)&&s.info!.platform==='darwin'&&s.info!.screen&&s.info!.accessibility&&s.info!.approved.includes(this.ownerKey(owner))}
|
|
|
+ /** File and shell grant: a separate approval tier held by the desktop app. */
|
|
|
+ private readyFull(s:HostState,owner:string){return this.hostOnline(s)&&s.info!.platform==='darwin'&&(s.info!.full??[]).includes(this.ownerKey(owner))}
|
|
|
+ fileTools(owner:string,agent:WorkspaceAgent){const s=this.sessionFor(agent);return !!s&&this.readyFull(s,owner)}
|
|
|
private localSession(){let s=this.sessions.get('local');if(!s){s=newHostState('local',false);this.sessions.set('local',s)}return s}
|
|
|
/** Agent binding first; otherwise the loopback host, otherwise the single
|
|
|
* connected remote host. Ambiguous remote sets require an explicit binding. */
|
|
|
@@ -80,8 +94,8 @@ export class DesktopEnvironments {
|
|
|
toolMode(owner:string,agent:WorkspaceAgent):Mode|undefined {if(allowsHostEnvironment(agent)){const s=this.sessionFor(agent);if(s&&this.ready(s,owner))return 'local'}return this.selected(owner,agent)}
|
|
|
private agent(owner:string,id:string){const a=this.store.require<WorkspaceAgent>(owner,'agent',id);this.nodes.requireSource(owner,a);if(a.archived||a.remoteAgentId||a.temporaryGoalId)throw new HttpError(409,'此机器人无法使用桌面环境','computer_unavailable');return a}
|
|
|
private resource(owner:string,agent:WorkspaceAgent,mode:Mode){return mode==='local'?'local':digest(this.ownerKey(owner)+':'+agent.id+':'+(agent.browserProfile??'persistent'))}
|
|
|
- hostStates(owner:string){return [...this.sessions.values()].map(s=>{const online=this.hostOnline(s),info=online?s.info!:undefined;return {id:s.key,name:info?.name??'',platform:info?.platform??'',online,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s,owner)},browser:{available:online}}}).sort((a,b)=>a.id==='local'?-1:b.id==='local'?1:a.name.localeCompare(b.name))}
|
|
|
- state(owner:string,agent:WorkspaceAgent){const mode=this.selected(owner,agent),s=this.sessionFor(agent),online=!!s&&this.hostOnline(s),info=online?s!.info:undefined;return {online,host:info?{id:s!.key,name:info.name,platform:info.platform}:null,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s!,owner)},browser:{available:online,profile:agent.browserProfile??'persistent'},selected:mode??null,agent:this.store.agentSummary(agent),hosts:this.hostStates(owner)}}
|
|
|
+ hostStates(owner:string){return [...this.sessions.values()].map(s=>{const online=this.hostOnline(s),info=online?s.info!:undefined;return {id:s.key,name:info?.name??'',platform:info?.platform??'',online,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s,owner),fullAuthorized:!!info&&(info.full??[]).includes(this.ownerKey(owner))},browser:{available:online}}}).sort((a,b)=>a.id==='local'?-1:b.id==='local'?1:a.name.localeCompare(b.name))}
|
|
|
+ state(owner:string,agent:WorkspaceAgent){const mode=this.selected(owner,agent),s=this.sessionFor(agent),online=!!s&&this.hostOnline(s),info=online?s!.info:undefined;return {online,host:info?{id:s!.key,name:info.name,platform:info.platform}:null,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s!,owner),fullAuthorized:!!info&&(info.full??[]).includes(this.ownerKey(owner))},browser:{available:online,profile:agent.browserProfile??'persistent'},selected:mode??null,agent:this.store.agentSummary(agent),hosts:this.hostStates(owner)}}
|
|
|
private check(s:HostState|undefined,owner:string,agentId:string,mode:Mode,epoch:string,version:number){const a=this.agent(owner,agentId);if(!s||!this.hostOnline(s)||s.info!.id!==epoch)throw new HttpError(409,'桌面端已断开,请在电脑上打开夭夭后重试。','desktop_offline');if(this.auth.pushAuthorizationVersion(owner)!==version||this.toolMode(owner,a)!==mode)throw new HttpError(410,'电脑授权已改变,请重新连接。','computer_control_expired');if(mode==='local'&&!this.ready(s,owner))throw new HttpError(403,'请在桌面端授权本机控制,并开启屏幕录制和辅助功能。','desktop_permission_required')}
|
|
|
private invalidate(s:HostState){for(const p of s.pending.values()){if(p.sent&&['input','browser'].includes(String(p.command.operation)))s.paused.add(String(p.command.resource));clearTimeout(p.timer);p.reject(new HttpError(409,'桌面连接已改变;执行结果未确认,不会重试。','desktop_disconnected'))}s.pending.clear();s.frames.clear();for(const [key,m] of s.manual){s.paused.add(key);this.store.remove('_system','computer-control',m.id)}s.manual.clear()}
|
|
|
close(){this.closed=true;for(const s of this.sessions.values())this.invalidate(s)}
|
|
|
@@ -103,7 +117,7 @@ export class DesktopEnvironments {
|
|
|
/** Revocation or replacement of a paired host fences its live commands. */
|
|
|
dropHost(id:string){const s=this.sessions.get(id);if(s){this.invalidate(s);this.sessions.delete(id)}}
|
|
|
private exchangeFor(s:HostState,body:unknown){
|
|
|
- const value=parse(z.object({host:z.object({id:z.string().uuid(),name:z.string().max(128),platform:z.string().max(16),screen:z.boolean(),accessibility:z.boolean(),approved:z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(100)}).strict(),results:z.array(z.object({id:z.string().uuid(),value:z.unknown().optional(),error:z.string().max(500).optional()}).strict()).max(20)}).strict(),body)
|
|
|
+ const value=parse(z.object({host:z.object({id:z.string().uuid(),name:z.string().max(128),platform:z.string().max(16),screen:z.boolean(),accessibility:z.boolean(),approved:z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(100),full:z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(100).optional()}).strict(),results:z.array(z.object({id:z.string().uuid(),value:z.unknown().optional(),error:z.string().max(500).optional()}).strict()).max(20)}).strict(),body)
|
|
|
if(this.closed)throw new HttpError(503,'服务已停止','desktop_closed')
|
|
|
if(s.info&&(s.info.id!==value.host.id||!this.hostOnline(s)))this.invalidate(s)
|
|
|
s.info=value.host;s.seen=Date.now()
|
|
|
@@ -136,6 +150,11 @@ export class DesktopEnvironments {
|
|
|
let result:any
|
|
|
if(id==='desktop_environment_view'){if(expected==='browser')await this.command(s!,{...c.command,operation:'open'},check);const frame=await this.capture(s!,{...c,check});this.toolFrames.set(signal,frame.id);result={content:[{type:'image',mimeType:'image/png',data:frame.data},{type:'text',text:JSON.stringify({width:frame.width,height:frame.height,host:s!.info!.name,mode:expected})}]}}
|
|
|
else if(id==='desktop_browser'&&expected==='browser'){result=await this.command(s!,{...c.command,operation:'browser',action:parse(browserSchema,input)},check);s!.frames.delete(c.key)}
|
|
|
+ else if(DESKTOP_FILE_TOOL_IDS.has(id)&&expected==='local'){
|
|
|
+ if(!this.readyFull(s!,owner))throw new HttpError(403,'请在桌面端授权文件与命令访问后重试。','desktop_full_required')
|
|
|
+ const action=id==='desktop_shell'?parse(shellSchema,input):parse(fileSchema,{op:id==='desktop_file_list'?'list':id==='desktop_file_read'?'read':'write',...(typeof input==='object'&&input?input:{})})
|
|
|
+ result=await this.command(s!,{...c.command,operation:id==='desktop_shell'?'shell':'file',action},check,id==='desktop_shell'?150000:60000)
|
|
|
+ }
|
|
|
else if(id==='desktop_environment_action'){const frame=s!.frames.get(c.key);if(!frame||this.toolFrames.get(signal)!==frame.id||Date.now()-frame.capturedAt>15000)throw new HttpError(409,'请先查看最新电脑画面','computer_frame_stale');result=await this.command(s!,{...c.command,operation:'input',action:parse(inputSchema,input),frame:{width:frame.width,height:frame.height}},check);s!.frames.delete(c.key)}
|
|
|
else throw new HttpError(404,'当前环境不支持这个工具','tool_not_found')
|
|
|
const note=s!.notes.get(owner+':'+agentId);if(note){s!.notes.delete(owner+':'+agentId);if(result.content)result.content.push({type:'text',text:'人工交还说明:'+note});else result={...result,handoffNote:note}}return result
|
|
|
@@ -149,12 +168,12 @@ export class DesktopEnvironments {
|
|
|
router.get('/api/app/agents/:id/desktop-environment',ctx=>{const owner=this.auth.require(ctx).id,a=this.store.require<WorkspaceAgent>(owner,'agent',ctx.params.id);this.nodes.requireSource(owner,a);ctx.set('Cache-Control','no-store');ctx.body=this.state(owner,a)})
|
|
|
router.post('/api/app/agents/:id/desktop-environment/authorize',async ctx=>{
|
|
|
const user=this.auth.require(ctx);this.auth.requireAdmin(ctx);const a=this.agent(user.id,ctx.params.id)
|
|
|
- const body=parse(z.object({hostId:z.union([z.literal('local'),z.string().uuid()]).optional()}).strict(),(ctx.request as any).body??{})
|
|
|
+ const body=parse(z.object({hostId:z.union([z.literal('local'),z.string().uuid()]).optional(),scope:z.enum(['screen','full']).default('screen')}).strict(),(ctx.request as any).body??{})
|
|
|
const s=body.hostId!==undefined?this.sessions.get(body.hostId):this.sessionFor(a)
|
|
|
if(!s||!this.hostOnline(s))throw new HttpError(409,'桌面端未连接,请在电脑上打开夭夭。','desktop_offline')
|
|
|
const owner=this.ownerKey(user.id),epoch=s.info!.id,authorizeKey=user.id+':'+s.key
|
|
|
let pending=this.authorizing.get(authorizeKey)
|
|
|
- if(!pending){pending=this.command(s,{operation:'authorize',owner,account:user.username},()=>{this.auth.requireAdmin(ctx);if(!this.hostOnline(s)||epoch!==s.info!.id)throw new HttpError(409,'桌面端已断开','desktop_offline')},120000).finally(()=>this.authorizing.delete(authorizeKey));this.authorizing.set(authorizeKey,pending)}
|
|
|
+ if(!pending){pending=this.command(s,{operation:'authorize',owner,account:user.username,scope:body.scope},()=>{this.auth.requireAdmin(ctx);if(!this.hostOnline(s)||epoch!==s.info!.id)throw new HttpError(409,'桌面端已断开','desktop_offline')},120000).finally(()=>this.authorizing.delete(authorizeKey));this.authorizing.set(authorizeKey,pending)}
|
|
|
ctx.body=await pending})
|
|
|
router.put('/api/app/agents/:id/browser-profile',ctx=>{const owner=this.auth.require(ctx).id,a=this.agent(owner,ctx.params.id),body=parse(z.object({profile:z.enum(['persistent','temporary'])}).strict(),(ctx.request as any).body);this.assertIdle(owner,a.id);this.store.updateAgent(owner,a.id,{browserProfile:body.profile});ctx.body={ok:true}})
|
|
|
router.get('/api/app/agents/:id/browser',async ctx=>{const owner=this.auth.require(ctx).id,a=this.agent(owner,ctx.params.id);if(this.selected(owner,a)!=='browser')throw new HttpError(409,'请先选择仅浏览器环境','desktop_unselected');const s=this.sessionFor(a);ctx.body=await this.command(s!,{...this.context(s,owner,a,'browser').command,operation:'browser-state'},this.context(s,owner,a,'browser').check)})
|