|
|
@@ -23,7 +23,7 @@ import {homedir,platform} from 'node:os'
|
|
|
|
|
|
export interface ComputerTarget {environmentId:string;ownerKey:string;agentId:string;hostAccess?:boolean;profileSession?:boolean;hermesRuntime?:boolean}
|
|
|
export interface WorkerSession {agentId?:string;id:string;profile:string;ownerKey:string;environmentId:string;cwd:string;configuredCwd:string;history:any[];outcome?:'complete'|'failed'|'uncertain';vmExecution?:'profile';profileSessionId?:string;hermesRuntime?:boolean}
|
|
|
-interface Live {workMarker?:string;paused?:boolean;pauseJob?:Promise<void>;segment?:number;toolCalls:Set<Promise<unknown>>;journal:Map<string,{name:string;result:unknown}>;proxyHandle?:ProxyHandle;session:WorkerSession;model:WorkerModel;contextConfig?:WorkerContextConfiguration;proxyEnv?:WorkerProxyEnvironment;lease:ComputerLease;controller:AbortController;worker?:HermesWorkerProcess;finishing?:Promise<void>;stopping?:Promise<void>;releasing?:Promise<void>;images:any[];running:boolean;received:boolean}
|
|
|
+interface Live {workMarker?:string;paused?:boolean;desktop?:{width:number;height:number};pauseJob?:Promise<void>;segment?:number;toolCalls:Set<Promise<unknown>>;journal:Map<string,{name:string;result:unknown}>;proxyHandle?:ProxyHandle;session:WorkerSession;model:WorkerModel;contextConfig?:WorkerContextConfiguration;proxyEnv?:WorkerProxyEnvironment;lease:ComputerLease;controller:AbortController;worker?:HermesWorkerProcess;finishing?:Promise<void>;stopping?:Promise<void>;releasing?:Promise<void>;images:any[];running:boolean;received:boolean}
|
|
|
const object=(properties:Record<string,unknown>,required:string[])=>({type:'object',properties,required,additionalProperties:false})
|
|
|
const text={type:'string'}
|
|
|
const TOOLS:WorkerTool[]=[
|
|
|
@@ -31,13 +31,20 @@ const TOOLS:WorkerTool[]=[
|
|
|
{name:'computer_shell',description:'在当前隔离电脑和 Profile 工作目录执行 shell 命令。默认使用 cua 用户；只有安装系统软件或修改系统路径时才选择 root。',inputSchema:object({command:text,user:{type:'string',enum:['cua','root'],default:'cua'}},['command'])},
|
|
|
{name:'computer_read_file',description:'读取隔离电脑内的 UTF-8 文本文件。',inputSchema:object({path:text},['path'])},
|
|
|
{name:'computer_write_file',description:'向隔离电脑写入 UTF-8 文本文件。',inputSchema:object({path:text,content:text},['path','content'])},
|
|
|
- {name:'computer_desktop_state',description:'查看隔离电脑当前桌面，返回真实截图。',inputSchema:object({},[])},
|
|
|
- {name:'computer_action',description:'在隔离电脑内调用 CUA 动作；先查看桌面并使用可用动作。',inputSchema:object({name:text,arguments:{type:'object'}},['name','arguments'])},
|
|
|
+ {name:'computer_desktop_state',description:'查看虚拟环境的真实桌面，返回截图和宽高。操作前必须先看。',inputSchema:object({},[])},
|
|
|
+ {name:'computer_action',description:'操作虚拟环境桌面。必须先查看桌面。kind 为 click、text、key 或 scroll，坐标用截图像素。',inputSchema:object({kind:{enum:['click','drag','text','key','scroll']},x:{type:'integer'},y:{type:'integer'},fromX:{type:'integer'},fromY:{type:'integer'},toX:{type:'integer'},toY:{type:'integer'},button:{enum:['left','right','middle']},count:{type:'integer'},text:text,key:text,modifiers:{type:'array',items:{enum:['ctrl','alt','shift','super']}},direction:{enum:['up','down','left','right']},amount:{type:'integer'}},['kind'])},
|
|
|
]
|
|
|
const pathField=z.string().min(1).max(4096).refine(value=>!value.includes('\0'))
|
|
|
+const desktopAction=z.discriminatedUnion('kind',[
|
|
|
+ z.object({kind:z.literal('click'),x:z.number().int().nonnegative(),y:z.number().int().nonnegative(),button:z.enum(['left','right','middle']).default('left'),count:z.number().int().min(1).max(2).default(1)}).strict(),
|
|
|
+ z.object({kind:z.literal('drag'),fromX:z.number().int().nonnegative(),fromY:z.number().int().nonnegative(),toX:z.number().int().nonnegative(),toY:z.number().int().nonnegative()}).strict(),
|
|
|
+ z.object({kind:z.literal('text'),text:z.string().max(16000)}).strict(),
|
|
|
+ z.object({kind:z.literal('key'),key:z.string().min(1).max(40).regex(/^[A-Za-z0-9_+ -]+$/),modifiers:z.array(z.enum(['ctrl','alt','shift','super'])).max(4).default([])}).strict(),
|
|
|
+ z.object({kind:z.literal('scroll'),direction:z.enum(['up','down','left','right']),amount:z.number().int().min(1).max(50).default(3)}).strict(),
|
|
|
+])
|
|
|
const PROFILE_COMPUTER_RULES='当前使用本机协作模式：沿用基础 Profile 的本机工具、配置和上下文，本地虚拟机是独立的 Linux 电脑。computer_shell/read_file/write_file/desktop_state/action/export 只操作虚拟机；本机文件、终端和程序使用 Profile 已授权的原生工具。computer_copy_file 在两个环境间显式复制文件。调用前明确目标环境，文件路径和浏览器登录互不通用；需要回传虚拟机文件时使用 computer_export。人工接管期间停止操作，交还后先检查当前状态。'
|
|
|
const HERMES_COMPUTER_RULES='本轮由 Hermes 管理模型、配置、识图、技能和授权资源。使用 Hermes 原生 skills_list、skill_view 和 skill_manage 发现、读取和维护技能；遵循当前 Profile 的技能规则。技能依赖本机登录态、环境变量或授权文件时，使用 Hermes 原生工具完成相应步骤；适合 Linux 的脚本与输入文件可经 computer_copy_file 传入虚拟机，再用 computer_shell 执行。路径、软件和登录态在本机与虚拟机之间不通用。不要自行复制 Profile 配置或凭据到虚拟机。'
|
|
|
-const ISOLATED_COMPUTER_RULES='当前环境是虚拟环境，也就是服务端上的隔离虚拟机。命令、文件、浏览器和桌面操作都通过 computer_* 在这台 Linux 电脑内部完成。不要使用服务端本机的终端、文件或桌面，也不要在节点离线时改去服务端或脚本机。脚本机和云虚拟机是另外勾选的环境，不从这台虚拟机里打开。人工接管期间停止操作，交还后先检查当前状态。'
|
|
|
+const ISOLATED_COMPUTER_RULES='当前环境是虚拟环境，也就是服务端上的隔离虚拟机。先看 computer_desktop_state，再按截图像素用 computer_action 点击、输入、按键或滚动。命令和文件也通过 computer_* 在这台 Linux 电脑内部完成。不要使用服务端本机的终端、文件或桌面，也不要在节点离线时改去服务端或脚本机。脚本机和云虚拟机是另外勾选的环境，不从这台虚拟机里打开。人工接管期间停止操作，交还后先检查当前状态。'
|
|
|
const ISOLATED_HERMES_RULES='本轮由 Hermes 管理模型、配置和这次会话。使用 Hermes 原生 skills_list、skill_view 和 skill_manage 发现、读取和维护技能。技能和 computer_* 都留在这台虚拟机里；不要把 Profile 的本机登录、文件或凭据复制进来。'
|
|
|
export interface ProxyHandle {readonly proxy:ComputerPublicProxy;release():Promise<void>}
|
|
|
export class ComputerRuntime {
|
|
|
@@ -418,15 +425,36 @@ export class ComputerGateway {
|
|
|
await execute(['python3','-c','import pathlib,sys; p=pathlib.Path(sys.argv[1]); p.parent.mkdir(parents=True,exist_ok=True); p.write_bytes(sys.stdin.buffer.read())',body.path],Buffer.from(body.content));return {ok:true}
|
|
|
}
|
|
|
if(name==='computer_action'){
|
|
|
- const body=z.object({name:z.string().regex(/^[a-zA-Z][a-zA-Z0-9_]{0,99}$/),arguments:z.record(z.string(),z.unknown())}).strict().parse(args)
|
|
|
- return execute([CUA_DRIVER,'call',body.name,JSON.stringify(body.arguments),'--socket',CUA_SOCKET],undefined,'gui')
|
|
|
+ if(!live.desktop)throw new HttpError(409,'请先查看虚拟环境桌面','computer_frame_stale')
|
|
|
+ const frame=live.desktop,action=desktopAction.parse(args),common={scope:'desktop',session:`yaoyao-vm-${live.session.id}`}
|
|
|
+ let tool:string,toolArgs:Record<string,unknown>,argv:string[]|undefined
|
|
|
+ if(action.kind==='click'){
|
|
|
+ if(action.x>=frame.width||action.y>=frame.height)throw new HttpError(400,'点击位置超出画面','computer_point_invalid')
|
|
|
+ tool='click';toolArgs={...common,x:action.x,y:action.y,button:action.button,count:action.count}
|
|
|
+ }else if(action.kind==='drag'){
|
|
|
+ if(action.fromX>=frame.width||action.toX>=frame.width||action.fromY>=frame.height||action.toY>=frame.height)throw new HttpError(400,'拖动位置超出画面','computer_point_invalid')
|
|
|
+ tool='drag';toolArgs={}
|
|
|
+ argv=['env','DISPLAY=:1','xdotool','mousemove','--sync',String(action.fromX),String(action.fromY),'mousedown','1','mousemove','--sync',String(action.toX),String(action.toY),'mouseup','1']
|
|
|
+ }else if(action.kind==='text'){tool='type_text';toolArgs={...common,text:action.text}}
|
|
|
+ else if(action.kind==='key'){
|
|
|
+ const key=({Return:'enter',BackSpace:'backspace',Escape:'esc'} as Record<string,string>)[action.key]??action.key.toLowerCase()
|
|
|
+ tool=action.modifiers.length?'hotkey':'press_key';toolArgs=action.modifiers.length?{...common,keys:[...action.modifiers,key]}:{...common,key}
|
|
|
+ }else{tool='scroll';toolArgs={...common,x:Math.floor(frame.width/2),y:Math.floor(frame.height/2),by:'line',direction:action.direction,amount:action.amount}}
|
|
|
+ live.desktop=undefined
|
|
|
+ const response=await execute(argv??[CUA_DRIVER,'call',tool!,JSON.stringify(toolArgs!),'--socket',CUA_SOCKET],undefined,'gui')
|
|
|
+ let parsed:any;try{parsed=JSON.parse(response.stdout)}catch{}
|
|
|
+ if(parsed?.code||parsed?.error||parsed?.isError)throw new HttpError(502,'电脑未接受这次输入，请刷新画面后重试','computer_action_failed')
|
|
|
+ return {ok:true}
|
|
|
}
|
|
|
if(name==='computer_desktop_state'){
|
|
|
z.object({}).strict().parse(args)
|
|
|
const path=`/tmp/yaoyao-frame-${randomUUID()}.png`
|
|
|
const state=await execute([CUA_DRIVER,'call','get_desktop_state','{}','--socket',CUA_SOCKET,'--screenshot-out-file',path],undefined,'gui')
|
|
|
const data=(await execute(['base64','-w0',path])).stdout.trim()
|
|
|
- return {_multimodal:true,text_summary:state.stdout,content:[{type:'image_url',image_url:{url:`data:image/png;base64,${data}`}}]}
|
|
|
+ const header=Buffer.from(data.slice(0,32),'base64'),width=header.length>=24?header.readUInt32BE(16):0,height=header.length>=24?header.readUInt32BE(20):0
|
|
|
+ if(width<1||height<1)throw new HttpError(502,'虚拟环境画面无效','computer_frame_failed')
|
|
|
+ live.desktop={width,height}
|
|
|
+ return {_multimodal:true,width,height,text_summary:state.stdout,content:[{type:'image_url',image_url:{url:`data:image/png;base64,${data}`}}]}
|
|
|
}
|
|
|
throw new HttpError(403,'本轮未授权该工具','computer_tool_forbidden')
|
|
|
})
|
