|
@@ -23,7 +23,7 @@ import {homedir,platform} from 'node:os'
|
|
|
|
|
|
|
|
export interface ComputerTarget {environmentId:string;ownerKey:string;agentId:string;hostAccess?:boolean;profileSession?:boolean;hermesRuntime?:boolean}
|
|
export interface ComputerTarget {environmentId:string;ownerKey:string;agentId:string;hostAccess?:boolean;profileSession?:boolean;hermesRuntime?:boolean}
|
|
|
export interface WorkerSession {agentId?:string;id:string;profile:string;ownerKey:string;environmentId:string;cwd:string;configuredCwd:string;history:any[];outcome?:'complete'|'failed'|'uncertain';vmExecution?:'profile';profileSessionId?:string;hermesRuntime?:boolean}
|
|
export interface WorkerSession {agentId?:string;id:string;profile:string;ownerKey:string;environmentId:string;cwd:string;configuredCwd:string;history:any[];outcome?:'complete'|'failed'|'uncertain';vmExecution?:'profile';profileSessionId?:string;hermesRuntime?:boolean}
|
|
|
-interface Live {workMarker?:string;paused?:boolean;pauseJob?:Promise<ComputerLease>;segment?:number;toolCalls:Set<Promise<unknown>>;journal:Map<string,{name:string;result:unknown}>;proxy?:ComputerPublicProxy;session:WorkerSession;model:WorkerModel;contextConfig?:WorkerContextConfiguration;proxyEnv?:WorkerProxyEnvironment;lease:ComputerLease;controller:AbortController;worker?:HermesWorkerProcess;finishing?:Promise<void>;stopping?:Promise<void>;releasing?:Promise<void>;images:any[];running:boolean;received:boolean}
|
|
|
|
|
|
|
+interface Live {workMarker?:string;paused?:boolean;pauseJob?:Promise<void>;segment?:number;toolCalls:Set<Promise<unknown>>;journal:Map<string,{name:string;result:unknown}>;proxyHandle?:ProxyHandle;session:WorkerSession;model:WorkerModel;contextConfig?:WorkerContextConfiguration;proxyEnv?:WorkerProxyEnvironment;lease:ComputerLease;controller:AbortController;worker?:HermesWorkerProcess;finishing?:Promise<void>;stopping?:Promise<void>;releasing?:Promise<void>;images:any[];running:boolean;received:boolean}
|
|
|
const object=(properties:Record<string,unknown>,required:string[])=>({type:'object',properties,required,additionalProperties:false})
|
|
const object=(properties:Record<string,unknown>,required:string[])=>({type:'object',properties,required,additionalProperties:false})
|
|
|
const text={type:'string'}
|
|
const text={type:'string'}
|
|
|
const TOOLS:WorkerTool[]=[
|
|
const TOOLS:WorkerTool[]=[
|
|
@@ -38,6 +38,7 @@ const pathField=z.string().min(1).max(4096).refine(value=>!value.includes('\0'))
|
|
|
const PROFILE_COMPUTER_RULES='当前使用本机协作模式:沿用基础 Profile 的本机工具、配置和上下文,本地虚拟机是独立的 Linux 电脑。computer_shell/read_file/write_file/desktop_state/action/export 只操作虚拟机;本机文件、终端和程序使用 Profile 已授权的原生工具。computer_copy_file 在两个环境间显式复制文件。调用前明确目标环境,文件路径和浏览器登录互不通用;需要回传虚拟机文件时使用 computer_export。人工接管期间停止操作,交还后先检查当前状态。'
|
|
const PROFILE_COMPUTER_RULES='当前使用本机协作模式:沿用基础 Profile 的本机工具、配置和上下文,本地虚拟机是独立的 Linux 电脑。computer_shell/read_file/write_file/desktop_state/action/export 只操作虚拟机;本机文件、终端和程序使用 Profile 已授权的原生工具。computer_copy_file 在两个环境间显式复制文件。调用前明确目标环境,文件路径和浏览器登录互不通用;需要回传虚拟机文件时使用 computer_export。人工接管期间停止操作,交还后先检查当前状态。'
|
|
|
const HERMES_COMPUTER_RULES='本轮由 Hermes 管理模型、配置、识图、技能和授权资源。使用 Hermes 原生 skills_list、skill_view 和 skill_manage 发现、读取和维护技能;遵循当前 Profile 的技能规则。技能依赖本机登录态、环境变量或授权文件时,使用 Hermes 原生工具完成相应步骤;适合 Linux 的脚本与输入文件可经 computer_copy_file 传入虚拟机,再用 computer_shell 执行。路径、软件和登录态在本机与虚拟机之间不通用。不要自行复制 Profile 配置或凭据到虚拟机。'
|
|
const HERMES_COMPUTER_RULES='本轮由 Hermes 管理模型、配置、识图、技能和授权资源。使用 Hermes 原生 skills_list、skill_view 和 skill_manage 发现、读取和维护技能;遵循当前 Profile 的技能规则。技能依赖本机登录态、环境变量或授权文件时,使用 Hermes 原生工具完成相应步骤;适合 Linux 的脚本与输入文件可经 computer_copy_file 传入虚拟机,再用 computer_shell 执行。路径、软件和登录态在本机与虚拟机之间不通用。不要自行复制 Profile 配置或凭据到虚拟机。'
|
|
|
const ISOLATED_COMPUTER_RULES='当前使用隔离虚拟机:命令、文件处理和浏览器操作默认通过 computer_* 工具在指定的 Linux 电脑执行。需要使用本机文件或运行本机技能时,遵循 Hermes 返回的授权请求;拒绝后停止该操作。人工接管期间停止操作,交还后先检查当前状态。'
|
|
const ISOLATED_COMPUTER_RULES='当前使用隔离虚拟机:命令、文件处理和浏览器操作默认通过 computer_* 工具在指定的 Linux 电脑执行。需要使用本机文件或运行本机技能时,遵循 Hermes 返回的授权请求;拒绝后停止该操作。人工接管期间停止操作,交还后先检查当前状态。'
|
|
|
|
|
+export interface ProxyHandle {readonly proxy:ComputerPublicProxy;release():Promise<void>}
|
|
|
export class ComputerRuntime {
|
|
export class ComputerRuntime {
|
|
|
retainDesktops=false
|
|
retainDesktops=false
|
|
|
readonly controls:ComputerControls
|
|
readonly controls:ComputerControls
|
|
@@ -45,10 +46,11 @@ export class ComputerRuntime {
|
|
|
readonly pool:ComputerPool
|
|
readonly pool:ComputerPool
|
|
|
readonly ready:Promise<void>
|
|
readonly ready:Promise<void>
|
|
|
readonly config:NonNullable<RunnerConfiguration['computers']>
|
|
readonly config:NonNullable<RunnerConfiguration['computers']>
|
|
|
- readonly gateways=new Map<string,ComputerGateway>()
|
|
|
|
|
|
|
+ readonly gateways=new Map<string,Set<ComputerGateway>>()
|
|
|
readonly workers=new Map<string,{environmentId:string;ownerKey:string;process:HermesWorkerProcess}>()
|
|
readonly workers=new Map<string,{environmentId:string;ownerKey:string;process:HermesWorkerProcess}>()
|
|
|
readonly script:string
|
|
readonly script:string
|
|
|
readonly proxyScript:string
|
|
readonly proxyScript:string
|
|
|
|
|
+ private proxyShares=new Map<string,{started:Promise<ComputerPublicProxy>;users:number}>()
|
|
|
constructor(readonly db:DatabaseSync,config:RunnerConfiguration,readonly home:string,script?:string,relay?:DesktopRelay,readonly hermesTarget?:GatewayTarget){
|
|
constructor(readonly db:DatabaseSync,config:RunnerConfiguration,readonly home:string,script?:string,relay?:DesktopRelay,readonly hermesTarget?:GatewayTarget){
|
|
|
if(!config.computers)throw new HttpError(409,'执行节点尚未配置隔离电脑','computer_unavailable')
|
|
if(!config.computers)throw new HttpError(409,'执行节点尚未配置隔离电脑','computer_unavailable')
|
|
|
this.config=config.computers
|
|
this.config=config.computers
|
|
@@ -66,6 +68,30 @@ export class ComputerRuntime {
|
|
|
this.controls=new ComputerControls(this)
|
|
this.controls=new ComputerControls(this)
|
|
|
this.ready=this.pool.recover();void this.ready.catch(()=>{})
|
|
this.ready=this.pool.recover();void this.ready.catch(()=>{})
|
|
|
}
|
|
}
|
|
|
|
|
+ attachGateway(gateway:ComputerGateway){let gateways=this.gateways.get(gateway.meta.environmentId);if(!gateways){gateways=new Set();this.gateways.set(gateway.meta.environmentId,gateways)}gateways.add(gateway)}
|
|
|
|
|
+ detachGateway(gateway:ComputerGateway){const gateways=this.gateways.get(gateway.meta.environmentId);if(gateways){gateways.delete(gateway);if(!gateways.size)this.gateways.delete(gateway.meta.environmentId)}}
|
|
|
|
|
+ activeGateways(environmentId:string):ComputerGateway[]{return [...(this.gateways.get(environmentId)??[])].filter(gateway=>gateway.active||gateway.paused)}
|
|
|
|
|
+ /** One guest proxy per environment, shared by every holder; the guest
|
|
|
|
|
+ * reserves port 3128, so concurrent holders must never start their own. */
|
|
|
|
|
+ acquireProxy(spec:ComputerSpecification,authorize:()=>void):Promise<ProxyHandle>{
|
|
|
|
|
+ let share=this.proxyShares.get(spec.id)
|
|
|
|
|
+ if(!share){
|
|
|
|
|
+ const started=ComputerPublicProxy.start(this.provider,spec,this.proxyScript,authorize,async()=>authorize(),()=>{if(this.proxyShares.get(spec.id)===share)this.proxyShares.delete(spec.id)})
|
|
|
|
|
+ share={started,users:0};this.proxyShares.set(spec.id,share)
|
|
|
|
|
+ void started.catch(()=>{if(this.proxyShares.get(spec.id)===share)this.proxyShares.delete(spec.id)})
|
|
|
|
|
+ }
|
|
|
|
|
+ return share.started.then(proxy=>{
|
|
|
|
|
+ share!.users++
|
|
|
|
|
+ let released=false
|
|
|
|
|
+ return {proxy,release:async()=>{
|
|
|
|
|
+ if(released)return;released=true
|
|
|
|
|
+ const current=this.proxyShares.get(spec.id)
|
|
|
|
|
+ if(current===share&&--share!.users>0)return
|
|
|
|
|
+ if(current===share)this.proxyShares.delete(spec.id)
|
|
|
|
|
+ await proxy.close().catch(()=>{})
|
|
|
|
|
+ }}
|
|
|
|
|
+ })
|
|
|
|
|
+ }
|
|
|
assertTarget(meta:ComputerTarget){if(this.db.prepare('SELECT id FROM retired_computers WHERE id=?').get(meta.environmentId))throw new HttpError(410,'临时电脑已退役','computer_retired')}
|
|
assertTarget(meta:ComputerTarget){if(this.db.prepare('SELECT id FROM retired_computers WHERE id=?').get(meta.environmentId))throw new HttpError(410,'临时电脑已退役','computer_retired')}
|
|
|
imageFor(meta:ComputerTarget):string {
|
|
imageFor(meta:ComputerTarget):string {
|
|
|
const spec=this.pool.definition(meta.ownerKey,meta.environmentId)
|
|
const spec=this.pool.definition(meta.ownerKey,meta.environmentId)
|
|
@@ -80,7 +106,7 @@ export class ComputerRuntime {
|
|
|
this.db.prepare('INSERT OR IGNORE INTO retired_computers VALUES(?,?,NULL)').run(meta.environmentId,meta.ownerKey)
|
|
this.db.prepare('INSERT OR IGNORE INTO retired_computers VALUES(?,?,NULL)').run(meta.environmentId,meta.ownerKey)
|
|
|
await Promise.all([...this.workers.values()].filter(worker=>worker.environmentId===meta.environmentId&&worker.ownerKey===meta.ownerKey).map(worker=>worker.process.close()))
|
|
await Promise.all([...this.workers.values()].filter(worker=>worker.environmentId===meta.environmentId&&worker.ownerKey===meta.ownerKey).map(worker=>worker.process.close()))
|
|
|
const current=this.pool.status(meta.ownerKey).find(row=>row.environmentId===meta.environmentId)
|
|
const current=this.pool.status(meta.ownerKey).find(row=>row.environmentId===meta.environmentId)
|
|
|
- if(current?.holderId)await this.pool.stopHolder(meta.ownerKey,meta.environmentId,current.holderId)
|
|
|
|
|
|
|
+ for(const holderId of current?.holderIds??[])await this.pool.stopHolder(meta.ownerKey,meta.environmentId,holderId)
|
|
|
let spec=old?.spec?JSON.parse(old.spec):undefined
|
|
let spec=old?.spec?JSON.parse(old.spec):undefined
|
|
|
spec??=this.pool.definition(meta.ownerKey,meta.environmentId)
|
|
spec??=this.pool.definition(meta.ownerKey,meta.environmentId)
|
|
|
if(spec)this.db.prepare('UPDATE retired_computers SET spec=? WHERE id=?').run(JSON.stringify(spec),meta.environmentId)
|
|
if(spec)this.db.prepare('UPDATE retired_computers SET spec=? WHERE id=?').run(JSON.stringify(spec),meta.environmentId)
|
|
@@ -165,7 +191,7 @@ export class ComputerGateway {
|
|
|
private profileSession?:ProfileComputerSession
|
|
private profileSession?:ProfileComputerSession
|
|
|
private skillSession?:ProfileSkillSession
|
|
private skillSession?:ProfileSkillSession
|
|
|
private get hermesManaged(){return this.meta.hermesRuntime===true}
|
|
private get hermesManaged(){return this.meta.hermesRuntime===true}
|
|
|
- private get rules(){return this.hermesManaged?`${this.meta.profileSession?PROFILE_COMPUTER_RULES:ISOLATED_COMPUTER_RULES}\n${HERMES_COMPUTER_RULES}`:`${PROFILE_COMPUTER_RULES}\n${SKILL_RULES}`}
|
|
|
|
|
|
|
+ private get rules(){const sharing=this.meta.environmentId!==this.meta.agentId?'多个机器人共用这台虚拟机:命令和文件操作并行执行,桌面操作交替进行;动手前注意其他机器人可能正在使用同一桌面,避免覆盖彼此的工作。':'';return this.hermesManaged?`${sharing}${this.meta.profileSession?PROFILE_COMPUTER_RULES:ISOLATED_COMPUTER_RULES}\n${HERMES_COMPUTER_RULES}`:`${sharing}${PROFILE_COMPUTER_RULES}\n${SKILL_RULES}`}
|
|
|
private team?:{id:string;catalog:WorkerTool[];call(name:string,args:Record<string,unknown>,callId:string):Promise<unknown>;workspaceMemory:boolean}
|
|
private team?:{id:string;catalog:WorkerTool[];call(name:string,args:Record<string,unknown>,callId:string):Promise<unknown>;workspaceMemory:boolean}
|
|
|
constructor(readonly runtime:ComputerRuntime,readonly meta:ComputerTarget,readonly workId:string,readonly check:()=>Promise<void>,readonly publish:(body:Record<string,unknown>)=>Promise<any>,readonly profileTarget?:GatewayTarget){ }
|
|
constructor(readonly runtime:ComputerRuntime,readonly meta:ComputerTarget,readonly workId:string,readonly check:()=>Promise<void>,readonly publish:(body:Record<string,unknown>)=>Promise<any>,readonly profileTarget?:GatewayTarget){ }
|
|
|
async connect(){this.runtime.assertTarget(this.meta);await this.runtime.ready;await this.authorized();if(this.hermesManaged&&!this.profileTarget)throw new HttpError(409,'缺少 Hermes 会话连接,不能回退到独立 Worker。','computer_profile_unavailable')}
|
|
async connect(){this.runtime.assertTarget(this.meta);await this.runtime.ready;await this.authorized();if(this.hermesManaged&&!this.profileTarget)throw new HttpError(409,'缺少 Hermes 会话连接,不能回退到独立 Worker。','computer_profile_unavailable')}
|
|
@@ -180,7 +206,7 @@ export class ComputerGateway {
|
|
|
if(method==='session.create'||method==='session.resume'){
|
|
if(method==='session.create'||method==='session.resume'){
|
|
|
if(this.live)throw new HttpError(409,'电脑通道已有会话','computer_session_busy')
|
|
if(this.live)throw new HttpError(409,'电脑通道已有会话','computer_session_busy')
|
|
|
await this.authorized()
|
|
await this.authorized()
|
|
|
- if(method==='session.resume'&¶ms.recoverOnly===true){const session=this.runtime.session(String(params.session_id),this.meta,String(params.profile));this.recoverySession=session;if(session.profileSessionId){await this.openProfile(session,params);await this.profileSession!.stop()}const resource=this.runtime.pool.status(this.meta.ownerKey).find(item=>item.environmentId===this.meta.environmentId);if(resource?.holderId===this.workId)await this.runtime.pool.stopHolder(this.meta.ownerKey,this.meta.environmentId,this.workId);return {session_id:session.id,stored_session_id:session.id,running:false,info:{profile_name:session.profile}}}
|
|
|
|
|
|
|
+ if(method==='session.resume'&¶ms.recoverOnly===true){const session=this.runtime.session(String(params.session_id),this.meta,String(params.profile));this.recoverySession=session;if(session.profileSessionId){await this.openProfile(session,params);await this.profileSession!.stop()}const resource=this.runtime.pool.status(this.meta.ownerKey).find(item=>item.environmentId===this.meta.environmentId);if(resource?.holderIds.includes(this.workId))await this.runtime.pool.stopHolder(this.meta.ownerKey,this.meta.environmentId,this.workId);return {session_id:session.id,stored_session_id:session.id,running:false,info:{profile_name:session.profile}}}
|
|
|
if(this.runtime.imageFor(this.meta)===UNCONFIGURED_COMPUTER_IMAGE)throw new HttpError(409,'请先在应用设置的本地虚拟机页面完成准备','computer_image_required')
|
|
if(this.runtime.imageFor(this.meta)===UNCONFIGURED_COMPUTER_IMAGE)throw new HttpError(409,'请先在应用设置的本地虚拟机页面完成准备','computer_image_required')
|
|
|
const profile=String(params.profile),resolved=this.hermesManaged?{cwd:this.runtime.pool.definition(this.meta.ownerKey,this.meta.environmentId)?.cwd??COMPUTER_WORKSPACE,configuredCwd:'.'}:await (this.meta.profileSession?this.runtime.resolveWorkspace(profile,this.controller.signal):this.runtime.resolve(profile,this.controller.signal))
|
|
const profile=String(params.profile),resolved=this.hermesManaged?{cwd:this.runtime.pool.definition(this.meta.ownerKey,this.meta.environmentId)?.cwd??COMPUTER_WORKSPACE,configuredCwd:'.'}:await (this.meta.profileSession?this.runtime.resolveWorkspace(profile,this.controller.signal):this.runtime.resolve(profile,this.controller.signal))
|
|
|
if(this.meta.hostAccess){const configured=String(resolved.configuredCwd??'.');this.hostCwd=['.','auto','cwd'].includes(configured)?homedir():hostPath(homedir(),configured)}
|
|
if(this.meta.hostAccess){const configured=String(resolved.configuredCwd??'.');this.hostCwd=['.','auto','cwd'].includes(configured)?homedir():hostPath(homedir(),configured)}
|
|
@@ -194,10 +220,10 @@ export class ComputerGateway {
|
|
|
const controller=new AbortController(),lease=await this.runtime.pool.acquire(this.spec(session),this.workId,()=>{this.guard();if(controller.signal.aborted)throw new Error('cancelled')},controller.signal)
|
|
const controller=new AbortController(),lease=await this.runtime.pool.acquire(this.spec(session),this.workId,()=>{this.guard();if(controller.signal.aborted)throw new Error('cancelled')},controller.signal)
|
|
|
if(this.closed){controller.abort();await this.runtime.pool.release(lease);throw new Error('电脑通道已关闭')}
|
|
if(this.closed){controller.abort();await this.runtime.pool.release(lease);throw new Error('电脑通道已关闭')}
|
|
|
this.live={session,lease,controller,model:('model' in resolved?resolved.model:undefined) as WorkerModel,contextConfig:'contextConfig' in resolved?resolved.contextConfig as WorkerContextConfiguration:undefined,proxyEnv:'proxyEnv' in resolved?resolved.proxyEnv as WorkerProxyEnvironment:undefined,images:[],running:false,received:false,toolCalls:new Set(),journal:new Map()}
|
|
this.live={session,lease,controller,model:('model' in resolved?resolved.model:undefined) as WorkerModel,contextConfig:'contextConfig' in resolved?resolved.contextConfig as WorkerContextConfiguration:undefined,proxyEnv:'proxyEnv' in resolved?resolved.proxyEnv as WorkerProxyEnvironment:undefined,images:[],running:false,received:false,toolCalls:new Set(),journal:new Map()}
|
|
|
- this.runtime.save(session);this.runtime.gateways.set(this.meta.environmentId,this)
|
|
|
|
|
|
|
+ this.runtime.save(session);this.runtime.attachGateway(this)
|
|
|
if(this.runtime.config.network==='public-proxy'){
|
|
if(this.runtime.config.network==='public-proxy'){
|
|
|
const live=this.live
|
|
const live=this.live
|
|
|
- try{live.proxy=await ComputerPublicProxy.start(this.runtime.provider,this.spec(session),this.runtime.proxyScript,()=>this.runtime.pool.authorize(live.lease),()=>this.authorized(),()=>{void this.interrupt().finally(()=>{if(!this.closed)this.onDisconnect()}).catch(()=>{})})}
|
|
|
|
|
|
|
+ try{live.proxyHandle=await this.runtime.acquireProxy(this.spec(session),()=>{if(!this.runtime.pool.hasHolders(this.meta.ownerKey,this.meta.environmentId))throw new HttpError(403,'电脑任务授权已失效','computer_authorization_revoked')})}
|
|
|
catch(error){await this.interrupt();throw error}
|
|
catch(error){await this.interrupt();throw error}
|
|
|
}
|
|
}
|
|
|
this.timer=setInterval(()=>{if(this.checking)return;this.checking=true;void this.authorized().then(()=>this.runtime.pool.renew(this.live!.lease)).catch(async()=>{await this.interrupt().catch(()=>{});if(!this.closed)this.onDisconnect()}).finally(()=>{this.checking=false})},5000);this.timer.unref()
|
|
this.timer=setInterval(()=>{if(this.checking)return;this.checking=true;void this.authorized().then(()=>this.runtime.pool.renew(this.live!.lease)).catch(async()=>{await this.interrupt().catch(()=>{});if(!this.closed)this.onDisconnect()}).finally(()=>{this.checking=false})},5000);this.timer.unref()
|
|
@@ -332,7 +358,7 @@ export class ComputerGateway {
|
|
|
return await this.skillSession.call(name,args)
|
|
return await this.skillSession.call(name,args)
|
|
|
}
|
|
}
|
|
|
return await this.runtime.pool.use(live.lease,async context=>{
|
|
return await this.runtime.pool.use(live.lease,async context=>{
|
|
|
- const execute=(argv:string[],input?:Buffer)=>this.runtime.provider.execute(this.spec(live.session),argv,{...context,input})
|
|
|
|
|
|
|
+ const execute=(argv:string[],input?:Buffer,lane?:'exec'|'gui')=>this.runtime.provider.execute(this.spec(live.session),argv,{...context,input,...(lane?{lane}:{})})
|
|
|
if(this.meta.hostAccess&&name.startsWith('host_'))return hostTool(name,args,{...context,cwd:this.hostCwd})
|
|
if(this.meta.hostAccess&&name.startsWith('host_'))return hostTool(name,args,{...context,cwd:this.hostCwd})
|
|
|
if((this.meta.hostAccess||this.hermesManaged)&&name==='computer_copy_file'){
|
|
if((this.meta.hostAccess||this.hermesManaged)&&name==='computer_copy_file'){
|
|
|
const body=z.object({from:z.enum(['host','vm']),source:pathField,destination:pathField}).strict().parse(args),host={...context,cwd:this.hostCwd}
|
|
const body=z.object({from:z.enum(['host','vm']),source:pathField,destination:pathField}).strict().parse(args),host={...context,cwd:this.hostCwd}
|
|
@@ -385,12 +411,12 @@ export class ComputerGateway {
|
|
|
}
|
|
}
|
|
|
if(name==='computer_action'){
|
|
if(name==='computer_action'){
|
|
|
const body=z.object({name:z.string().regex(/^[a-zA-Z][a-zA-Z0-9_]{0,99}$/),arguments:z.record(z.string(),z.unknown())}).strict().parse(args)
|
|
const body=z.object({name:z.string().regex(/^[a-zA-Z][a-zA-Z0-9_]{0,99}$/),arguments:z.record(z.string(),z.unknown())}).strict().parse(args)
|
|
|
- return execute([CUA_DRIVER,'call',body.name,JSON.stringify(body.arguments),'--socket',CUA_SOCKET])
|
|
|
|
|
|
|
+ return execute([CUA_DRIVER,'call',body.name,JSON.stringify(body.arguments),'--socket',CUA_SOCKET],undefined,'gui')
|
|
|
}
|
|
}
|
|
|
if(name==='computer_desktop_state'){
|
|
if(name==='computer_desktop_state'){
|
|
|
z.object({}).strict().parse(args)
|
|
z.object({}).strict().parse(args)
|
|
|
const path=`/tmp/yaoyao-frame-${randomUUID()}.png`
|
|
const path=`/tmp/yaoyao-frame-${randomUUID()}.png`
|
|
|
- const state=await execute([CUA_DRIVER,'call','get_desktop_state','{}','--socket',CUA_SOCKET,'--screenshot-out-file',path])
|
|
|
|
|
|
|
+ const state=await execute([CUA_DRIVER,'call','get_desktop_state','{}','--socket',CUA_SOCKET,'--screenshot-out-file',path],undefined,'gui')
|
|
|
const data=(await execute(['base64','-w0',path])).stdout.trim()
|
|
const data=(await execute(['base64','-w0',path])).stdout.trim()
|
|
|
return {_multimodal:true,text_summary:state.stdout,content:[{type:'image_url',image_url:{url:`data:image/png;base64,${data}`}}]}
|
|
return {_multimodal:true,text_summary:state.stdout,content:[{type:'image_url',image_url:{url:`data:image/png;base64,${data}`}}]}
|
|
|
}
|
|
}
|
|
@@ -425,7 +451,7 @@ export class ComputerGateway {
|
|
|
private release(live:Live,keepRunning=false){
|
|
private release(live:Live,keepRunning=false){
|
|
|
clearInterval(this.timer)
|
|
clearInterval(this.timer)
|
|
|
live.releasing??=(async()=>{
|
|
live.releasing??=(async()=>{
|
|
|
- try{await live.proxy?.close();await this.runtime.pool.release(live.lease,keepRunning)}
|
|
|
|
|
|
|
+ try{await live.proxyHandle?.release();await this.runtime.pool.release(live.lease,keepRunning)}
|
|
|
catch(error){await this.runtime.pool.release(live.lease).catch(()=>{});throw error}
|
|
catch(error){await this.runtime.pool.release(live.lease).catch(()=>{});throw error}
|
|
|
})()
|
|
})()
|
|
|
return live.releasing
|
|
return live.releasing
|
|
@@ -438,11 +464,14 @@ export class ComputerGateway {
|
|
|
get paused(){return this.live?.paused===true}
|
|
get paused(){return this.live?.paused===true}
|
|
|
get lease(){return this.live?.lease}
|
|
get lease(){return this.live?.lease}
|
|
|
get specification(){return this.live?this.spec(this.live.session):undefined}
|
|
get specification(){return this.live?this.spec(this.live.session):undefined}
|
|
|
- async takeControl(controlId:string,authorize:()=>void):Promise<ComputerLease>{
|
|
|
|
|
|
|
+ /** Pauses this turn for a human takeover: settles in-flight operations and
|
|
|
|
|
+ * records pending tool calls so the model can resume coherently. The pool
|
|
|
|
|
+ * lease stays — the human acquires a holder of the same shared desktop. */
|
|
|
|
|
+ async takeControl(controlId:string,authorize:()=>void):Promise<void>{
|
|
|
const live=this.live
|
|
const live=this.live
|
|
|
if(!live||!this.active)throw new HttpError(409,'机器人已不在执行,请重新打开电脑','computer_not_active')
|
|
if(!live||!this.active)throw new HttpError(409,'机器人已不在执行,请重新打开电脑','computer_not_active')
|
|
|
- if(live.pauseJob)return live.pauseJob
|
|
|
|
|
- live.paused=true;this.event('computer.paused',{stage:'pausing'})
|
|
|
|
|
|
|
+ if(live.pauseJob)await live.pauseJob
|
|
|
|
|
+ if(!live.paused){live.paused=true;this.event('computer.paused',{stage:'pausing'})}
|
|
|
live.pauseJob=(async()=>{
|
|
live.pauseJob=(async()=>{
|
|
|
await this.profileSession?.stop()
|
|
await this.profileSession?.stop()
|
|
|
await live.worker?.close()
|
|
await live.worker?.close()
|
|
@@ -455,17 +484,14 @@ export class ComputerGateway {
|
|
|
history.push({role:'tool',name:tool.function?.name??entry?.name,tool_call_id:tool.id,content});done.add(tool.id)
|
|
history.push({role:'tool',name:tool.function?.name??entry?.name,tool_call_id:tool.id,content});done.add(tool.id)
|
|
|
}
|
|
}
|
|
|
live.session.history=history;this.runtime.save(live.session)
|
|
live.session.history=history;this.runtime.save(live.session)
|
|
|
- live.lease=await this.runtime.pool.transfer(live.lease,`human:${controlId}`,authorize)
|
|
|
|
|
this.event('computer.paused',{stage:'human',generation:live.lease.generation})
|
|
this.event('computer.paused',{stage:'human',generation:live.lease.generation})
|
|
|
- return live.lease
|
|
|
|
|
})()
|
|
})()
|
|
|
- return live.pauseJob
|
|
|
|
|
|
|
+ await live.pauseJob
|
|
|
}
|
|
}
|
|
|
async giveBack(notes:string):Promise<void>{
|
|
async giveBack(notes:string):Promise<void>{
|
|
|
const live=this.live
|
|
const live=this.live
|
|
|
if(!live?.paused)throw new HttpError(409,'机器人没有处于接管状态','computer_not_paused')
|
|
if(!live?.paused)throw new HttpError(409,'机器人没有处于接管状态','computer_not_paused')
|
|
|
await live.pauseJob;await this.authorized()
|
|
await live.pauseJob;await this.authorized()
|
|
|
- live.lease=await this.runtime.pool.transfer(live.lease,this.workId,()=>this.guard())
|
|
|
|
|
live.paused=false;live.pauseJob=undefined;live.journal.clear()
|
|
live.paused=false;live.pauseJob=undefined;live.journal.clear()
|
|
|
this.event('computer.resumed',{generation:live.lease.generation,notes})
|
|
this.event('computer.resumed',{generation:live.lease.generation,notes})
|
|
|
if(this.profileSession){
|
|
if(this.profileSession){
|
|
@@ -490,5 +516,5 @@ export class ComputerGateway {
|
|
|
return live.stopping
|
|
return live.stopping
|
|
|
}
|
|
}
|
|
|
async stopControl(){await this.interrupt();this.event('message.complete',{text:'电脑控制已结束,本轮已停止',status:'interrupted'})}
|
|
async stopControl(){await this.interrupt();this.event('message.complete',{text:'电脑控制已结束,本轮已停止',status:'interrupted'})}
|
|
|
- async close(){if(this.runtime.gateways.get(this.meta.environmentId)===this)this.runtime.gateways.delete(this.meta.environmentId);this.closed=true;this.controller.abort();this.team=undefined;clearInterval(this.timer);try{if(this.live?.running||this.live&&!this.live.releasing)await this.interrupt()}finally{await this.profileSession?.close()}}
|
|
|
|
|
|
|
+ async close(){this.runtime.detachGateway(this);this.closed=true;this.controller.abort();this.team=undefined;clearInterval(this.timer);try{if(this.live?.running||this.live&&!this.live.releasing)await this.interrupt()}finally{await this.profileSession?.close()}}
|
|
|
}
|
|
}
|