|
@@ -54,7 +54,7 @@ interface HostState {
|
|
|
}
|
|
}
|
|
|
const newHostState=(key:string,remote:boolean):HostState=>({key,remote,seen:0,pending:new Map(),queue:new Map(),manual:new Map(),paused:new Set(),frames:new Map(),generation:new Map(),notes:new Map(),activity:new Map()})
|
|
const newHostState=(key:string,remote:boolean):HostState=>({key,remote,seen:0,pending:new Map(),queue:new Map(),manual:new Map(),paused:new Set(),frames:new Map(),generation:new Map(),notes:new Map(),activity:new Map()})
|
|
|
export const DESKTOP_ENVIRONMENT_TOOLS=[
|
|
export const DESKTOP_ENVIRONMENT_TOOLS=[
|
|
|
- {id:'desktop_environment_view',name:'desktop_environment_view',description:'查看电脑真实桌面截图,返回截图尺寸、环境类型和电脑名称。操作前必须先看。可用 host 指定目标电脑;省略或写「本机」是夭夭所在的服务器。',inputSchema:{type:'object',properties:{host:{type:'string',description:'可选,目标电脑的名称或编号;缺省用默认电脑'}},additionalProperties:false}},
|
|
|
|
|
|
|
+ {id:'desktop_environment_view',name:'desktop_environment_view',description:'查看电脑真实桌面截图,返回截图尺寸、环境类型和电脑名称。操作前必须先看。可用 host 指定目标电脑;省略或写「本机」时,优先对应用户当前发消息所在的那台电脑(有来源时),否则用默认绑定。',inputSchema:{type:'object',properties:{host:{type:'string',description:'可选,目标电脑的名称或编号;缺省用默认电脑'}},additionalProperties:false}},
|
|
|
{id:'desktop_environment_action',name:'desktop_environment_action',description:'操作电脑桌面。先查看真实桌面,再使用 click、text、key、scroll 或 drag;坐标用截图像素。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{host:{type:'string'},kind:{enum:['click','drag','text','key','scroll']},x:{type:'integer'},y:{type:'integer'},fromX:{type:'integer'},fromY:{type:'integer'},toX:{type:'integer'},toY:{type:'integer'},button:{enum:['left','right','middle']},count:{type:'integer'},text:{type:'string'},key:{type:'string'},modifiers:{type:'array',items:{enum:['ctrl','alt','shift','super']}},direction:{enum:['up','down','left','right']},amount:{type:'integer'}},required:['kind'],additionalProperties:false}},
|
|
{id:'desktop_environment_action',name:'desktop_environment_action',description:'操作电脑桌面。先查看真实桌面,再使用 click、text、key、scroll 或 drag;坐标用截图像素。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{host:{type:'string'},kind:{enum:['click','drag','text','key','scroll']},x:{type:'integer'},y:{type:'integer'},fromX:{type:'integer'},fromY:{type:'integer'},toX:{type:'integer'},toY:{type:'integer'},button:{enum:['left','right','middle']},count:{type:'integer'},text:{type:'string'},key:{type:'string'},modifiers:{type:'array',items:{enum:['ctrl','alt','shift','super']}},direction:{enum:['up','down','left','right']},amount:{type:'integer'}},required:['kind'],additionalProperties:false}},
|
|
|
{id:'desktop_file_list',name:'desktop_file_list',description:'列出当前电脑允许范围内的目录内容,返回名称、类型、大小与修改时间。路径相对于用户主目录,可用 ~。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{path:{type:'string'},host:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
{id:'desktop_file_list',name:'desktop_file_list',description:'列出当前电脑允许范围内的目录内容,返回名称、类型、大小与修改时间。路径相对于用户主目录,可用 ~。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{path:{type:'string'},host:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
|
{id:'desktop_file_read',name:'desktop_file_read',description:'读取当前电脑上的一个文件,返回 base64 内容与大小。路径相对于用户主目录;超出允许范围会被拒绝。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{path:{type:'string'},host:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
{id:'desktop_file_read',name:'desktop_file_read',description:'读取当前电脑上的一个文件,返回 base64 内容与大小。路径相对于用户主目录;超出允许范围会被拒绝。可用 host 指定目标电脑。',inputSchema:{type:'object',properties:{path:{type:'string'},host:{type:'string'}},required:['path'],additionalProperties:false}},
|
|
@@ -63,9 +63,9 @@ export const DESKTOP_ENVIRONMENT_TOOLS=[
|
|
|
]
|
|
]
|
|
|
export const DESKTOP_FILE_TOOL_IDS=new Set(['desktop_file_list','desktop_file_read','desktop_file_write','desktop_shell'])
|
|
export const DESKTOP_FILE_TOOL_IDS=new Set(['desktop_file_list','desktop_file_read','desktop_file_write','desktop_shell'])
|
|
|
/** Satellite Macs connected as script machines. */
|
|
/** Satellite Macs connected as script machines. */
|
|
|
-export const DESKTOP_ENVIRONMENT_RULES='当前环境是脚本机,也就是客户连接的 Mac。电脑操作、文件读写和命令必须使用 desktop_*,并用 host 写明电脑名称。先查看真实桌面,再使用 click、text、key、scroll 或 drag;页面变化后重新截图,不得猜测旧坐标。文件与 shell 在用户主目录范围内真实生效。服务器本机、虚拟环境和云虚拟机都不是脚本机。人工接管时等待交还。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
|
|
|
|
+export const DESKTOP_ENVIRONMENT_RULES='当前环境是脚本机,也就是客户连接的 Mac。电脑操作、文件读写和命令必须使用 desktop_*;可用 host 写明电脑名称或编号。省略 host 或写「本机」时:若本轮用户消息带来源设备,就用那台电脑;否则用 Bot 的默认绑定。先查看真实桌面,再使用 click、text、key、scroll 或 drag;页面变化后重新截图,不得猜测旧坐标。文件与 shell 在用户主目录范围内真实生效。服务器、虚拟环境和云虚拟机都不是脚本机。人工接管时等待交还。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
/** Yaoyao host Mac — same look-then-act pattern as the cloud computer. */
|
|
/** Yaoyao host Mac — same look-then-act pattern as the cloud computer. */
|
|
|
-export const SERVER_COMPUTER_RULES='当前环境是服务器,也就是夭夭正在运行的这台电脑。电脑操作、文件读写和命令必须使用 desktop_*;省略 host 或写「本机」只看这台电脑。先查看真实桌面,再使用 click、text、key、scroll 或 drag;页面变化后重新截图,不得猜测旧坐标。文件与 shell 在用户主目录范围内真实生效。脚本机、虚拟环境和云虚拟机都不是这台服务器。人工接管时等待交还。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
|
|
|
|
+export const SERVER_COMPUTER_RULES='当前环境是服务器,也就是夭夭正在运行的这台电脑。电脑名称与设置里一致,可改名;点名用这个名字。电脑操作、文件读写和命令必须使用 desktop_*。省略 host 或写「本机」时:若本轮用户消息来自服务器,就用这台电脑;若来自某台脚本机,则「本机」指那台脚本机;没有来源时默认仍是这台服务器。先查看真实桌面,再使用 click、text、key、scroll 或 drag;页面变化后重新截图,不得猜测旧坐标。文件与 shell 在用户主目录范围内真实生效。脚本机、虚拟环境和云虚拟机都不是这台服务器本身。人工接管时等待交还。电脑断开时报告原因,不得宣称操作成功。'
|
|
|
|
|
|
|
|
/** The loopback native transport is admitted only by serviceInstance's
|
|
/** The loopback native transport is admitted only by serviceInstance's
|
|
|
* private capability; remote hosts are admitted by DesktopHostHub's bearer
|
|
* private capability; remote hosts are admitted by DesktopHostHub's bearer
|
|
@@ -85,11 +85,11 @@ export class DesktopEnvironments {
|
|
|
private readyFull(s:HostState,owner:string){return this.hostOnline(s)&&s.info!.platform==='darwin'&&(s.info!.full??[]).includes(this.ownerKey(owner))}
|
|
private readyFull(s:HostState,owner:string){return this.hostOnline(s)&&s.info!.platform==='darwin'&&(s.info!.full??[]).includes(this.ownerKey(owner))}
|
|
|
fileTools(owner:string,agent:WorkspaceAgent){const s=this.sessionFor(agent);return !!s&&this.readyFull(s,owner)}
|
|
fileTools(owner:string,agent:WorkspaceAgent){const s=this.sessionFor(agent);return !!s&&this.readyFull(s,owner)}
|
|
|
private localSession(){let s=this.sessions.get('local');if(!s){s=newHostState('local',false);this.sessions.set('local',s)}return s}
|
|
private localSession(){let s=this.sessions.get('local');if(!s){s=newHostState('local',false);this.sessions.set('local',s)}return s}
|
|
|
- /** 本机 is the computer running YaoYao. A non-local binding still wins for the
|
|
|
|
|
- * default session. Other script machines are selected by name, never by being the only one online. */
|
|
|
|
|
|
|
+ /** Default session: agent desktopHost binding, else server local.
|
|
|
|
|
+ * 「本机」 at tool time prefers the user message deviceHost when provided. */
|
|
|
private computers(){try{const s=readHostTools(this.store.home);return {script:s.scriptMachine,server:s.serverComputer}}catch{return {script:false,server:false}}}
|
|
private computers(){try{const s=readHostTools(this.store.home);return {script:s.scriptMachine,server:s.serverComputer}}catch{return {script:false,server:false}}}
|
|
|
private names(){try{return readComputerNames(this.store.home)}catch{return {}}}
|
|
private names(){try{return readComputerNames(this.store.home)}catch{return {}}}
|
|
|
- private label(s:HostState){const custom=this.names()[s.key];if(s.key==='local')return custom||'本机';return custom||s.info?.name||''}
|
|
|
|
|
|
|
+ private label(s:HostState){const custom=this.names()[s.key];if(s.key==='local')return custom||'服务器';return custom||s.info?.name||''}
|
|
|
private hostOpen(s:HostState){const g=this.computers();return s.key==='local'?g.server:g.script}
|
|
private hostOpen(s:HostState){const g=this.computers();return s.key==='local'?g.server:g.script}
|
|
|
private sessionFor(agent:WorkspaceAgent):HostState|undefined{
|
|
private sessionFor(agent:WorkspaceAgent):HostState|undefined{
|
|
|
const g=this.computers()
|
|
const g=this.computers()
|
|
@@ -100,18 +100,28 @@ export class DesktopEnvironments {
|
|
|
if(!g.server)return undefined
|
|
if(!g.server)return undefined
|
|
|
return this.sessions.get('local')
|
|
return this.sessions.get('local')
|
|
|
}
|
|
}
|
|
|
- private namedHost(host:string):HostState{
|
|
|
|
|
|
|
+ /** Resolve an explicit host name/id. When host is 本机/local, prefer contextDeviceHost. */
|
|
|
|
|
+ private namedHost(host:string,contextDeviceHost?:string):HostState{
|
|
|
const g=this.computers()
|
|
const g=this.computers()
|
|
|
if(host==='本机'||host==='local'){
|
|
if(host==='本机'||host==='local'){
|
|
|
- if(!g.server)throw new HttpError(409,'全局设置未开放夭夭所在电脑的操作。','desktop_offline')
|
|
|
|
|
|
|
+ const contextual=this.contextSession(contextDeviceHost)
|
|
|
|
|
+ if(contextual)return contextual
|
|
|
|
|
+ if(!g.server)throw new HttpError(409,'全局设置未开放服务器电脑的操作。','desktop_offline')
|
|
|
const local=this.sessions.get('local')
|
|
const local=this.sessions.get('local')
|
|
|
if(local&&this.hostOnline(local))return local
|
|
if(local&&this.hostOnline(local))return local
|
|
|
- throw new HttpError(409,'夭夭所在的电脑未连接,请在这台电脑上打开夭夭。','desktop_offline')
|
|
|
|
|
|
|
+ throw new HttpError(409,'服务器电脑未连接,请在这台电脑上打开夭夭。','desktop_offline')
|
|
|
}
|
|
}
|
|
|
if(!g.script)throw new HttpError(409,'全局设置未开放脚本机操作。','desktop_offline')
|
|
if(!g.script)throw new HttpError(409,'全局设置未开放脚本机操作。','desktop_offline')
|
|
|
const names=this.names();for(const s of this.sessions.values())if(s.key===host||(s.info&&s.info.name.toLowerCase()===host.toLowerCase())||(names[s.key]&&names[s.key].toLowerCase()===host.toLowerCase()))return s
|
|
const names=this.names();for(const s of this.sessions.values())if(s.key===host||(s.info&&s.info.name.toLowerCase()===host.toLowerCase())||(names[s.key]&&names[s.key].toLowerCase()===host.toLowerCase()))return s
|
|
|
throw new HttpError(409,`指定的电脑不在线:${host.slice(0,64)}`,'desktop_host_unknown')
|
|
throw new HttpError(409,`指定的电脑不在线:${host.slice(0,64)}`,'desktop_host_unknown')
|
|
|
}
|
|
}
|
|
|
|
|
+ /** Session for the user's messaging device when it is online and globally open. */
|
|
|
|
|
+ private contextSession(deviceHost?:string):HostState|undefined{
|
|
|
|
|
+ if(!deviceHost)return undefined
|
|
|
|
|
+ const s=this.sessions.get(deviceHost==='local'?'local':deviceHost)
|
|
|
|
|
+ if(!s||!this.hostOnline(s)||!this.hostOpen(s))return undefined
|
|
|
|
|
+ return s
|
|
|
|
|
+ }
|
|
|
selected(owner:string,agent:WorkspaceAgent):Mode|undefined {if(agent.archived||agent.remoteAgentId||agent.temporaryGoalId)return undefined;const g=this.computers();if(!g.script&&!g.server)return undefined;if(agent.envs)return 'local';if(agent.computer==='local')return 'local';if(agent.computer==='auto'&&agent.execution!=='computer'){const s=this.sessionFor(agent);if(s&&this.ready(s,owner))return 'local'}return undefined}
|
|
selected(owner:string,agent:WorkspaceAgent):Mode|undefined {if(agent.archived||agent.remoteAgentId||agent.temporaryGoalId)return undefined;const g=this.computers();if(!g.script&&!g.server)return undefined;if(agent.envs)return 'local';if(agent.computer==='local')return 'local';if(agent.computer==='auto'&&agent.execution!=='computer'){const s=this.sessionFor(agent);if(s&&this.ready(s,owner))return 'local'}return undefined}
|
|
|
toolMode(owner:string,agent:WorkspaceAgent):Mode|undefined {if(allowsHostEnvironment(agent)){const s=this.sessionFor(agent);if(s&&this.ready(s,owner))return 'local'}return this.selected(owner,agent)}
|
|
toolMode(owner:string,agent:WorkspaceAgent):Mode|undefined {if(allowsHostEnvironment(agent)){const s=this.sessionFor(agent);if(s&&this.ready(s,owner))return 'local'}return this.selected(owner,agent)}
|
|
|
/** Intent for parallel desktop environments (screen control / browser). */
|
|
/** Intent for parallel desktop environments (screen control / browser). */
|
|
@@ -130,13 +140,35 @@ export class DesktopEnvironments {
|
|
|
if(typeof host==='string'&&host)return this.namedHost(host)
|
|
if(typeof host==='string'&&host)return this.namedHost(host)
|
|
|
return this.sessionFor(a)
|
|
return this.sessionFor(a)
|
|
|
}
|
|
}
|
|
|
- private targetFor(agent:WorkspaceAgent,host:unknown):HostState{
|
|
|
|
|
- if(typeof host==='string'&&host)return this.namedHost(host)
|
|
|
|
|
|
|
+ private targetFor(agent:WorkspaceAgent,host:unknown,contextDeviceHost?:string):HostState{
|
|
|
|
|
+ if(typeof host==='string'&&host)return this.namedHost(host,contextDeviceHost)
|
|
|
|
|
+ const contextual=this.contextSession(contextDeviceHost)
|
|
|
|
|
+ if(contextual)return contextual
|
|
|
const s=this.sessionFor(agent)
|
|
const s=this.sessionFor(agent)
|
|
|
if(!s)throw new HttpError(409,'桌面端未连接,请在电脑上打开夭夭。','desktop_offline')
|
|
if(!s)throw new HttpError(409,'桌面端未连接,请在电脑上打开夭夭。','desktop_offline')
|
|
|
return s
|
|
return s
|
|
|
}
|
|
}
|
|
|
- onlineHostsLine(owner:string){const names=[...this.sessions.values()].filter(s=>this.hostOnline(s)&&this.hostOpen(s)).map(s=>s.key==='local'?(this.names()[s.key]?this.names()[s.key]+'(本机)':'本机'):(this.label(s)||s.key));return names.length?`当前在线电脑:${names.join('、')}。desktop_* 工具可用 host 参数(名称或编号)选择目标电脑;省略 host 或写「本机」时使用夭夭所在的这台电脑。`:''}
|
|
|
|
|
|
|
+ onlineHostsLine(owner:string,deviceHost?:string){
|
|
|
|
|
+ const names=[...this.sessions.values()].filter(s=>this.hostOnline(s)&&this.hostOpen(s)).map(s=>{
|
|
|
|
|
+ const label=s.key==='local'?(this.names()[s.key]||'服务器'):(this.label(s)||s.key)
|
|
|
|
|
+ const kind=s.key==='local'?'服务器':'脚本机'
|
|
|
|
|
+ const mine=deviceHost&&(deviceHost==='local'?s.key==='local':s.key===deviceHost)
|
|
|
|
|
+ return mine?`${label}(${kind}·本机)`:`${label}(${kind})`
|
|
|
|
|
+ })
|
|
|
|
|
+ const tip=deviceHost
|
|
|
|
|
+ ?'省略 host 或写「本机」时,优先使用用户当前发消息所在的那台电脑;也可用设置里的电脑名称指定其它电脑。'
|
|
|
|
|
+ :'省略 host 或写「本机」时,没有用户设备来源则默认服务器,否则用 Bot 绑定;也可用设置里的电脑名称指定。'
|
|
|
|
|
+ return names.length?`当前在线电脑:${names.join('、')}。desktop_* 工具可用 host 参数(名称或编号)选择目标电脑;${tip}`:''
|
|
|
|
|
+ }
|
|
|
|
|
+ deviceContextLine(owner:string,deviceHost:string){
|
|
|
|
|
+ const s=this.contextSession(deviceHost)
|
|
|
|
|
+ if(!s){
|
|
|
|
|
+ const named=this.names()[deviceHost==='local'?'local':deviceHost]
|
|
|
|
|
+ return `本轮用户消息来源设备:${named||deviceHost}(当前未在线或未开放)。`
|
|
|
|
|
+ }
|
|
|
|
|
+ const kind=s.key==='local'?'服务器':'脚本机'
|
|
|
|
|
+ return `本轮用户消息来自${kind}「${this.label(s)}」。名字与设置里的电脑名称相同;对本轮而言,「本机」指这台电脑。`
|
|
|
|
|
+ }
|
|
|
private agent(owner:string,id:string){const a=this.store.require<WorkspaceAgent>(owner,'agent',id);this.nodes.requireSource(owner,a);if(a.archived||a.remoteAgentId||a.temporaryGoalId)throw new HttpError(409,'此机器人无法使用桌面环境','computer_unavailable');return a}
|
|
private agent(owner:string,id:string){const a=this.store.require<WorkspaceAgent>(owner,'agent',id);this.nodes.requireSource(owner,a);if(a.archived||a.remoteAgentId||a.temporaryGoalId)throw new HttpError(409,'此机器人无法使用桌面环境','computer_unavailable');return a}
|
|
|
private resource(owner:string,agent:WorkspaceAgent,mode:Mode){return mode==='local'?'local':digest(this.ownerKey(owner)+':'+agent.id+':'+(agent.browserProfile??'persistent'))}
|
|
private resource(owner:string,agent:WorkspaceAgent,mode:Mode){return mode==='local'?'local':digest(this.ownerKey(owner)+':'+agent.id+':'+(agent.browserProfile??'persistent'))}
|
|
|
hostStates(owner:string){return [...this.sessions.values()].map(s=>{const online=this.hostOnline(s),info=online?s.info!:undefined;return {id:s.key,name:this.label(s),platform:info?.platform??'',online,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s,owner),fullAuthorized:!!info&&(info.full??[]).includes(this.ownerKey(owner))},browser:{available:online}}}).sort((a,b)=>a.id==='local'?-1:b.id==='local'?1:a.name.localeCompare(b.name))}
|
|
hostStates(owner:string){return [...this.sessions.values()].map(s=>{const online=this.hostOnline(s),info=online?s.info!:undefined;return {id:s.key,name:this.label(s),platform:info?.platform??'',online,local:{supported:info?.platform==='darwin',authorized:!!info&&info.approved.includes(this.ownerKey(owner)),screen:info?.screen===true,accessibility:info?.accessibility===true,ready:!!info&&this.ready(s,owner),fullAuthorized:!!info&&(info.full??[]).includes(this.ownerKey(owner))},browser:{available:online}}}).sort((a,b)=>a.id==='local'?-1:b.id==='local'?1:a.name.localeCompare(b.name))}
|
|
@@ -199,11 +231,11 @@ export class DesktopEnvironments {
|
|
|
}
|
|
}
|
|
|
const mode=this.toolMode(owner,a);if(!mode)throw new HttpError(409,'当前未选择桌面环境','desktop_unselected');this.context(this.sessionFor(a),owner,a,mode).check();await requireTeamToolBridge(target,a.profile)
|
|
const mode=this.toolMode(owner,a);if(!mode)throw new HttpError(409,'当前未选择桌面环境','desktop_unselected');this.context(this.sessionFor(a),owner,a,mode).check();await requireTeamToolBridge(target,a.profile)
|
|
|
}
|
|
}
|
|
|
- async call(owner:string,agentId:string,id:string,input:unknown,signal:AbortSignal,assertActive:()=>void,expected:Mode,epoch:string|Record<string,string>|undefined){
|
|
|
|
|
|
|
+ async call(owner:string,agentId:string,id:string,input:unknown,signal:AbortSignal,assertActive:()=>void,expected:Mode,epoch:string|Record<string,string>|undefined,deviceHost?:string){
|
|
|
if(id==='desktop_browser'||expected==='browser')throw new HttpError(404,'当前环境不支持这个工具','tool_not_found')
|
|
if(id==='desktop_browser'||expected==='browser')throw new HttpError(404,'当前环境不支持这个工具','tool_not_found')
|
|
|
const a=this.agent(owner,agentId)
|
|
const a=this.agent(owner,agentId)
|
|
|
const {host,...arguments_}=typeof input==='object'&&input?input as Record<string,unknown>:{}
|
|
const {host,...arguments_}=typeof input==='object'&&input?input as Record<string,unknown>:{}
|
|
|
- const s=this.targetFor(a,host)
|
|
|
|
|
|
|
+ const s=this.targetFor(a,host,deviceHost)
|
|
|
const c=this.context(s,owner,a,expected)
|
|
const c=this.context(s,owner,a,expected)
|
|
|
const pinned=typeof epoch==='string'?(s===this.sessionFor(a)?epoch:undefined):epoch?.[s.key]
|
|
const pinned=typeof epoch==='string'?(s===this.sessionFor(a)?epoch:undefined):epoch?.[s.key]
|
|
|
const check=()=>{signal.throwIfAborted();assertActive();c.check();if(pinned!==undefined&&s.info?.id!==pinned)throw new HttpError(410,'桌面连接已改变','desktop_disconnected')};check()
|
|
const check=()=>{signal.throwIfAborted();assertActive();c.check();if(pinned!==undefined&&s.info?.id!==pinned)throw new HttpError(410,'桌面连接已改变','desktop_disconnected')};check()
|