| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208 |
- """Schema for the generic `computer_use` tool (model-facing; value is byte-frozen —
- the schema goes to the model every turn, so prompt-cache parity depends on it).
- Model-agnostic: any tool-calling model can drive this. Vision-capable models
- should prefer `capture(mode='som')` then `click(element=N)` — much more reliable
- than pixel coordinates, which remain supported for models trained on them.
- """
- from __future__ import annotations
- from typing import Any, Dict
- # One consolidated tool with an `action` discriminator keeps the schema compact
- # and the per-turn token cost low. Property groups: capture (mode, app, pid,
- # window_id) / targeting (element, coordinate, button, modifiers) / drag / scroll /
- # set_value / type-key-wait / focus_app / delivery ladder / return shape.
- _PROPERTIES: dict[str, Any] = {
- "action": {
- "type": "string",
- "enum": [
- "capture",
- "click",
- "double_click",
- "right_click",
- "middle_click",
- "drag",
- "scroll",
- "type",
- "key",
- "set_value",
- "wait",
- "list_apps",
- "list_windows",
- "focus_app",
- ],
- "description": (
- "Which action to perform. `capture` is free (no side effects). All other actions "
- "require approval unless auto-approved. Use `set_value` for select/popup elements and "
- "sliders — it selects the matching option directly without opening the native menu (no "
- "focus steal)."
- ),
- },
- "mode": {
- "type": "string",
- "enum": ["som", "vision", "ax"],
- "description": (
- "Capture mode. `som` (default) is a screenshot with numbered overlays on every "
- "interactable element plus the AX tree — best for vision models, lets you click by "
- "element index. `vision` is a plain screenshot. `ax` is the accessibility tree only "
- "(no image; useful for text-only models)."
- ),
- },
- "app": {
- "type": "string",
- "description": (
- "Optional. Limit capture/action to one app (name e.g. 'Safari', or bundle ID). Omitted "
- "= frontmost window. app='screen' = composited full-screen grab (image only, no "
- "clickable elements); app='desktop' = the OS desktop/shell surface (wallpaper, icons, "
- "taskbar) with its elements."
- ),
- },
- "pid": {
- "type": "integer",
- "description": (
- "Optional exact process target for action='capture'. Pair with window_id when "
- "discovery cannot resolve an X11 app."
- ),
- },
- "window_id": {
- "type": "integer",
- "description": (
- "Optional exact native window target for action='capture'. Pair with pid when an "
- "external cua-driver list_windows lookup has already identified the window."
- ),
- },
- "element": {
- "type": "integer",
- "description": (
- "The 1-based SOM index returned by the last `capture(mode='som')` call. Strongly "
- "preferred over raw coordinates."
- ),
- },
- "coordinate": {
- "type": "array",
- "items": {"type": "integer"},
- "minItems": 2,
- "maxItems": 2,
- "description": (
- "Pixel coordinates [x, y] relative to the captured window screenshot (top-left "
- "origin). Only use this if no element index is available."
- ),
- },
- "button": {
- "type": "string",
- "enum": ["left", "right", "middle"],
- "description": "Mouse button. Defaults to left.",
- },
- "modifiers": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "cmd",
- "shift",
- "option",
- "alt",
- "ctrl",
- "fn",
- "win",
- "windows",
- "super",
- "meta",
- ],
- },
- "description": "Modifier keys held during the action.",
- },
- "from_element": {"type": "integer", "description": "Source element index (drag)."},
- "to_element": {"type": "integer", "description": "Target element index (drag)."},
- "from_coordinate": {
- "type": "array",
- "items": {"type": "integer"},
- "minItems": 2,
- "maxItems": 2,
- "description": "Source [x,y] (drag; use when no element available).",
- },
- "to_coordinate": {
- "type": "array",
- "items": {"type": "integer"},
- "minItems": 2,
- "maxItems": 2,
- "description": "Target [x,y] (drag; use when no element available).",
- },
- "direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."},
- "amount": {"type": "integer", "description": "Scroll wheel ticks. Default 3."},
- "value": {
- "type": "string",
- "description": (
- "For action='set_value': the value to set on the element. For AXPopUpButton / select "
- "dropdowns, pass the option's display label (e.g. 'Blue'). For sliders and other "
- "AXValue-settable elements, pass the numeric or string value."
- ),
- },
- "text": {"type": "string", "description": "Text to type (respects the current layout)."},
- "keys": {
- "type": "string",
- "description": (
- "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine."
- ),
- },
- "seconds": {"type": "number", "description": "wait: seconds to pause (max 30)."},
- "raise_window": {
- "type": "boolean",
- "description": (
- "Only for action='focus_app'. If true, brings the window to front (DISRUPTS the user). "
- "Default false — input is routed to the app without raising, matching the background "
- "co-work model."
- ),
- },
- "delivery_mode": {
- "type": "string",
- "enum": ["background", "foreground"],
- "description": (
- "For input actions (click, type, key, drag, scroll). `background` (DEFAULT) delivers "
- "without raising the window or stealing focus. `foreground` briefly fronts the window "
- "then restores focus — a visible change needing its own approval; use it only when a "
- "result's verdict tells you to escalate there. Each result's `verdict` carries the "
- "next step; follow it rather than guessing."
- ),
- },
- "bring_to_front": {
- "type": "boolean",
- "description": (
- "Optional and only valid with delivery_mode='foreground'. Explicitly invokes "
- "cua-driver's standalone bring_to_front tool before the input; it is never passed as "
- "an input property. This persistent focus change has a separate approval scope. "
- "Default false."
- ),
- },
- "capture_after": {
- "type": "boolean",
- "description": (
- "If true, take a follow-up capture after the action and include it in the response. "
- "Saves a round-trip when you need to verify an action's effect."
- ),
- },
- }
- COMPUTER_USE_SCHEMA: dict[str, Any] = {
- "name": "computer_use",
- "description": (
- "Drive the desktop via cua-driver: screenshots, mouse, keyboard, scroll, drag. macOS, "
- "Windows, and Linux. Input is background-FIRST, not background-only: the default delivery "
- "routes to the target window without stealing the user's cursor or focus (works even on "
- "hidden/minimized windows), and when a result's `verdict` says to escalate you climb — "
- "pixel coordinates, or delivery_mode='foreground' (briefly fronts the window; separate "
- "approval). Each result carries a `verdict` with the next step; follow it — never repeat "
- "confirmed input, and re-capture to verify an unverifiable one before retrying. Workflow: "
- "action='capture' (mode='som' gives numbered element overlays), then click by `element` "
- "index; re-capture after state-changing actions (or pass capture_after=true). Image "
- "captures include a shareable `screenshot_path`; deliver it via the platform's MEDIA "
- "syntax when the user asks to see it — not for captures used only for control."
- ),
- "parameters": {"type": "object", "properties": _PROPERTIES, "required": ["action"]},
- }
- def get_computer_use_schema() -> dict[str, Any]:
- """Return the generic OpenAI function-calling schema."""
- return COMPUTER_USE_SCHEMA
|