schema.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. """Schema for the generic `computer_use` tool (model-facing; value is byte-frozen —
  2. the schema goes to the model every turn, so prompt-cache parity depends on it).
  3. Model-agnostic: any tool-calling model can drive this. Vision-capable models
  4. should prefer `capture(mode='som')` then `click(element=N)` — much more reliable
  5. than pixel coordinates, which remain supported for models trained on them.
  6. """
  7. from __future__ import annotations
  8. from typing import Any, Dict
  9. # One consolidated tool with an `action` discriminator keeps the schema compact
  10. # and the per-turn token cost low. Property groups: capture (mode, app, pid,
  11. # window_id) / targeting (element, coordinate, button, modifiers) / drag / scroll /
  12. # set_value / type-key-wait / focus_app / delivery ladder / return shape.
  13. _PROPERTIES: dict[str, Any] = {
  14. "action": {
  15. "type": "string",
  16. "enum": [
  17. "capture",
  18. "click",
  19. "double_click",
  20. "right_click",
  21. "middle_click",
  22. "drag",
  23. "scroll",
  24. "type",
  25. "key",
  26. "set_value",
  27. "wait",
  28. "list_apps",
  29. "list_windows",
  30. "focus_app",
  31. ],
  32. "description": (
  33. "Which action to perform. `capture` is free (no side effects). All other actions "
  34. "require approval unless auto-approved. Use `set_value` for select/popup elements and "
  35. "sliders — it selects the matching option directly without opening the native menu (no "
  36. "focus steal)."
  37. ),
  38. },
  39. "mode": {
  40. "type": "string",
  41. "enum": ["som", "vision", "ax"],
  42. "description": (
  43. "Capture mode. `som` (default) is a screenshot with numbered overlays on every "
  44. "interactable element plus the AX tree — best for vision models, lets you click by "
  45. "element index. `vision` is a plain screenshot. `ax` is the accessibility tree only "
  46. "(no image; useful for text-only models)."
  47. ),
  48. },
  49. "app": {
  50. "type": "string",
  51. "description": (
  52. "Optional. Limit capture/action to one app (name e.g. 'Safari', or bundle ID). Omitted "
  53. "= frontmost window. app='screen' = composited full-screen grab (image only, no "
  54. "clickable elements); app='desktop' = the OS desktop/shell surface (wallpaper, icons, "
  55. "taskbar) with its elements."
  56. ),
  57. },
  58. "pid": {
  59. "type": "integer",
  60. "description": (
  61. "Optional exact process target for action='capture'. Pair with window_id when "
  62. "discovery cannot resolve an X11 app."
  63. ),
  64. },
  65. "window_id": {
  66. "type": "integer",
  67. "description": (
  68. "Optional exact native window target for action='capture'. Pair with pid when an "
  69. "external cua-driver list_windows lookup has already identified the window."
  70. ),
  71. },
  72. "element": {
  73. "type": "integer",
  74. "description": (
  75. "The 1-based SOM index returned by the last `capture(mode='som')` call. Strongly "
  76. "preferred over raw coordinates."
  77. ),
  78. },
  79. "coordinate": {
  80. "type": "array",
  81. "items": {"type": "integer"},
  82. "minItems": 2,
  83. "maxItems": 2,
  84. "description": (
  85. "Pixel coordinates [x, y] relative to the captured window screenshot (top-left "
  86. "origin). Only use this if no element index is available."
  87. ),
  88. },
  89. "button": {
  90. "type": "string",
  91. "enum": ["left", "right", "middle"],
  92. "description": "Mouse button. Defaults to left.",
  93. },
  94. "modifiers": {
  95. "type": "array",
  96. "items": {
  97. "type": "string",
  98. "enum": [
  99. "cmd",
  100. "shift",
  101. "option",
  102. "alt",
  103. "ctrl",
  104. "fn",
  105. "win",
  106. "windows",
  107. "super",
  108. "meta",
  109. ],
  110. },
  111. "description": "Modifier keys held during the action.",
  112. },
  113. "from_element": {"type": "integer", "description": "Source element index (drag)."},
  114. "to_element": {"type": "integer", "description": "Target element index (drag)."},
  115. "from_coordinate": {
  116. "type": "array",
  117. "items": {"type": "integer"},
  118. "minItems": 2,
  119. "maxItems": 2,
  120. "description": "Source [x,y] (drag; use when no element available).",
  121. },
  122. "to_coordinate": {
  123. "type": "array",
  124. "items": {"type": "integer"},
  125. "minItems": 2,
  126. "maxItems": 2,
  127. "description": "Target [x,y] (drag; use when no element available).",
  128. },
  129. "direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."},
  130. "amount": {"type": "integer", "description": "Scroll wheel ticks. Default 3."},
  131. "value": {
  132. "type": "string",
  133. "description": (
  134. "For action='set_value': the value to set on the element. For AXPopUpButton / select "
  135. "dropdowns, pass the option's display label (e.g. 'Blue'). For sliders and other "
  136. "AXValue-settable elements, pass the numeric or string value."
  137. ),
  138. },
  139. "text": {"type": "string", "description": "Text to type (respects the current layout)."},
  140. "keys": {
  141. "type": "string",
  142. "description": (
  143. "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine."
  144. ),
  145. },
  146. "seconds": {"type": "number", "description": "wait: seconds to pause (max 30)."},
  147. "raise_window": {
  148. "type": "boolean",
  149. "description": (
  150. "Only for action='focus_app'. If true, brings the window to front (DISRUPTS the user). "
  151. "Default false — input is routed to the app without raising, matching the background "
  152. "co-work model."
  153. ),
  154. },
  155. "delivery_mode": {
  156. "type": "string",
  157. "enum": ["background", "foreground"],
  158. "description": (
  159. "For input actions (click, type, key, drag, scroll). `background` (DEFAULT) delivers "
  160. "without raising the window or stealing focus. `foreground` briefly fronts the window "
  161. "then restores focus — a visible change needing its own approval; use it only when a "
  162. "result's verdict tells you to escalate there. Each result's `verdict` carries the "
  163. "next step; follow it rather than guessing."
  164. ),
  165. },
  166. "bring_to_front": {
  167. "type": "boolean",
  168. "description": (
  169. "Optional and only valid with delivery_mode='foreground'. Explicitly invokes "
  170. "cua-driver's standalone bring_to_front tool before the input; it is never passed as "
  171. "an input property. This persistent focus change has a separate approval scope. "
  172. "Default false."
  173. ),
  174. },
  175. "capture_after": {
  176. "type": "boolean",
  177. "description": (
  178. "If true, take a follow-up capture after the action and include it in the response. "
  179. "Saves a round-trip when you need to verify an action's effect."
  180. ),
  181. },
  182. }
  183. COMPUTER_USE_SCHEMA: dict[str, Any] = {
  184. "name": "computer_use",
  185. "description": (
  186. "Drive the desktop via cua-driver: screenshots, mouse, keyboard, scroll, drag. macOS, "
  187. "Windows, and Linux. Input is background-FIRST, not background-only: the default delivery "
  188. "routes to the target window without stealing the user's cursor or focus (works even on "
  189. "hidden/minimized windows), and when a result's `verdict` says to escalate you climb — "
  190. "pixel coordinates, or delivery_mode='foreground' (briefly fronts the window; separate "
  191. "approval). Each result carries a `verdict` with the next step; follow it — never repeat "
  192. "confirmed input, and re-capture to verify an unverifiable one before retrying. Workflow: "
  193. "action='capture' (mode='som' gives numbered element overlays), then click by `element` "
  194. "index; re-capture after state-changing actions (or pass capture_after=true). Image "
  195. "captures include a shareable `screenshot_path`; deliver it via the platform's MEDIA "
  196. "syntax when the user asks to see it — not for captures used only for control."
  197. ),
  198. "parameters": {"type": "object", "properties": _PROPERTIES, "required": ["action"]},
  199. }
  200. def get_computer_use_schema() -> dict[str, Any]:
  201. """Return the generic OpenAI function-calling schema."""
  202. return COMPUTER_USE_SCHEMA