cua_backend_parse.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. """Pure parsing helpers for the cua-driver backend: MCP result flattening, ``list_windows`` /
  2. ``get_window_state`` payload normalisation, key combos. No I/O, no module state — every function
  3. depends only on its inputs, so the MCP and CLI transports share them safely."""
  4. from __future__ import annotations
  5. import contextlib
  6. import json
  7. import re
  8. from typing import Any, Dict, List, Optional, Tuple
  9. from agent.image_eviction_policy import image_dimensions_from_bytes
  10. from tools.computer_use.backend import ActionResult, UIElement
  11. # Linux/X11 surfaces GNOME Shell / desktop backdrop windows ahead of real app windows with no useful z-order; they
  12. # are targetable but capture as empty, so default capture skips them.
  13. _NON_APP_WINDOW_TITLE_PREFIXES = ("@!", "Desktop", "gnome-shell", "GNOME Shell") # "@!" = GNOME helpers
  14. _ELEMENT_LINE_RE = re.compile(
  15. r'^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)'
  16. r'(?:'
  17. r'\s*=\s*"([^"]*)"' # = "value"
  18. r'|\s+"([^"]*)"' # "value"
  19. r'|\s+\((?!\d+\))([^)]*)\)' # (value) but not a pure-digit (order) number
  20. r')?'
  21. r'(?:\s+(?:\(\d+\)\s+)?id=([^\s\[\]]+))?', # optional id=value (after an optional (order))
  22. re.MULTILINE,
  23. )
  24. """get_window_state AX-tree markdown line: ``[N] AXRole`` + label in one of four forms (``= "value"``, ``"quoted"``,
  25. ``(paren)``, ``id=Label`` optionally after an ``(order)`` number). A parenthesised pure-digit group is an ORDER
  26. index, not a label, and is excluded so the id= label wins. Group 1 index, group 2 role, groups 3-6 the label."""
  27. def _mcp_field(obj, snake: str, camel: str, default=None):
  28. """Read an MCP model field across the 1.x -> 2.x rename: mcp 2.0 exposes snake_case attributes and keeps
  29. camelCase only as a serialization alias, so ``getattr(result, "isError", False)`` is False for every 2.x result
  30. and a denied call looks like success. Deliberately duplicated from ``tools.mcp_tool.mcp_field`` so computer_use
  31. never loads the much larger config-driven MCP client module."""
  32. return getattr(obj, snake, getattr(obj, camel, default))
  33. def _action_result_from(name: str, ok: bool, message: str, meta: dict[str, Any],
  34. structured: dict[str, Any], *, requested_delivery: Optional[str] = None) -> ActionResult:
  35. """Build an ActionResult, lifting cua-driver's structured verdict. structuredContent is canonical, the flattened
  36. ``meta`` copy the fallback. Every structured field is additive: a driver that omits one leaves the attribute
  37. ``None`` so old drivers see unchanged behavior.
  38. See the action response shape in cua-driver's mcp-tool-notes and NousResearch/hermes-agent#67052.
  39. """
  40. sc = structured if isinstance(structured, dict) else {}
  41. def _raw(key: str) -> Any:
  42. return sc.get(key) if key in sc else meta.get(key)
  43. def _typed(value: Any, typ) -> Any:
  44. return value if isinstance(value, typ) else None
  45. return ActionResult(
  46. ok=ok, action=name, message=message, meta=meta,
  47. verified=_typed(_raw("verified"), bool), effect=_typed(_raw("effect"), str),
  48. escalation=_typed(_raw("escalation"), dict), path=_typed(_raw("path"), str),
  49. degraded=_typed(_raw("degraded"), bool),
  50. # What we asked for; the driver's `path` records the rung that ran.
  51. delivery_mode=_typed(requested_delivery, str),
  52. # Refusal/limitation code — drivers spell it "code" or "reason_code".
  53. code=_typed(_raw("code") or _raw("reason_code"), str),
  54. )
  55. def _z_index_uninformative(windows: list[dict[str, Any]]) -> bool:
  56. """True when every window shares the same z_index (common on Linux/X11)."""
  57. return len({w.get("z_index", 0) for w in windows}) <= 1
  58. def _parse_xprop_net_active_window(stdout: str) -> Optional[int]:
  59. """Parse ``xprop -root _NET_ACTIVE_WINDOW`` stdout into a window id: the ``window id # 0x...`` form, falling
  60. back to the first hex token."""
  61. text = stdout or ""
  62. match = re.search(r"window id # (0x[0-9a-fA-F]+)", text) or re.search(r"(0x[0-9a-fA-F]+)", text)
  63. return int(match.group(1), 16) if match else None
  64. def _is_real_app_window(w: dict[str, Any]) -> bool:
  65. """Return False for desktop/shell helper windows that capture as empty."""
  66. title = w.get("title", "")
  67. return not any(title.startswith(p) or title.lower().startswith(p.lower()) for p in _NON_APP_WINDOW_TITLE_PREFIXES)
  68. def _parse_elements_from_tree(markdown: str) -> list[UIElement]:
  69. """Parse UIElements from get_window_state AX-tree markdown — last-resort fallback for drivers without
  70. ``structuredContent.elements``. Bounds are always ``(0, 0, 0, 0)`` (the markdown carries none), fine for
  71. element-index clicks since the driver resolves the frame.
  72. Last-resort fallback for cua-driver builds that don't carry the canonical ``structuredContent.elements``
  73. array (see ``_parse_elements_from_structured`` — Surface 2 of #47072 prefers that path).
  74. """
  75. return [
  76. # groups 3-6: value / quoted / paren / id= label (first non-None wins)
  77. UIElement(index=int(m.group(1)), role=m.group(2),
  78. label=m.group(3) or m.group(4) or m.group(5) or m.group(6) or "", bounds=(0, 0, 0, 0))
  79. for m in _ELEMENT_LINE_RE.finditer(markdown)
  80. ]
  81. def _parse_elements_from_structured(raw_elements: list[dict[str, Any]]) -> list[UIElement]:
  82. """Read the canonical ``structuredContent.elements`` array: ``element_index``, ``role``, ``label`` and, when
  83. AT-SPI / AXFrame returned usable bounds, ``frame`` ``{x, y, w, h}`` — so real pixel bounds survive (the
  84. markdown path loses them). Malformed entries are skipped.
  85. Surface 2 of NousResearch/hermes-agent#47072: read the canonical ``structuredContent.elements`` array
  86. cua-driver-rs emits on every ``get_window_state`` response (trycua/cua#1961).
  87. """
  88. elements: list[UIElement] = []
  89. for raw in raw_elements:
  90. idx = raw.get("element_index") if isinstance(raw, dict) else None
  91. if not isinstance(idx, int):
  92. continue
  93. role, label, frame, token = (raw.get(k) for k in ("role", "label", "frame", "element_token"))
  94. bounds: tuple[int, int, int, int] = (0, 0, 0, 0)
  95. with contextlib.suppress(TypeError, ValueError):
  96. if isinstance(frame, dict) and frame:
  97. bounds = tuple(int(frame.get(k, 0)) for k in ("x", "y", "w", "h")) # type: ignore[assignment]
  98. elements.append(UIElement(
  99. index=idx, role=role if isinstance(role, str) else "", label=label if isinstance(label, str) else "",
  100. bounds=bounds,
  101. # Opaque `s{snapshot_hex}:{index}` token — the driver owns parse + LRU semantics.
  102. element_token=token if isinstance(token, str) and token else None,
  103. ))
  104. return elements
  105. def _image_dimensions_from_bytes(raw: bytes) -> tuple[int, int]:
  106. """Best-effort PNG/JPEG dimension sniff; ``(0, 0)`` when unreadable or non-positive."""
  107. dims = image_dimensions_from_bytes(raw)
  108. return dims if dims and dims[0] > 0 and dims[1] > 0 else (0, 0)
  109. def _split_tree_text(full_text: str) -> tuple[str, str]:
  110. """Split get_window_state text into (summary_line, tree_markdown)."""
  111. summary, _, tree = full_text.partition("\n")
  112. return summary, tree
  113. _MODIFIER_NAMES = frozenset({"cmd", "command", "shift", "option", "alt", "ctrl", "control", "fn"})
  114. _KEY_ALIASES = {"command": "cmd", "alt": "option", "control": "ctrl"}
  115. def _parse_key_combo(keys: str) -> tuple[Optional[str], list[str]]:
  116. """Parse 'cmd+s' / 'ctrl-alt-t' into (key, modifiers); last non-modifier wins."""
  117. modifiers: list[str] = []
  118. key = None
  119. for part in (p.strip().lower() for p in re.split(r'[+\-]', keys) if p.strip()):
  120. normalized = _KEY_ALIASES.get(part, part)
  121. if normalized in _MODIFIER_NAMES:
  122. modifiers.append(normalized)
  123. else:
  124. key = part
  125. return key, modifiers
  126. def _tool_envelope(data: Any, images: list[str], structured: Any, is_error: bool,
  127. image_mime_types: Optional[list[str]] = None) -> dict[str, Any]:
  128. """The normalised tool-result dict every transport emits: ``{data, images, [image_mime_types,] structuredContent,
  129. isError}``. ``image_mime_types`` is only present when the transport can report it (MCP image parts)."""
  130. out: dict[str, Any] = {"data": data, "images": images}
  131. if image_mime_types is not None:
  132. out["image_mime_types"] = image_mime_types
  133. out["structuredContent"], out["isError"] = structured, is_error
  134. return out
  135. def _extract_tool_result(mcp_result: Any) -> dict[str, Any]:
  136. """Flatten an mcp CallToolResult into ``{data, images, image_mime_types, structuredContent, isError}``. ``data``
  137. is the joined text parts (parsed as JSON when it looks like JSON); ``image_mime_types`` is parallel to
  138. ``images`` with ``""`` where the part carried no mimeType (older drivers — callers then sniff the base64 prefix).
  139. `image_mime_types` is the explicit `mimeType` cua-driver emits on every image part as of trycua/cua#1961
  140. (Surface 7 of NousResearch/hermes-agent#47072). Each entry corresponds index-for-index with `images`; an
  141. empty string entry signals the part carried no mimeType (older cua-driver build), and the caller should
  142. fall back to base64-prefix sniffing.
  143. """
  144. data: Any = None
  145. images: list[str] = []
  146. image_mime_types: list[str] = []
  147. text_chunks: list[str] = []
  148. for part in getattr(mcp_result, "content", []) or []:
  149. ptype = getattr(part, "type", None)
  150. if ptype == "text":
  151. text_chunks.append(getattr(part, "text", "") or "")
  152. elif ptype == "image" and getattr(part, "data", None):
  153. images.append(part.data)
  154. image_mime_types.append(_mcp_field(part, "mime_type", "mimeType") or "")
  155. if text_chunks:
  156. joined = "\n".join(t for t in text_chunks if t)
  157. try:
  158. data = json.loads(joined) if joined.strip().startswith(("{", "[")) else joined
  159. except json.JSONDecodeError:
  160. data = joined
  161. return _tool_envelope(
  162. data, images, _mcp_field(mcp_result, "structured_content", "structuredContent") or None,
  163. # Identity, not truthiness: mocks/proxies synthesize truthy attributes.
  164. _mcp_field(mcp_result, "is_error", "isError", False) is True, image_mime_types)
  165. def _image_from_tool_result(out: dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
  166. """Pull ``(b64, mime_type)`` out of a flattened tool result. cua-driver delivers screenshots as an MCP ``image``
  167. part (``out["images"]``) or as ``screenshot_png_b64`` in structuredContent (newer builds, CLI transport);
  168. checking both keeps capture() robust to the driver moving it."""
  169. images = out.get("images") or []
  170. if images and images[0]:
  171. mimes = out.get("image_mime_types") or []
  172. return images[0], (mimes[0] if mimes and mimes[0] else None)
  173. structured = out.get("structuredContent") or {}
  174. b64 = structured.get("screenshot_png_b64") or structured.get("png_b64")
  175. if b64:
  176. return b64, (structured.get("screenshot_mime_type") or structured.get("mime_type") or None)
  177. return None, None
  178. def _int_or_none(value: Any) -> Optional[int]:
  179. """``int(value)`` for int/numeric-str inputs; None for bools, other types and malformed strings."""
  180. if isinstance(value, bool) or not isinstance(value, (int, str)):
  181. return None
  182. try:
  183. return int(value)
  184. except ValueError:
  185. return None
  186. def _positive_int(value: Any) -> Optional[int]:
  187. """Return a positive integer, rejecting booleans and malformed values."""
  188. parsed = _int_or_none(value)
  189. return parsed if parsed is not None and parsed > 0 else None
  190. def _is_placeholder_id(value: Any) -> bool:
  191. """True when *value* is a schema-filler id (``0`` / negative) rather than a target: some providers zero-fill
  192. every optional integer, and treating that as targeting would drop the caller's ``app=``. Non-numeric values
  193. are NOT placeholders — they still reach validation."""
  194. parsed = _int_or_none(value)
  195. return parsed is not None and parsed <= 0
  196. def _ingest_windows(raw_windows: list[dict[str, Any]]) -> list[dict[str, Any]]:
  197. """Normalise cua-driver ``list_windows`` entries, dropping unusable ones. Every downstream call needs integer
  198. ``pid`` and ``window_id``; on X11 the PID comes from the optional ``_NET_WM_PID`` property, so root/panel/popup
  199. windows report ``pid: null`` — skip those instead of aborting the enumeration. ``z_index``: higher = closer to
  200. front; Wayland's null (undefined stacking) sorts lowest so real windows stay above the desktop."""
  201. windows: list[dict[str, Any]] = []
  202. for w in raw_windows:
  203. if not isinstance(w, dict): # untrusted compatibility envelopes
  204. continue
  205. pid_int, window_id_int = _positive_int(w.get("pid")), _positive_int(w.get("window_id"))
  206. if pid_int is None or window_id_int is None:
  207. continue
  208. z_raw, app_name, title = w.get("z_index"), w.get("app_name", ""), w.get("title", "")
  209. windows.append({
  210. "app_name": app_name if isinstance(app_name, str) else "",
  211. "pid": pid_int,
  212. "window_id": window_id_int,
  213. # Only explicit False means off-screen; null (Linux 0.6.x) means unknown.
  214. "off_screen": w.get("is_on_screen") is False,
  215. "title": title if isinstance(title, str) else "",
  216. "z_index": z_raw if isinstance(z_raw, (int, float)) and not isinstance(z_raw, bool) else 0,
  217. })
  218. return windows
  219. def _windows_from_tool_result(out: dict[str, Any]) -> list[dict[str, Any]]:
  220. """Return list_windows payloads across cua-driver result shapes: structuredContent.windows, then ``windows`` /
  221. ``_legacy_windows`` in the text payload, then on the envelope itself."""
  222. candidates = ((out.get("structuredContent"), ("windows",)),
  223. (out.get("data"), ("windows", "_legacy_windows")),
  224. (out, ("windows", "_legacy_windows")))
  225. for container, keys in candidates:
  226. if isinstance(container, dict):
  227. for key in keys:
  228. value = container.get(key)
  229. if isinstance(value, list) and value:
  230. return value
  231. return []
  232. def _apps_from_windows(windows: list[dict[str, Any]]) -> list[dict[str, Any]]:
  233. apps: list[dict[str, Any]] = []
  234. seen: set[tuple[str, int]] = set()
  235. for summary in _ingest_windows(windows):
  236. name, key = summary["app_name"], (summary["app_name"], summary["pid"])
  237. if name and key not in seen:
  238. seen.add(key)
  239. apps.append({"name": name, "pid": summary["pid"]})
  240. return apps