drive_preview_tool.py 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152
  1. #!/usr/bin/env python3
  2. """Interact with the in-app browser / preview pane in the Hermes desktop GUI (click, type, scroll).
  3. Elements are addressed by legible refs from ``action="elements"`` (``btn-sign-in``); a ref survives
  4. re-renders and only a navigation retires it, so the renderer answers with a *delta* instead of
  5. re-sending the inventory. Round-trips through the gateway's blocking-prompt bridge
  6. (``preview.act.request`` / ``preview.act.respond``); this module is schema + a thin dispatcher over
  7. the platform-injected callback. ``desktop_ui`` toolset: desktop-sourced sessions only.
  8. """
  9. from typing import Callable, Optional
  10. from tools.desktop_ui import passthrough_json
  11. from tools.registry import registry, tool_error
  12. ACTIONS = ("elements", "click", "hover", "type", "scroll", "press", "strobe", "back", "forward", "reload")
  13. SCROLL_TO = ("top", "bottom")
  14. # Verbs that need something to act on — a ref from the last inventory, or a
  15. # raw CSS selector. `scroll` is deliberately absent: bare, it scrolls the page.
  16. NEEDS_TARGET = ("click", "hover", "type", "press")
  17. def drive_preview_tool(
  18. action: str = "", ref: Optional[str] = None, selector: Optional[str] = None, text: Optional[str] = None,
  19. key: Optional[str] = None, submit: Optional[bool] = None, amount: Optional[int] = None,
  20. to: Optional[str] = None, limit: Optional[int] = None, full: Optional[bool] = None,
  21. allow_shortcut: Optional[bool] = None,
  22. callback: Optional[Callable] = None) -> str:
  23. """Dispatch one interaction to the desktop renderer and return its outcome."""
  24. if callback is None:
  25. return tool_error("drive_preview is only available in the Hermes desktop app.")
  26. verb = (action or "").strip().lower()
  27. if verb not in ACTIONS:
  28. return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
  29. if verb in NEEDS_TARGET and not (ref or selector):
  30. return tool_error(f"{verb} needs a ref from action='elements' (e.g. 'btn-sign-in') or a CSS selector.")
  31. if verb == "type" and text is None:
  32. return tool_error("type needs the text to enter.")
  33. if verb == "press" and not key:
  34. return tool_error("press needs a key, e.g. 'Enter' or 'Escape'.")
  35. if to is not None and to not in SCROLL_TO:
  36. return tool_error(f"to must be one of: {', '.join(SCROLL_TO)}.")
  37. try:
  38. fields = (
  39. ("action", verb), ("ref", ref), ("selector", selector), ("text", text), ("key", key),
  40. ("submit", submit), ("full", full), ("to", to), ("allow_shortcut", allow_shortcut),
  41. ("amount", None if amount is None else int(amount)), ("max", None if limit is None else int(limit)),
  42. )
  43. except (TypeError, ValueError):
  44. return tool_error("amount and max must be integers.")
  45. try:
  46. raw = callback({name: val for name, val in fields if val is not None})
  47. except Exception as exc:
  48. return tool_error(f"Failed to act on the in-app browser: {exc}")
  49. if not raw:
  50. return tool_error(
  51. "No GUI window answered with a page: no preview tab is open. "
  52. "Open a page with open_preview first. If the pane IS open, the desktop app "
  53. "may be older than this backend — its bridge-unavailable error names that case.")
  54. return passthrough_json(raw)
  55. ACT_PREVIEW_SCHEMA = {
  56. "name": "drive_preview",
  57. # Response-shape teaching kept only where skipping it wastes calls (delta
  58. # semantics, rebound refs, strobe's burst): a model that doesn't know them
  59. # re-reads pages or loops strobe.
  60. # See #95681.
  61. "description": (
  62. "Use the desktop preview-pane page: log in, fill forms, click through flows. "
  63. "It is the page `desktop_preview` opens. ALWAYS "
  64. "start with action='elements' — it inventories clickable/typable "
  65. "things as refs ('btn-sign-in') with role/label/value; act by ref, "
  66. "not guessed selectors. Refs survive re-renders and only die on "
  67. "navigation (you'll be told they're stale — call elements again). "
  68. "After the first full inventory, actions answer with a DELTA: "
  69. "'added' in full, 'changed' as ref + moved fields, 'removed'/"
  70. "'rebound' as ref lists ('rebound' needs nothing from you — the ref "
  71. "already follows the rebuilt element). Anything unmentioned is "
  72. "unchanged; do not re-read to check. Input is real (pointer travels, "
  73. "hover menus open). Actions: elements, click, hover (park the "
  74. "pointer — opens dropdowns before clicking in), type (submit=true "
  75. "also presses Enter), scroll, press, strobe (visual flourish only — "
  76. "one call runs a multi-second burst; never loop it), back/forward/"
  77. "reload. Moves draw live and fade; annotate_preview leaves a lasting "
  78. "mark. A printable press on body/html is refused unless allow_shortcut "
  79. "is true. Page text only: desktop_preview action=read. Separate automated "
  80. "browser: browser_* tools."
  81. ),
  82. "parameters": {
  83. "type": "object",
  84. "properties": {
  85. "action": {
  86. "type": "string",
  87. "enum": list(ACTIONS),
  88. "description": "Start with 'elements'.",
  89. },
  90. "ref": {
  91. "type": "string",
  92. "description": "Element ref from an earlier elements call.",
  93. },
  94. "selector": {
  95. "type": "string",
  96. "description": "CSS selector fallback. Prefer ref.",
  97. },
  98. "text": {"type": "string", "description": "type: the text."},
  99. "submit": {
  100. "type": "boolean",
  101. "description": "type: press Enter + submit the form after.",
  102. },
  103. "key": {
  104. "type": "string",
  105. "description": "press: key name ('Enter', 'Escape', 'ArrowDown').",
  106. },
  107. "allow_shortcut": {
  108. "type": "boolean",
  109. "description": "press: allow a printable key on body/html. Off by default.",
  110. },
  111. "amount": {
  112. "type": "integer",
  113. "description": "scroll: pixels (negative = up; default ~one screen).",
  114. },
  115. "to": {
  116. "type": "string",
  117. "enum": list(SCROLL_TO),
  118. "description": "scroll: jump to top/bottom instead.",
  119. },
  120. "max": {
  121. "type": "integer",
  122. "description": "elements: cap the inventory.",
  123. },
  124. "full": {
  125. "type": "boolean",
  126. "description": "elements: full re-read instead of a delta. Rarely needed.",
  127. },
  128. },
  129. "required": ["action"],
  130. },
  131. }
  132. registry.register(
  133. name="drive_preview",
  134. toolset="desktop_ui",
  135. schema=ACT_PREVIEW_SCHEMA,
  136. handler=lambda args, **kw: drive_preview_tool(
  137. action=args.get("action", ""), limit=args.get("max"), callback=kw.get("callback"),
  138. **{k: args.get(k) for k in (
  139. "ref", "selector", "text", "key", "submit", "amount", "to", "full", "allow_shortcut")},
  140. ),
  141. emoji="🖱️")