tour_tool.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164
  1. """Guided tour (highlight + narrate UI elements) in the Hermes desktop GUI: ``start`` with no
  2. steps runs the app's own built-in tour (``preset`` quick or full); for a custom one the agent
  3. discovers targets (``action="targets"``), then highlights one step at a time (``show``) or hands
  4. over a step list the user pages (``start`` + ``steps``). Round-trips through the gateway
  5. blocking-prompt bridge (``tour.request``/``tour.respond``) so the agent learns whether the
  6. selector matched. Registered in ``desktop_ui`` and hidden from the model when tours are off: a
  7. tour covers the whole screen, so "off" must mean the model is never told the tool exists rather
  8. than offered a call that fails."""
  9. import json
  10. from typing import Callable, Optional
  11. from tools import desktop_ui
  12. from tools.registry import registry, tool_error
  13. from tools.tour_presets import TourPreset
  14. ACTIONS = ("targets", "show", "start", "next", "prev", "stop")
  15. SURFACES = ("app", "preview")
  16. SIDES = ("top", "right", "bottom", "left")
  17. PRESETS = tuple(TourPreset)
  18. def tour_tool(action: str = "", surface: Optional[str] = None, selector: Optional[str] = None,
  19. title: Optional[str] = None, text: Optional[str] = None, side: Optional[str] = None,
  20. steps: Optional[list] = None, step_index: Optional[int] = None,
  21. preset: Optional[str] = None, callback: Optional[Callable] = None) -> str:
  22. """Dispatch one tour action to the desktop renderer and return its outcome."""
  23. if callback is None:
  24. return tool_error("tour is only available in the Hermes desktop app.")
  25. verb = (action or "").strip().lower()
  26. if verb not in ACTIONS:
  27. return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
  28. where = (surface or "app").strip().lower()
  29. if where not in SURFACES:
  30. return tool_error(f"surface must be one of: {', '.join(SURFACES)}.")
  31. if side is not None and side not in SIDES:
  32. return tool_error(f"side must be one of: {', '.join(SIDES)}.")
  33. if preset is not None and preset not in PRESETS:
  34. return tool_error(f"preset must be one of: {', '.join(PRESETS)}.")
  35. if preset is not None and steps is not None:
  36. return tool_error("preset picks a built-in tour; pass steps or preset, not both.")
  37. # Every highlighted moment needs something to point at or something to say.
  38. if verb == "show" and not (selector or title or text):
  39. return tool_error("show needs a selector (and/or title/text for the popover).")
  40. # start without steps is the app's built-in tour, which only exists on the app surface.
  41. if verb == "start" and steps is None and where != "app":
  42. return tool_error("The built-in tour runs on surface='app'; pass steps for a preview tour.")
  43. if verb == "start" and steps is not None:
  44. if not isinstance(steps, list) or not steps:
  45. return tool_error("start needs a non-empty steps array, or no steps for the built-in tour.")
  46. for i, step in enumerate(steps):
  47. if not isinstance(step, dict):
  48. return tool_error(f"steps[{i}] must be an object.")
  49. if not (step.get("selector") or step.get("title") or step.get("text")):
  50. return tool_error(f"steps[{i}] needs a selector and/or title/text.")
  51. fields = {"action": verb, "surface": where, "selector": selector, "title": title,
  52. "text": text, "side": side, "steps": steps, "step_index": step_index, "preset": preset}
  53. try:
  54. raw = callback({key: val for key, val in fields.items() if val is not None})
  55. except Exception as exc:
  56. return tool_error(f"Tour action failed: {exc}")
  57. if not raw:
  58. return tool_error("The tour request timed out, or no GUI window answered. "
  59. "For surface='preview' open a page in the preview pane first.")
  60. # The renderer answers with a JSON object; pass it through, else wrap it.
  61. try:
  62. return json.dumps(json.loads(raw), ensure_ascii=False)
  63. except (TypeError, ValueError):
  64. return json.dumps({"text": str(raw)}, ensure_ascii=False)
  65. _STEP_SCHEMA = {
  66. "type": "object",
  67. "properties": {
  68. "selector": {
  69. "type": "string",
  70. "description": "Element to highlight; omit = centered narration.",
  71. },
  72. "title": {"type": "string", "description": "Popover title."},
  73. "text": {"type": "string", "description": "Popover body."},
  74. "side": {
  75. "type": "string",
  76. "enum": list(SIDES),
  77. "description": "Popover side; omit to auto-place.",
  78. },
  79. },
  80. }
  81. TOUR_SCHEMA = {
  82. "name": "gui_tour",
  83. # Description keeps the targets-first flow + stable-selector preference:
  84. # without them the model guesses selectors on re-rendering UI.
  85. # See #95681.
  86. "description": (
  87. "Run a guided desktop GUI tour: dim the screen, highlight elements, add popovers. Surfaces: 'app' (Hermes itself) "
  88. "or 'preview' (the page in the preview pane). A general look around "
  89. "the app is ONE call: action='start' with no steps runs the app's "
  90. "built-in tour (preset 'quick' or 'full', default full). For a "
  91. "custom tour, call action='targets' first — prefer targets marked "
  92. "stable:true (their selectors survive re-renders); re-scan if one "
  93. "stops matching. Then narrate with action='show' (one highlight per "
  94. "call, replaces the last — pair each with a chat message) or hand "
  95. "over with action='start' + steps (user gets Next/Prev; "
  96. "'next'/'prev' also page it). 'stop' clears. Use for "
  97. "how-does-X-work / where-is-Y walkthroughs."
  98. ),
  99. "parameters": {
  100. "type": "object",
  101. "properties": {
  102. "action": {
  103. "type": "string",
  104. "enum": list(ACTIONS),
  105. "description": "start alone runs the built-in tour; targets first for custom steps.",
  106. },
  107. "surface": {
  108. "type": "string",
  109. "enum": list(SURFACES),
  110. "description": "'app' (default) or 'preview'.",
  111. },
  112. "selector": {
  113. "type": "string",
  114. "description": "show: selector from targets (prefer stable). Omit = centered narration.",
  115. },
  116. "title": {"type": "string", "description": "show: popover title."},
  117. "text": {"type": "string", "description": "show: popover body."},
  118. "side": {
  119. "type": "string",
  120. "enum": list(SIDES),
  121. "description": "show: popover side; omit to auto-place.",
  122. },
  123. "steps": {
  124. "type": "array",
  125. "items": _STEP_SCHEMA,
  126. "description": "start: ordered steps; omit for the built-in tour.",
  127. },
  128. "preset": {
  129. "type": "string",
  130. "enum": list(PRESETS),
  131. "description": "start without steps: 'quick' (four essentials) or 'full'.",
  132. },
  133. "step_index": {
  134. "type": "integer",
  135. "description": "start: 0-indexed first step.",
  136. },
  137. },
  138. "required": ["action"],
  139. },
  140. }
  141. def check_tours_enabled() -> bool:
  142. """The user's Settings → Appearance switch. On unless they turned it off."""
  143. return desktop_ui.user_enabled("in_app_tours", default=True)
  144. registry.register(
  145. name="gui_tour", toolset="desktop_ui", schema=TOUR_SCHEMA, check_fn=check_tours_enabled,
  146. handler=lambda args, **kw: tour_tool(
  147. action=args.get("action", ""), callback=kw.get("callback"),
  148. **{k: args.get(k) for k in ("surface", "selector", "title", "text", "side", "steps",
  149. "step_index", "preset")}),
  150. emoji="🧭")