annotate_preview_tool.py 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101
  1. """Persistent element annotations in the Hermes desktop GUI's in-app browser.
  2. Unlike ``drive_preview``'s self-retiring marks, an annotation outlines an element
  3. (or, with ``hold``, the whole visible field) until removed. Annotations bind to
  4. elements, not coordinates: they ride scrolls and vanish with their element, so
  5. navigation clears them. Same ``preview.act`` bridge as ``drive_preview`` (the
  6. renderer resolves refs and owns the overlay). ``desktop_ui`` toolset only.
  7. """
  8. import json
  9. from typing import Callable, Optional
  10. from tools.registry import registry, tool_error
  11. ACTIONS = ("add", "hold", "remove", "clear")
  12. # Renderer verbs keyed by ours; `clear` is `unpin` with no target = "all of them".
  13. WIRE = {"add": "pin", "hold": "hold", "remove": "unpin", "clear": "unpin"}
  14. def annotate_preview_tool(
  15. action: str = "add", ref: Optional[str] = None, selector: Optional[str] = None,
  16. label: Optional[str] = None, callback: Optional[Callable] = None,
  17. ) -> str:
  18. """Put one annotation up, take one down, or clear them all."""
  19. if callback is None:
  20. return tool_error("annotate_preview is only available in the Hermes desktop app.")
  21. verb = (action or "add").strip().lower()
  22. if verb not in ACTIONS:
  23. return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
  24. if verb in ("add", "remove") and not (ref or selector):
  25. return tool_error(f"{verb} needs a ref from drive_preview action='elements' (e.g. 'btn-sign-in') or a CSS selector.")
  26. targeted = verb not in ("clear", "hold")
  27. fields = (("action", WIRE[verb]), ("ref", ref if targeted else None), ("selector", selector if targeted else None), ("text", label))
  28. payload = {name: val for name, val in fields if val is not None}
  29. try:
  30. raw = callback(payload)
  31. except Exception as exc:
  32. return tool_error(f"Failed to annotate the in-app browser: {exc}")
  33. if not raw:
  34. return tool_error(
  35. "No GUI window answered with a page: no preview tab is open. "
  36. "Open a page with open_preview first. If the pane IS open, the desktop app "
  37. "may be older than this backend — its bridge-unavailable error names that case.")
  38. try:
  39. return json.dumps(json.loads(raw), ensure_ascii=False)
  40. except (TypeError, ValueError):
  41. return json.dumps({"text": str(raw)}, ensure_ascii=False)
  42. ANNOTATE_PREVIEW_SCHEMA = {
  43. "name": "annotate_preview",
  44. "description": (
  45. "Highlight preview-pane elements with lasting marks (drive_preview's fade). "
  46. "Point at findings, flag what you're about to change, keep your place. Use the refs "
  47. "from drive_preview action='elements'. add: outline one element "
  48. "(optional short label — a word or two, drawn on the page). hold: "
  49. "freeze the whole visible field, every element outlined and named. "
  50. "remove/clear: take one/all down. Marks follow their element on "
  51. "scroll; navigation clears them."
  52. ),
  53. "parameters": {
  54. "type": "object",
  55. "properties": {
  56. "action": {
  57. "type": "string",
  58. "enum": list(ACTIONS),
  59. "description": "Defaults to 'add'.",
  60. },
  61. "ref": {
  62. "type": "string",
  63. "description": "Ref from drive_preview elements.",
  64. },
  65. "selector": {
  66. "type": "string",
  67. "description": "CSS selector fallback. Prefer ref.",
  68. },
  69. "label": {
  70. "type": "string",
  71. "description": "Optional caption, e.g. 'cheapest'.",
  72. },
  73. },
  74. "required": [],
  75. },
  76. }
  77. registry.register(
  78. name="annotate_preview",
  79. toolset="desktop_ui",
  80. schema=ANNOTATE_PREVIEW_SCHEMA,
  81. handler=lambda args, **kw: annotate_preview_tool(
  82. action=args.get("action", "add"),
  83. ref=args.get("ref"),
  84. selector=args.get("selector"),
  85. label=args.get("label"),
  86. callback=kw.get("callback"),
  87. ),
  88. emoji="🔖",
  89. )