watch_http_json.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. #!/usr/bin/env python3
  2. """Watch any JSON endpoint that returns a list of objects; dedup by ID field.
  3. Usage (via cron with --no-agent):
  4. hermes cron create api-events \\
  5. --schedule "*/1 * * * *" --no-agent \\
  6. --script "$HERMES_HOME/skills/devops/watchers/scripts/watch_http_json.py" \\
  7. --script-args "--name api --url https://api.example.com/events \\
  8. --id-field event_id --items-path data.events"
  9. The response can be:
  10. - a top-level JSON list (default), or
  11. - a JSON object with a dotted ``--items-path`` pointing to the list.
  12. Each item is deduped by ``--id-field`` (default "id").
  13. Optional ``--header KEY:VALUE`` flags pass HTTP headers (repeatable).
  14. """
  15. from __future__ import annotations
  16. import argparse
  17. import json
  18. import sys
  19. import urllib.error
  20. import urllib.request
  21. from pathlib import Path
  22. sys.path.insert(0, str(Path(__file__).parent))
  23. from _watermark import Watermark, format_items_as_markdown # type: ignore
  24. def _dig(obj, path: str):
  25. """Dotted-path lookup: _dig({'a':{'b':[1,2]}}, 'a.b') → [1,2]."""
  26. if not path:
  27. return obj
  28. cur = obj
  29. for part in path.split("."):
  30. if isinstance(cur, dict) and part in cur:
  31. cur = cur[part]
  32. else:
  33. return None
  34. return cur
  35. def _parse_header(s: str):
  36. if ":" not in s:
  37. raise argparse.ArgumentTypeError(
  38. f"--header expects 'KEY: VALUE' (got {s!r})"
  39. )
  40. k, v = s.split(":", 1)
  41. return (k.strip(), v.strip())
  42. def main() -> int:
  43. p = argparse.ArgumentParser(description="Poll a JSON endpoint.")
  44. p.add_argument("--name", required=True, help="Watcher name (used for state file)")
  45. p.add_argument("--url", required=True, help="JSON endpoint URL")
  46. p.add_argument("--id-field", default="id",
  47. help="Field used to dedup items (default: 'id')")
  48. p.add_argument("--items-path", default="",
  49. help="Dotted path to the list inside the JSON response (e.g. 'data.events')")
  50. p.add_argument("--title-field", default="title",
  51. help="Field used as the item title in the rendered output (default: 'title')")
  52. p.add_argument("--url-field", default="url",
  53. help="Field used as the item URL in the rendered output (default: 'url')")
  54. p.add_argument("--body-field", default="",
  55. help="Optional body field to include as a snippet under each item")
  56. p.add_argument("--max", type=int, default=20,
  57. help="Max new items to emit per tick (default: 20)")
  58. p.add_argument("--header", action="append", type=_parse_header, default=[],
  59. metavar="KEY: VALUE",
  60. help="HTTP header (repeatable)")
  61. p.add_argument("--timeout", type=float, default=20.0,
  62. help="HTTP timeout in seconds (default: 20)")
  63. args = p.parse_args()
  64. req = urllib.request.Request(args.url, headers={"User-Agent": "Hermes-Watcher/1.0"})
  65. for k, v in args.header:
  66. req.add_header(k, v)
  67. try:
  68. with urllib.request.urlopen(req, timeout=args.timeout) as resp:
  69. raw = resp.read()
  70. except urllib.error.HTTPError as e:
  71. print(f"watch_http_json: HTTP {e.code} from {args.url}", file=sys.stderr)
  72. return 2
  73. except (urllib.error.URLError, TimeoutError, OSError) as e:
  74. print(f"watch_http_json: network error: {e}", file=sys.stderr)
  75. return 2
  76. try:
  77. data = json.loads(raw.decode("utf-8"))
  78. except (UnicodeDecodeError, json.JSONDecodeError) as e:
  79. print(f"watch_http_json: response is not valid JSON: {e}", file=sys.stderr)
  80. return 2
  81. items = _dig(data, args.items_path) if args.items_path else data
  82. if not isinstance(items, list):
  83. print(
  84. f"watch_http_json: items_path={args.items_path!r} did not resolve to a list "
  85. f"(got {type(items).__name__})",
  86. file=sys.stderr,
  87. )
  88. return 2
  89. # Keep only dicts — skip any bare strings / numbers so filter_new doesn't crash.
  90. items = [i for i in items if isinstance(i, dict)]
  91. wm = Watermark.load(args.name)
  92. new_items = wm.filter_new(items, id_key=args.id_field)
  93. wm.save()
  94. if args.max > 0:
  95. new_items = new_items[: args.max]
  96. body_key = args.body_field or None
  97. output = format_items_as_markdown(
  98. new_items,
  99. title_key=args.title_field,
  100. url_key=args.url_field,
  101. body_key=body_key,
  102. )
  103. if output:
  104. sys.stdout.write(output)
  105. return 0
  106. if __name__ == "__main__":
  107. sys.exit(main())