_watermark.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. """Shared watermark helper used by the three watcher scripts.
  2. A watermark is just a JSON file that records the IDs we've seen on previous
  3. runs, so the next run only emits items we haven't seen before.
  4. Contract:
  5. - First run: record all IDs from the fetched batch, emit nothing.
  6. - Subsequent runs: emit items whose ID isn't in the stored set.
  7. - Bounded: keep at most `max_seen` IDs (default 500).
  8. - Atomic: write to a .tmp file and rename, so a crashed script can't
  9. leave a half-written state file that permanently breaks dedup.
  10. Import and use from any custom watcher script:
  11. from _watermark import Watermark
  12. wm = Watermark.load("my-feed-name")
  13. new_items = wm.filter_new(fetched_items, id_key="id")
  14. wm.save()
  15. """
  16. from __future__ import annotations
  17. import json
  18. import os
  19. from pathlib import Path
  20. from typing import Any, Dict, Iterable, List, Optional
  21. def _state_dir() -> Path:
  22. """Where watermark files live — respects WATCHER_STATE_DIR override."""
  23. override = os.environ.get("WATCHER_STATE_DIR")
  24. if override:
  25. return Path(override)
  26. # Default: $HERMES_HOME/watcher-state/, falling back to ~/.hermes/watcher-state/.
  27. hermes_home = os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes")
  28. return Path(hermes_home) / "watcher-state"
  29. class Watermark:
  30. """Per-watcher state. Persisted to <state_dir>/<name>.json."""
  31. def __init__(self, name: str, *, max_seen: int = 500) -> None:
  32. if not name or not name.replace("-", "").replace("_", "").isalnum():
  33. raise ValueError(
  34. f"watermark name must be alphanumeric + '-'/'_' (got {name!r})"
  35. )
  36. self.name = name
  37. self.max_seen = max_seen
  38. self._path = _state_dir() / f"{name}.json"
  39. self._data: dict[str, Any] = {"seen_ids": [], "first_run": True}
  40. @classmethod
  41. def load(cls, name: str, *, max_seen: int = 500) -> Watermark:
  42. wm = cls(name, max_seen=max_seen)
  43. if wm._path.exists():
  44. try:
  45. wm._data = json.loads(wm._path.read_text(encoding="utf-8"))
  46. wm._data.setdefault("seen_ids", [])
  47. wm._data["first_run"] = False
  48. except (OSError, json.JSONDecodeError):
  49. # Corrupt state file — treat as a first run but don't crash.
  50. wm._data = {"seen_ids": [], "first_run": True}
  51. return wm
  52. @property
  53. def is_first_run(self) -> bool:
  54. return bool(self._data.get("first_run", True))
  55. @property
  56. def seen(self) -> list[str]:
  57. return list(self._data.get("seen_ids", []))
  58. def filter_new(
  59. self, items: Iterable[dict[str, Any]], *, id_key: str = "id"
  60. ) -> list[dict[str, Any]]:
  61. """Return items whose id isn't in the stored set.
  62. Side effect: updates the in-memory seen set with every id in the
  63. batch (so save() persists the full new watermark). On first run,
  64. records every id but returns an empty list (baseline, no replay).
  65. """
  66. existing = set(str(x) for x in self._data.get("seen_ids", []))
  67. was_first_run = self.is_first_run
  68. new_items: list[dict[str, Any]] = []
  69. batch_ids: list[str] = []
  70. for item in items:
  71. ident = item.get(id_key)
  72. if ident is None:
  73. continue
  74. ident_str = str(ident)
  75. batch_ids.append(ident_str)
  76. if ident_str in existing:
  77. continue
  78. if was_first_run:
  79. continue # record but don't emit
  80. new_items.append(item)
  81. combined = list(existing) + [i for i in batch_ids if i not in existing]
  82. if len(combined) > self.max_seen:
  83. combined = combined[-self.max_seen:]
  84. self._data["seen_ids"] = combined
  85. self._data["first_run"] = False
  86. return new_items
  87. def save(self) -> None:
  88. self._path.parent.mkdir(parents=True, exist_ok=True)
  89. tmp = self._path.with_suffix(".tmp")
  90. tmp.write_text(
  91. json.dumps(self._data, indent=2, sort_keys=True),
  92. encoding="utf-8",
  93. )
  94. os.replace(tmp, self._path)
  95. def format_items_as_markdown(
  96. items: list[dict[str, Any]],
  97. *,
  98. title_key: str = "title",
  99. url_key: str = "url",
  100. body_key: Optional[str] = None,
  101. max_body_chars: int = 500,
  102. ) -> str:
  103. """Render a list of items as Markdown for cron delivery.
  104. One heading per item + its URL + optional snippet of body. Output is
  105. empty string when items is empty — cron will then treat stdout as
  106. silent and skip delivery (existing behavior).
  107. """
  108. if not items:
  109. return ""
  110. lines: list[str] = []
  111. for item in items:
  112. title = (item.get(title_key) or "(no title)").strip()
  113. url = (item.get(url_key) or "").strip()
  114. lines.append(f"## {title}")
  115. if url:
  116. lines.append(url)
  117. if body_key:
  118. body = (item.get(body_key) or "").strip()
  119. if body:
  120. if len(body) > max_body_chars:
  121. body = body[:max_body_chars].rstrip() + "…"
  122. lines.append("")
  123. lines.append(body)
  124. lines.append("")
  125. return "\n".join(lines).rstrip() + "\n"