feed.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. #!/usr/bin/env python3
  2. """Read RSS / Atom / JSON Feed sources and discover feeds behind a page URL.
  3. Standard library only, so it runs in any Hermes environment without an install
  4. step. Output is JSON (``--json``) or a compact text listing.
  5. python3 feed.py read https://example.com/feed.xml [--limit N] [--since 2026-09-01]
  6. python3 feed.py discover https://example.com/
  7. python3 feed.py read https://example.com/ -> discovers, then reads the first feed
  8. """
  9. from __future__ import annotations
  10. import argparse
  11. import html
  12. import json
  13. import re
  14. import sys
  15. import urllib.error
  16. import urllib.parse
  17. import urllib.request
  18. import xml.etree.ElementTree as ET
  19. from datetime import datetime, timezone, UTC
  20. from email.utils import parsedate_to_datetime
  21. USER_AGENT = "hermes-agent/1.0 (rss-feeds skill; +https://github.com/NousResearch/hermes-agent)"
  22. TIMEOUT = 20
  23. NS = {
  24. "atom": "http://www.w3.org/2005/Atom",
  25. "dc": "http://purl.org/dc/elements/1.1/",
  26. "content": "http://purl.org/rss/1.0/modules/content/",
  27. "media": "http://search.yahoo.com/mrss/",
  28. }
  29. FEED_TYPES = ("application/rss+xml", "application/atom+xml", "application/feed+json", "application/json")
  30. COMMON_FEED_PATHS = ("/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed.json", "/blog/feed", "/blog/rss.xml")
  31. _TAG_RE = re.compile(r"<[^>]+>")
  32. _WS_RE = re.compile(r"\s+")
  33. def fetch(url: str) -> tuple[bytes, str]:
  34. req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "*/*"})
  35. with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
  36. return resp.read(), resp.headers.get("Content-Type", "")
  37. def strip_html(text: str | None) -> str:
  38. if not text:
  39. return ""
  40. return _WS_RE.sub(" ", html.unescape(_TAG_RE.sub(" ", text))).strip()
  41. def parse_date(value: str | None) -> str | None:
  42. """Normalise RFC 822 (RSS) and ISO 8601 (Atom/JSON Feed) dates to UTC ISO."""
  43. if not value:
  44. return None
  45. value = value.strip()
  46. try:
  47. dt = parsedate_to_datetime(value)
  48. except (TypeError, ValueError):
  49. try:
  50. dt = datetime.fromisoformat(value)
  51. except ValueError:
  52. return value
  53. if dt.tzinfo is None:
  54. dt = dt.replace(tzinfo=UTC)
  55. return dt.astimezone(UTC).isoformat()
  56. def _text(el, *paths) -> str | None:
  57. for p in paths:
  58. found = el.find(p, NS)
  59. if found is not None and (found.text or "").strip():
  60. return found.text
  61. return None
  62. def _atom_link(entry) -> str | None:
  63. alternate = None
  64. for link in entry.findall("atom:link", NS) + entry.findall("link"):
  65. href = link.get("href")
  66. if not href:
  67. continue
  68. rel = link.get("rel", "alternate")
  69. if rel == "alternate":
  70. return href
  71. alternate = alternate or href
  72. return alternate
  73. def parse_xml(data: bytes) -> dict:
  74. root = ET.fromstring(data)
  75. tag = root.tag.rsplit("}", 1)[-1].lower()
  76. if tag == "feed": # Atom
  77. title = strip_html(_text(root, "atom:title"))
  78. entries = []
  79. for e in root.findall("atom:entry", NS):
  80. entries.append({
  81. "title": strip_html(_text(e, "atom:title")),
  82. "link": _atom_link(e),
  83. "published": parse_date(_text(e, "atom:published", "atom:updated")),
  84. "author": strip_html(_text(e, "atom:author/atom:name", "dc:creator")),
  85. "summary": strip_html(_text(e, "atom:summary", "atom:content"))[:2000],
  86. })
  87. return {"format": "atom", "title": title, "entries": entries}
  88. channel = root.find("channel") if tag == "rss" else root # RSS 2.0 vs RDF/RSS 1.0
  89. if channel is None:
  90. raise ValueError(f"unrecognised XML root <{tag}>")
  91. entries = []
  92. for item in channel.iter("item") if tag == "rss" else root.iter("{http://purl.org/rss/1.0/}item"):
  93. entries.append({
  94. "title": strip_html(_text(item, "title", "{http://purl.org/rss/1.0/}title")),
  95. "link": (_text(item, "link", "{http://purl.org/rss/1.0/}link") or "").strip() or None,
  96. "published": parse_date(_text(item, "pubDate", "dc:date")),
  97. "author": strip_html(_text(item, "dc:creator", "author")),
  98. "summary": strip_html(_text(item, "content:encoded", "description", "{http://purl.org/rss/1.0/}description"))[:2000],
  99. })
  100. return {"format": "rss", "title": strip_html(_text(channel, "title", "{http://purl.org/rss/1.0/}title")), "entries": entries}
  101. def parse_json_feed(data: bytes) -> dict:
  102. doc = json.loads(data)
  103. entries = []
  104. for item in doc.get("items", []):
  105. authors = item.get("authors") or ([item["author"]] if item.get("author") else [])
  106. entries.append({
  107. "title": strip_html(item.get("title")),
  108. "link": item.get("url") or item.get("external_url"),
  109. "published": parse_date(item.get("date_published") or item.get("date_modified")),
  110. "author": ", ".join(a.get("name", "") for a in authors if isinstance(a, dict)) or None,
  111. "summary": strip_html(item.get("summary") or item.get("content_text") or item.get("content_html"))[:2000],
  112. })
  113. return {"format": "jsonfeed", "title": strip_html(doc.get("title")), "entries": entries}
  114. def parse_feed(data: bytes, content_type: str = "") -> dict:
  115. head = data.lstrip()[:1]
  116. if head == b"{" or "json" in content_type:
  117. return parse_json_feed(data)
  118. return parse_xml(data)
  119. def discover(page_url: str, page_html: bytes | None = None) -> list[str]:
  120. """Return candidate feed URLs for a page: <link rel=alternate> first, then well-known paths."""
  121. if page_html is None:
  122. page_html, _ = fetch(page_url)
  123. text = page_html.decode("utf-8", "replace")
  124. found: list[str] = []
  125. for m in re.finditer(r"<link\b[^>]*>", text, re.IGNORECASE):
  126. tag = m.group(0)
  127. type_m = re.search(r"""type\s*=\s*["']([^"']+)""", tag, re.IGNORECASE)
  128. href_m = re.search(r"""href\s*=\s*["']([^"']+)""", tag, re.IGNORECASE)
  129. rel_m = re.search(r"""rel\s*=\s*["']([^"']+)""", tag, re.IGNORECASE)
  130. if not href_m or not type_m or type_m.group(1).lower() not in FEED_TYPES:
  131. continue
  132. if rel_m and "alternate" not in rel_m.group(1).lower():
  133. continue
  134. url = urllib.parse.urljoin(page_url, html.unescape(href_m.group(1)))
  135. if url not in found:
  136. found.append(url)
  137. if found:
  138. return found
  139. parsed = urllib.parse.urlsplit(page_url)
  140. base = f"{parsed.scheme}://{parsed.netloc}"
  141. return [base + p for p in COMMON_FEED_PATHS]
  142. def looks_like_feed(data: bytes, content_type: str) -> bool:
  143. head = data.lstrip()[:300].lower()
  144. return head.startswith(b"{") and b"items" in data[:2000] or b"<rss" in head or b"<feed" in head or b"<rdf" in head
  145. def read(url: str) -> dict:
  146. data, ctype = fetch(url)
  147. if looks_like_feed(data, ctype):
  148. feed = parse_feed(data, ctype)
  149. feed["url"] = url
  150. return feed
  151. candidates = discover(url, data)
  152. errors = []
  153. for cand in candidates:
  154. try:
  155. cdata, cctype = fetch(cand)
  156. except (urllib.error.URLError, OSError) as exc:
  157. errors.append(f"{cand}: {exc}")
  158. continue
  159. if looks_like_feed(cdata, cctype):
  160. feed = parse_feed(cdata, cctype)
  161. feed["url"] = cand
  162. feed["discovered_from"] = url
  163. return feed
  164. raise SystemExit(f"no feed found at {url}; tried {len(candidates)} candidates\n" + "\n".join(errors))
  165. def filter_entries(entries: list[dict], limit: int, since: str | None) -> list[dict]:
  166. if since:
  167. cutoff = datetime.fromisoformat(since).replace(tzinfo=UTC) if "T" not in since else datetime.fromisoformat(since)
  168. if cutoff.tzinfo is None:
  169. cutoff = cutoff.replace(tzinfo=UTC)
  170. entries = [e for e in entries if e["published"] and datetime.fromisoformat(e["published"]) >= cutoff]
  171. entries.sort(key=lambda e: e["published"] or "", reverse=True)
  172. return entries[:limit]
  173. def render_text(feed: dict) -> str:
  174. lines = [f"{feed.get('title') or '(untitled feed)'} [{feed['format']}] {feed['url']}"]
  175. for e in feed["entries"]:
  176. when = (e["published"] or "")[:10]
  177. by = f" — {e['author']}" if e.get("author") else ""
  178. lines.append(f"- {when} {e['title'] or '(no title)'}{by}\n {e['link'] or ''}")
  179. if e.get("summary"):
  180. lines.append(f" {e['summary'][:300]}")
  181. return "\n".join(lines)
  182. def main(argv: list[str] | None = None) -> int:
  183. ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  184. sub = ap.add_subparsers(dest="cmd", required=True)
  185. r = sub.add_parser("read", help="read a feed (or discover one behind a page URL)")
  186. r.add_argument("url")
  187. r.add_argument("--limit", type=int, default=20)
  188. r.add_argument("--since", help="ISO date/datetime; drop older entries")
  189. r.add_argument("--json", action="store_true")
  190. d = sub.add_parser("discover", help="list feed URLs advertised by a page")
  191. d.add_argument("url")
  192. d.add_argument("--json", action="store_true")
  193. args = ap.parse_args(argv)
  194. try:
  195. if args.cmd == "discover":
  196. urls = discover(args.url)
  197. print(json.dumps(urls, indent=2) if args.json else "\n".join(urls))
  198. return 0
  199. feed = read(args.url)
  200. feed["entries"] = filter_entries(feed["entries"], args.limit, args.since)
  201. print(json.dumps(feed, indent=2, ensure_ascii=False) if args.json else render_text(feed))
  202. return 0
  203. except urllib.error.HTTPError as exc:
  204. print(f"HTTP {exc.code} for {exc.url}", file=sys.stderr)
  205. return 2
  206. except (urllib.error.URLError, ET.ParseError, ValueError, json.JSONDecodeError) as exc:
  207. print(f"error: {exc}", file=sys.stderr)
  208. return 2
  209. if __name__ == "__main__":
  210. sys.exit(main())