delivery.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Delivery routing for cron job outputs and agent responses, by target: explicit ("telegram:123456789"),
  2. platform home channel ("telegram"), origin (back to where the job was created), or local (files)."""
  3. import logging
  4. import re
  5. from pathlib import Path
  6. from datetime import datetime
  7. from dataclasses import dataclass
  8. from typing import Dict, List, Optional, Any
  9. from hermes_cli.config import get_hermes_home
  10. from .config import Platform, GatewayConfig, PlatformConfig
  11. from .session import SessionSource
  12. from .dead_targets import DeadTargetRegistry, classify_dead_error
  13. logger = logging.getLogger(__name__)
  14. # Cap before gateway-level truncation of cron output for non-chunking platform delivery. Telegram's hard
  15. # API limit is 4096; the headroom covers the "full output saved to …" footer. Adapters that split long
  16. # messages natively (splits_long_messages) bypass this entirely.
  17. MAX_PLATFORM_OUTPUT = 4000
  18. # Matches strings that are *only* a "silence" narration with optional markdown wrappers (*(silent)*,
  19. # _silent_, 🔇, a bare ".", "…"). Anchored so messages that merely *contain* "silent" never match.
  20. _SILENCE_NARRATION = re.compile(
  21. r'^[\s*_~`]*\(?\s*(silent|silence|no\s+response|no\s+reply)\s*\.?\)?[\s*_~`]*$'
  22. r'|^[\s*_~`]*[\U0001F507\.\u2026]+[\s*_~`]*$',
  23. re.IGNORECASE,
  24. )
  25. _THREAD_ROUTING_KEYS = ("thread_id", "message_thread_id", "direct_messages_topic_id", "telegram_direct_messages_topic_id")
  26. def _is_silence_narration(content: Optional[str]) -> bool:
  27. """True when ``content`` is *only* a silence-narration token (length-guarded)."""
  28. stripped = content.strip() if content else ""
  29. return bool(stripped) and len(stripped) <= 64 and bool(_SILENCE_NARRATION.match(stripped))
  30. class PartialDeliveryError(RuntimeError):
  31. """A split send failed after earlier chunks were delivered (``raw_response["partial_overflow"]``).
  32. Callers must not fall back to re-sending the whole payload: the recipient already has the head."""
  33. def __init__(self, message: str, result: Any):
  34. super().__init__(message)
  35. self.result = result
  36. @dataclass(frozen=True)
  37. class DeliveryTransport:
  38. """Resolved live transport for one logical delivery platform."""
  39. adapter: Any
  40. config: Optional[PlatformConfig]
  41. transport_platform: Platform
  42. @property
  43. def is_relay(self) -> bool:
  44. return self.transport_platform == Platform.RELAY
  45. async def send(self, logical_platform: Platform, chat_id: str, content: str,
  46. metadata: Optional[dict[str, Any]]) -> Any:
  47. """Send through this transport while preserving the logical platform."""
  48. return await (self.adapter.send_for_platform(logical_platform, chat_id, content, metadata=metadata)
  49. if self.is_relay else self.adapter.send(chat_id, content, metadata=metadata))
  50. def resolve_delivery_transport(platform: Platform, config: GatewayConfig,
  51. adapters: Optional[dict[Platform, Any]]) -> Optional[DeliveryTransport]:
  52. """Resolve a logical platform to its live delivery transport. A concrete native adapter always wins;
  53. Relay is eligible only when its authenticated transport explicitly advertises that it fronts the
  54. logical platform, so restart-time delivery is independent of per-chat caches without letting Relay
  55. hijack unrelated platform targets."""
  56. live_adapters = adapters or {}
  57. native, native_config = live_adapters.get(platform), config.platforms.get(platform)
  58. # Explicitly supplied live adapters with no config block are honored, but an
  59. # explicitly disabled native adapter never shadows an enabled Relay transport.
  60. if native is not None and (native_config is None or native_config.enabled):
  61. return DeliveryTransport(native, native_config, platform)
  62. relay, relay_config = live_adapters.get(Platform.RELAY), config.platforms.get(Platform.RELAY)
  63. fronts_platform = getattr(relay, "fronts_platform", None)
  64. if (relay is not None and (relay_config is None or relay_config.enabled)
  65. and callable(fronts_platform) and fronts_platform(platform)):
  66. return DeliveryTransport(relay, relay_config, Platform.RELAY)
  67. return None
  68. def looks_like_telegram_private_chat_id(chat_id: Optional[str]) -> bool:
  69. """True when ``chat_id`` is a positive int — Telegram's private-chat shape (groups/channels are negative).
  70. Single source of truth, reused by the handoff seed path in ``gateway/run.py`` so handoff-created DM
  71. topics key the same way as inbound DM-topic messages."""
  72. try:
  73. return int(chat_id) > 0
  74. except (TypeError, ValueError):
  75. return False
  76. def _looks_like_int(value: Optional[str]) -> bool:
  77. try:
  78. return int(value) is not None
  79. except (TypeError, ValueError):
  80. return False
  81. def _send_result_error(result: Any) -> Optional[str]:
  82. """Error string of a failed SendResult object / plain result dict ("" if none), or None on success."""
  83. get = result.get if isinstance(result, dict) else (lambda name, default=None: getattr(result, name, default))
  84. return None if get("success", True) is not False else str(get("error") or "")
  85. @dataclass
  86. class DeliveryTarget:
  87. """One target: "origin", "local", "telegram" (home channel) or "telegram:123456[:thread]"."""
  88. platform: Platform
  89. chat_id: Optional[str] = None # None means use home channel
  90. thread_id: Optional[str] = None
  91. is_origin: bool = False
  92. is_explicit: bool = False # True if chat_id was explicitly specified
  93. # Raw target string when the platform name is unknown. The platform falls
  94. # back to LOCAL for routing, but the original name is preserved so
  95. # deliver() can report {success: False, error: unknown_platform} instead
  96. # of silently misrouting to local files.
  97. unknown_platform: Optional[str] = None
  98. @classmethod
  99. def parse(cls, target: str, origin: Optional[SessionSource] = None) -> "DeliveryTarget":
  100. """Parse "origin" | "local" | "<platform>" | "<platform>:<chat_id>[:<thread_id>]"."""
  101. target = target.strip()
  102. if target.lower() == "origin":
  103. return (cls(platform=origin.platform, chat_id=origin.chat_id, thread_id=origin.thread_id, is_origin=True)
  104. if origin else cls(platform=Platform.LOCAL, is_origin=True))
  105. # Platform names are case-insensitive; chat/thread ids keep case. Unknown platforms ->
  106. # LOCAL for routing but preserve the raw target so deliver() reports
  107. # unknown_platform instead of silently saving locally.
  108. parts = target.split(":", 2)
  109. try:
  110. platform = Platform(parts[0].lower())
  111. except ValueError:
  112. return cls(platform=Platform.LOCAL, unknown_platform=target)
  113. return (cls(platform=platform, chat_id=parts[1], thread_id=parts[2] if len(parts) > 2 else None, is_explicit=True)
  114. if len(parts) > 1 else cls(platform=platform))
  115. def to_string(self) -> str:
  116. """Convert back to string format."""
  117. if self.is_origin:
  118. return "origin"
  119. if self.unknown_platform is not None:
  120. return self.unknown_platform
  121. if self.platform == Platform.LOCAL:
  122. return "local"
  123. parts = [self.platform.value, self.chat_id, self.thread_id if self.chat_id else None]
  124. return ":".join(p for p in parts if p)
  125. async def _ensure_named_dm_topic(adapter: Any, chat_id: str, name: str, *, refresh: bool) -> str:
  126. """Create (or force-recreate) a named Telegram private DM topic; return its thread id."""
  127. verb, ensure_dm_topic = "refresh" if refresh else "create", getattr(adapter, "ensure_dm_topic", None)
  128. if ensure_dm_topic is None:
  129. raise RuntimeError(f"Telegram adapter cannot {verb} named private DM topics")
  130. thread_id = await ensure_dm_topic(chat_id, name, **({"force_create": True} if refresh else {}))
  131. if not thread_id:
  132. raise RuntimeError(f"Failed to {verb} Telegram private DM topic '{name}'")
  133. return str(thread_id)
  134. class DeliveryRouter:
  135. """Resolves delivery targets and dispatches messages to platform adapters."""
  136. def __init__(self, config: GatewayConfig, adapters: dict[Platform, Any] | None = None,
  137. dead_targets: Optional[DeadTargetRegistry] = None): # profile-local registry when omitted
  138. self.config = config
  139. self.adapters = adapters or {}
  140. self.output_dir = get_hermes_home() / "cron" / "output"
  141. self.dead_targets = dead_targets or DeadTargetRegistry()
  142. async def deliver(self, content: str, targets: list[DeliveryTarget], job_id: Optional[str] = None,
  143. job_name: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  144. """Deliver content to all targets; returns per-target results keyed by target string."""
  145. results = {}
  146. for target in targets:
  147. if target.unknown_platform is not None:
  148. results[target.to_string()] = {
  149. "success": False, "error": f"unknown_platform: {target.unknown_platform}"}
  150. continue
  151. # Skip targets proven permanently unreachable (deleted group, blocked bot, deactivated user) —
  152. # re-sending each tick wastes flood-control budget. Self-healing: a later successful send
  153. # clears the flag. LOCAL/origin-without-chat targets are never dead-tracked.
  154. tracked = target.platform != Platform.LOCAL and target.chat_id
  155. if tracked and self.dead_targets.is_dead(target.platform.value, target.chat_id):
  156. logger.info("Skipping delivery to known-dead target %s:%s (send to it again to clear)",
  157. target.platform.value, target.chat_id)
  158. results[target.to_string()] = {"success": False, "skipped": "dead_target",
  159. "error": "target previously confirmed unreachable"}
  160. continue
  161. try:
  162. if target.platform == Platform.LOCAL:
  163. result = self._deliver_local(content, job_id, job_name, metadata)
  164. else:
  165. result = await self._deliver_to_platform(target, content, metadata)
  166. if target.chat_id and _send_result_error(result) is None:
  167. self.dead_targets.clear(target.platform.value, target.chat_id)
  168. results[target.to_string()] = {"success": True, "result": result}
  169. except Exception as e:
  170. # Hard failures raise. Record a whole-chat death so future deliveries short-circuit.
  171. dead_kind = classify_dead_error(str(e)) if tracked else None
  172. if dead_kind:
  173. self.dead_targets.mark_dead(target.platform.value, target.chat_id,
  174. reason=f"{dead_kind}: {str(e)[:120]}")
  175. results[target.to_string()] = {"success": False, "error": str(e)}
  176. return results
  177. def _deliver_local(self, content: str, job_id: Optional[str], job_name: Optional[str],
  178. metadata: Optional[dict[str, Any]]) -> dict[str, Any]:
  179. """Save content to local files."""
  180. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  181. output_path = self.output_dir / (job_id or "misc") / f"{timestamp}.md"
  182. output_path.parent.mkdir(parents=True, exist_ok=True)
  183. lines = [f"# {job_name}" if job_name else "# Delivery Output", "",
  184. f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"]
  185. lines += [f"**Job ID:** {job_id}"] if job_id else []
  186. lines += [f"**{key}:** {value}" for key, value in (metadata or {}).items()] + ["", "---", "", content]
  187. output_path.write_text("\n".join(lines), encoding="utf-8")
  188. return {"path": str(output_path), "timestamp": timestamp}
  189. def _save_full_output(self, content: str, job_id: str) -> Path:
  190. """Save full cron output to disk and return the file path."""
  191. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  192. path = get_hermes_home() / "cron" / "output" / f"{job_id}_{timestamp}.txt"
  193. path.parent.mkdir(parents=True, exist_ok=True)
  194. path.write_text(content, encoding="utf-8")
  195. return path
  196. def _filter_silence_narration_enabled(self) -> bool:
  197. """filter silence narration based on gateway config without checking process env"""
  198. return bool(getattr(self.config, "filter_silence_narration", True))
  199. def _cap_oversized_output(self, adapter: Any, content: str, job_id: str) -> str:
  200. """Audit-save oversized cron output; truncate it for non-chunking adapters. Above MAX_PLATFORM_OUTPUT
  201. the full output is always written to disk as an audit trail, best-effort — a failed save (full disk,
  202. permissions) never blocks delivery. Non-chunking adapters then get the content truncated with a
  203. footer pointing to the saved file; ``splits_long_messages`` adapters receive the full payload."""
  204. if len(content) <= MAX_PLATFORM_OUTPUT:
  205. return content
  206. saved_path: Optional[Path] = None
  207. try:
  208. saved_path = self._save_full_output(content, job_id)
  209. except OSError as exc:
  210. logger.warning("Audit save failed for cron output (%d chars, job=%s): %s — "
  211. "delivery proceeds without audit copy", len(content), job_id, exc)
  212. if getattr(adapter, "splits_long_messages", False):
  213. if saved_path:
  214. logger.info("Cron output preserved for chunking adapter (%d chars) — "
  215. "full output saved to %s", len(content), saved_path)
  216. return content
  217. # The footer needs a valid path: if the best-effort save failed, retry
  218. # (a failure now is a real delivery problem and propagates).
  219. saved_path = saved_path or self._save_full_output(content, job_id)
  220. footer = f"\n\n... [truncated, full output saved to {saved_path}]"
  221. logger.info("Cron output truncated (%d chars) — full output: %s", len(content), saved_path)
  222. return content[:max(0, MAX_PLATFORM_OUTPUT - len(footer))] + footer
  223. async def _deliver_to_platform(self, target: DeliveryTarget, content: str,
  224. metadata: Optional[dict[str, Any]],
  225. transport: Optional[DeliveryTransport] = None,
  226. ) -> dict[str, Any]:
  227. """Deliver content to a messaging platform.
  228. ``transport`` carries an already-authorized transport past resolution:
  229. the cron live lane resolved and authorized it per target (including the
  230. SharedRouteAdapters satellite grant), and re-resolving from the plain
  231. adapters dict cannot re-derive that grant under satellite config
  232. (#115656). Omitted (None) preserves resolution for every other caller.
  233. """
  234. if transport is None:
  235. transport = resolve_delivery_transport(target.platform, self.config, self.adapters)
  236. if transport is None:
  237. raise ValueError(f"No adapter configured for {target.platform.value}")
  238. if not target.chat_id:
  239. raise ValueError(f"No chat ID for {target.platform.value} delivery")
  240. adapter = transport.adapter
  241. content = self._cap_oversized_output(adapter, content, (metadata or {}).get("job_id", "unknown"))
  242. # Substrate-level anti-loop guard: drop hallucinated "silence narration" (*(silent)*, 🔇, a bare ".")
  243. # before it reaches any adapter — in bot-to-bot channels these mirror back and forth until a model
  244. # crashes with "no content after all retries"; prompt rules drift across providers, so this single
  245. # chokepoint covers every platform. Local/file delivery is never filtered (saved silence has no loop
  246. # risk). Cron output is an ARTIFACT, not model chatter: a legitimately terse job ("...", a single 🔇)
  247. # has no mirror loop, and dropping it while returning success is how a cron gets logged as delivered
  248. # with nothing on the wire. Cron sends carry job_id in metadata; everything else is filtered.
  249. # See #77763.
  250. is_cron_artifact = "job_id" in (metadata or {})
  251. if self._filter_silence_narration_enabled() and not is_cron_artifact and _is_silence_narration(content):
  252. logger.warning("Dropped silence-narration outbound to %s (chat=%s): %r",
  253. target.platform.value, target.chat_id, content[:40])
  254. return {"success": True, "filtered": "silence_narration", "delivered": False}
  255. send_metadata = dict(metadata or {})
  256. home = self.config.get_home_channel(target.platform) if transport.is_relay else None
  257. if home is not None and home.chat_id == target.chat_id:
  258. send_metadata.update({k: v for k, v in (("user_id", home.user_id), ("scope_id", home.scope_id)) if v})
  259. # Caller-supplied thread routing always wins over target.thread_id.
  260. named_topic: Optional[str] = None # named Telegram private topic created for this send
  261. thread_id = target.thread_id
  262. if thread_id and not any(key in send_metadata for key in _THREAD_ROUTING_KEYS):
  263. send_metadata["thread_id"] = thread_id
  264. if target.platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(target.chat_id):
  265. if not _looks_like_int(thread_id):
  266. # Named topic: create via createForumTopic, use message_thread_id directly.
  267. named_topic = thread_id
  268. send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, thread_id, refresh=False)
  269. send_metadata["telegram_dm_topic_created_for_send"] = True
  270. elif send_metadata.get("telegram_reply_to_message_id") is None:
  271. # Legacy numeric private topic ids not created by this send path need a reply
  272. # anchor to stay visible in the requested lane.
  273. raise RuntimeError(
  274. "Telegram private DM topic delivery requires telegram_reply_to_message_id; "
  275. "send to the bare chat or provide a reply anchor"
  276. )
  277. else:
  278. send_metadata["telegram_dm_topic_reply_fallback"] = True
  279. for retry in (False, True):
  280. result = await transport.send(target.platform, target.chat_id, content, metadata=send_metadata or None)
  281. error = _send_result_error(result)
  282. if retry or error is None or not named_topic or "thread not found" not in error.lower():
  283. break
  284. # The named topic vanished under us: recreate it once and resend.
  285. send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, named_topic, refresh=True)
  286. send_metadata["telegram_dm_topic_created_for_send"] = True
  287. if error is not None:
  288. from gateway.platforms.base import BasePlatformAdapter
  289. if BasePlatformAdapter._is_partial_delivery(result):
  290. raise PartialDeliveryError(error, result)
  291. raise RuntimeError(error or f"{target.platform.value} delivery failed")
  292. return result