update_admission.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """Telegram admission before every PTB handler group, including native plugins.
  2. Claims belong to the receiving adapter across Application rebuilds. Completed
  3. history is bounded and has no TTL in memory; dispatch and its PTB tasks pin active
  4. claims. Completed IDs are also written to a per-bot receipt file under the adapter's
  5. Hermes home, so a replacement adapter or a restarted gateway still drops updates that
  6. Telegram redelivers because their getUpdates/webhook acknowledgement never landed.
  7. No cross-process coordination or exactly-once effects are promised.
  8. """
  9. import asyncio
  10. import json
  11. import logging
  12. import time
  13. from contextvars import ContextVar
  14. from dataclasses import dataclass
  15. from functools import wraps
  16. from telegram import Update
  17. from telegram.ext import Application, ApplicationHandlerStop, ConversationHandler, SimpleUpdateProcessor
  18. from gateway.platforms._shared import coerce_port
  19. from gateway.platforms.helpers import bounded_put
  20. from utils import atomic_json_write
  21. logger = logging.getLogger(__name__)
  22. _DEFAULT_BLOCK = object()
  23. DEFAULT_MAX_CONCURRENT_UPDATES = 32
  24. _SEEN_CAP = 4096
  25. # The Bot API keeps an unconfirmed update for at most 24 hours (getUpdates), so an older
  26. # receipt can never match a redelivery. That is also well inside the week of silence after
  27. # which Telegram may restart update IDs at a random value, so a set lookup stays safe
  28. # where a persisted numeric high-watermark would not.
  29. RECEIPT_TTL_SECONDS = 24 * 60 * 60
  30. def _receipt_path(adapter, bot_id):
  31. return adapter._update_receipt_dir / f"telegram_update_receipts_{bot_id}.json"
  32. def _load_receipts(adapter, bot_id) -> None:
  33. """Seed completed history from disk once per bot, before its first admission check."""
  34. if bot_id in adapter._update_receipts_loaded:
  35. return
  36. adapter._update_receipts_loaded.add(bot_id)
  37. path = _receipt_path(adapter, bot_id)
  38. try:
  39. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  40. except FileNotFoundError:
  41. return
  42. except (OSError, ValueError):
  43. logger.warning("[Telegram] Ignoring unreadable update receipts at %s", path, exc_info=True)
  44. return
  45. ids = payload.get("update_ids") if isinstance(payload, dict) else None
  46. if not isinstance(ids, dict):
  47. return
  48. cutoff = time.time() - RECEIPT_TTL_SECONDS
  49. fresh = sorted((float(ts), uid) for uid, ts in ids.items()
  50. if isinstance(ts, (int, float)) and ts > cutoff and str(uid).lstrip("-").isdigit())
  51. seen = adapter._seen_update_ids
  52. for ts, uid in fresh:
  53. key = f"{bot_id}:{uid}"
  54. if key not in seen:
  55. bounded_put(seen, key, ts, _SEEN_CAP)
  56. def _record_receipt(adapter, key: str) -> None:
  57. bounded_put(adapter._seen_update_ids, key, time.time(), _SEEN_CAP)
  58. adapter._update_receipts_dirty.add(key.split(":", 1)[0])
  59. task = adapter._update_receipt_flush
  60. if task is None or task.done():
  61. # One writer per adapter coalesces bursts; the fsync stays off the event loop.
  62. adapter._update_receipt_flush = asyncio.get_running_loop().create_task(
  63. _flush_receipts(adapter), name="telegram-update-receipts")
  64. async def _flush_receipts(adapter) -> None:
  65. from hermes_constants import mkdir_under_hermes_home
  66. while adapter._update_receipts_dirty:
  67. bot_id = adapter._update_receipts_dirty.pop()
  68. prefix, cutoff = f"{bot_id}:", time.time() - RECEIPT_TTL_SECONDS
  69. # Snapshot on the loop: admission mutates this dict only from loop callbacks.
  70. ids = {key[len(prefix):]: ts for key, ts in adapter._seen_update_ids.items()
  71. if key.startswith(prefix) and isinstance(ts, float) and ts > cutoff}
  72. path = _receipt_path(adapter, bot_id)
  73. try:
  74. mkdir_under_hermes_home(path.parent)
  75. await asyncio.to_thread(atomic_json_write, path, {"update_ids": ids}, indent=None)
  76. except Exception:
  77. logger.warning("[Telegram] Failed to persist update receipts to %s", path, exc_info=True)
  78. @dataclass
  79. class _Claim:
  80. key: str
  81. dispatch_task: asyncio.Task | None
  82. accepted: bool = False
  83. completed: bool = False
  84. failed: bool = False
  85. owners: int = 1
  86. class _ErrorCallback:
  87. """Admit callback entry while keeping PTB's original registration/removal key."""
  88. def __init__(self, callback):
  89. self.__wrapped__ = callback
  90. def __hash__(self):
  91. return hash(self.__wrapped__)
  92. def __eq__(self, other):
  93. if isinstance(other, _ErrorCallback):
  94. other = other.__wrapped__
  95. return self.__wrapped__ == other
  96. async def __call__(self, update, context):
  97. # Registration is a native extension boundary, unlike PTB's internal logger.
  98. # Resolve the receiving app so a shared callback never accepts another bot's claim.
  99. admission = getattr(context.application, "_current_claim", None)
  100. claim = admission.get() if admission is not None else None
  101. if claim is not None and claim.owners:
  102. claim.accepted = True
  103. return await self.__wrapped__(update, context)
  104. class PerChatUpdateProcessor(SimpleUpdateProcessor):
  105. """Concurrent across chats, FIFO within one chat.
  106. PTB's default processor (max_concurrent_updates=1) awaits every update inline, so one slow
  107. handler deafens all chats; plain concurrency fixes that but lets a chat's second update finish
  108. before its first. Each update here waits for the previous update of the same chat; updates
  109. without a chat (inline queries) run unchained.
  110. """
  111. __slots__ = ("_tails",)
  112. def __init__(self, max_concurrent_updates: int):
  113. super().__init__(max_concurrent_updates)
  114. self._tails: dict[int, asyncio.Future] = {}
  115. async def do_process_update(self, update, coroutine) -> None:
  116. chat = getattr(update, "effective_chat", None)
  117. if chat is None:
  118. await coroutine
  119. return
  120. prev = self._tails.get(chat.id)
  121. done = asyncio.get_running_loop().create_future()
  122. self._tails[chat.id] = done
  123. try:
  124. if prev is not None:
  125. # shield: cancelling this waiter must not cancel the predecessor's tail future.
  126. await asyncio.shield(prev)
  127. await coroutine
  128. finally:
  129. def release(_prev=None):
  130. done.set_result(None)
  131. if self._tails.get(chat.id) is done:
  132. del self._tails[chat.id]
  133. # A cancelled waiter hands its turn on only once its predecessor finishes (per-chat FIFO).
  134. if prev is None or prev.done():
  135. release()
  136. else:
  137. prev.add_done_callback(release)
  138. def build_update_processor(extra, name: str) -> PerChatUpdateProcessor:
  139. """``platforms.telegram.extra.max_concurrent_updates`` (default 32); bad values warn and fall back."""
  140. raw = extra.get("max_concurrent_updates", DEFAULT_MAX_CONCURRENT_UPDATES)
  141. limit = coerce_port(raw, 0)
  142. if limit < 1:
  143. logger.warning("[%s] Invalid max_concurrent_updates=%r; using %d",
  144. name, raw, DEFAULT_MAX_CONCURRENT_UPDATES)
  145. limit = DEFAULT_MAX_CONCURRENT_UPDATES
  146. return PerChatUpdateProcessor(limit)
  147. class TelegramApplication(Application):
  148. __slots__ = ("_current_claim", "adapter")
  149. def __init__(self, *, adapter, **kwargs):
  150. super().__init__(**kwargs)
  151. self.adapter = adapter
  152. if adapter._update_admission is None:
  153. adapter._update_admission = ContextVar[_Claim | None]("telegram_update_claim", default=None)
  154. self._current_claim: ContextVar[_Claim | None] = adapter._update_admission
  155. def add_handler(self, handler, group=0):
  156. super().add_handler(handler, group)
  157. self._admit_callbacks(handler)
  158. def _admit_callbacks(self, handler):
  159. # ConversationHandler has no callback. Keep its identity (including persistence,
  160. # state transitions and block resolution) and instrument its selected children.
  161. if isinstance(handler, ConversationHandler):
  162. for child in [*handler.entry_points, *handler.fallbacks,
  163. *(child for handlers in handler.states.values() for child in handlers)]:
  164. self._admit_callbacks(child)
  165. return
  166. callback = getattr(handler, "callback", None)
  167. if callback is None or getattr(callback, "_telegram_admitted", False):
  168. return
  169. @wraps(callback)
  170. async def admitted_callback(update, context):
  171. # Native plugins own arbitrary effects. Entry is the handoff; retrying a partly
  172. # completed plugin would be unsafe. Core preparation has explicit handoff markers.
  173. # A plugin may reuse the same handler across applications/reconnects. Resolve
  174. # the receiving claim at invocation, rather than closing over its first owner.
  175. application = context.application
  176. admission = getattr(application, "_current_claim", None)
  177. claim = admission.get() if admission is not None else None
  178. if claim is not None and getattr(callback, "__self__", None) is not application.adapter:
  179. claim.accepted = True
  180. result = await callback(update, context)
  181. if claim is not None:
  182. if claim.failed and not claim.accepted:
  183. # Stop subsequent groups only in blocking dispatch. PTB cannot stop
  184. # groups already scheduled via block=False; their outcomes settle below.
  185. if asyncio.current_task() is claim.dispatch_task:
  186. raise ApplicationHandlerStop
  187. else:
  188. claim.completed = True
  189. return result
  190. admitted_callback._telegram_admitted = True
  191. handler.callback = admitted_callback
  192. def add_error_handler(self, callback, block=_DEFAULT_BLOCK):
  193. if not isinstance(callback, _ErrorCallback):
  194. callback = _ErrorCallback(callback)
  195. # Omitted block must remain PTB's default, not explicit True (bot defaults differ).
  196. if block is _DEFAULT_BLOCK:
  197. super().add_error_handler(callback)
  198. else:
  199. super().add_error_handler(callback, block=block)
  200. def _Application__create_task(self, coroutine, update=None, is_error_handler=False, name=None):
  201. # PTB's process_error bypasses public create_task. Delegate at the shared task seam
  202. # without copying dispatch or losing its is_error_handler recursion protection.
  203. task = super()._Application__create_task(
  204. coroutine, update=update, is_error_handler=is_error_handler, name=name)
  205. claim = self._current_claim.get()
  206. if claim is not None and claim.owners:
  207. # Pin scheduled work before callback entry, including cancellation before entry.
  208. claim.owners += 1
  209. def finished(done):
  210. # PTB's task wrapper may never start, leaving its callback unawaited.
  211. if done.cancelled() and asyncio.iscoroutine(coroutine):
  212. coroutine.close()
  213. self._release_claim(claim, done)
  214. task.add_done_callback(finished)
  215. return task
  216. def _release_claim(self, claim, task=None):
  217. if task is not None and task.cancelled():
  218. claim.failed = True
  219. claim.owners -= 1
  220. if claim.owners:
  221. return
  222. del self.adapter._inflight_update_ids[claim.key]
  223. if claim.accepted or (claim.completed and not claim.failed):
  224. _record_receipt(self.adapter, claim.key)
  225. async def process_error(self, update, error, job=None, coroutine=None):
  226. claim = self._current_claim.get()
  227. if claim is not None:
  228. claim.failed = True
  229. stopped = await super().process_error(update, error, job, coroutine)
  230. # PTB catches callback errors. Do not let a later observer accept a failed preparation.
  231. return stopped or (claim is not None and not claim.accepted)
  232. async def process_update(self, update):
  233. if not isinstance(update, Update):
  234. return await super().process_update(update)
  235. bot_id = self.bot.id
  236. key = f"{bot_id}:{update.update_id}"
  237. _load_receipts(self.adapter, bot_id)
  238. # Dispatch happened even when preparation fails before the group-99 observer.
  239. self.adapter._updates_dispatched_total += 1
  240. seen = self.adapter._seen_update_ids
  241. pending = self.adapter._inflight_update_ids
  242. if key in seen or key in pending:
  243. return
  244. claim = _Claim(key, asyncio.current_task())
  245. # Atomic on PTB's event loop: no await between lookup and claim. Dispatch and its
  246. # PTB tasks share ownership; completed-history pressure cannot evict active work.
  247. pending[key] = claim
  248. token = self._current_claim.set(claim)
  249. try:
  250. await super().process_update(update)
  251. except BaseException:
  252. claim.failed = True
  253. raise
  254. finally:
  255. self._current_claim.reset(token)
  256. self._release_claim(claim)