telegram_held_inbound.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142
  1. """Telegram held-inbound queue: events PTB already acked that cannot dispatch yet are held,
  2. redispatched on reconnect, and handed to the adapter that replaces this one (#132829)."""
  3. import asyncio
  4. import logging
  5. import weakref
  6. from typing import Optional
  7. from gateway.platforms.event import MessageEvent
  8. from gateway.platforms.helpers import cancel_task
  9. logger = logging.getLogger("plugins.platforms.telegram.adapter")
  10. class TelegramHeldInboundMixin:
  11. """Hold / redispatch / hand-over of inbound events for ``TelegramAdapter``."""
  12. def _schedule_held_inbound_redispatch(self) -> None:
  13. """Ensure a tracked drain runs when held events exist and delivery is live (no-op while
  14. down or after permanent fatal; an in-flight drain schedules its own follow-up)."""
  15. if self._is_permanent_fatal() or self._should_drop_delayed_delivery():
  16. return
  17. if not getattr(self, "_held_inbound_events", None):
  18. return
  19. try:
  20. loop = asyncio.get_running_loop()
  21. except RuntimeError:
  22. return
  23. prior = getattr(self, "_held_inbound_redispatch_task", None)
  24. try:
  25. current = asyncio.current_task()
  26. except RuntimeError:
  27. current = None
  28. if prior is not None and not prior.done() and prior is not current:
  29. return
  30. self._held_inbound_redispatch_task = loop.create_task(self._redispatch_held_inbound(prior=None if prior is current else prior))
  31. def _hold_inbound_event(self, event: MessageEvent, *, where: str, schedule: bool = True) -> None:
  32. """Preserve an inbound event that cannot be dispatched now (PTB already acked the update, so dropping is silent loss).
  33. Capped, identity-deduped; permanent fatal discards. ``schedule=False`` inside a drain avoids poison-event loops.
  34. The disconnect drop-guard (#55971) correctly prevents dispatch into a torn-down session. Destroying
  35. the event is wrong: by the time we reach enqueue/flush, python-telegram-bot has already acked the
  36. update and advanced the offset — silent permanent loss, no log, no error.
  37. """
  38. if self._is_permanent_fatal():
  39. logger.warning(
  40. "[Telegram] Discarding inbound under non-retryable fatal (%s, %d chars)", where, len(getattr(event, "text", None) or ""))
  41. return
  42. successor = getattr(self, "_held_inbound_successor", None)
  43. target = successor() if successor is not None else None
  44. if target is not None: # retired by a runner rebuild (#132829)
  45. self._accept_update() # the claim being dispatched is ours, not the successor's
  46. target._adopt_held_event(event, where=f"{where}-forwarded", schedule=schedule)
  47. return
  48. held = getattr(self, "_held_inbound_events", None)
  49. if held is None:
  50. self._held_inbound_events = held = []
  51. if any(existing is event for existing in held):
  52. return
  53. max_n = int(getattr(self, "HELD_INBOUND_MAX", 64) or 64)
  54. while len(held) >= max_n:
  55. dropped = held.pop(0)
  56. logger.warning(
  57. "[Telegram] Held-inbound queue full (%d); dropping oldest (%d chars)", max_n, len(getattr(dropped, "text", None) or ""))
  58. held.append(event)
  59. self._accept_update()
  60. logger.warning(
  61. "[Telegram] Holding inbound (%s, %d chars, queue=%d)%s", where, len(getattr(event, "text", None) or ""), len(held),
  62. " - will redispatch on reconnect" if self._should_drop_delayed_delivery() else (" - scheduling redispatch" if schedule else ""))
  63. # A live-path hold must not orphan the event waiting for a reconnect that never comes.
  64. if schedule and not self._should_drop_delayed_delivery():
  65. self._schedule_held_inbound_redispatch()
  66. def adopt_held_inbound(self, predecessor: "TelegramHeldInboundMixin") -> None:
  67. """Take over the hold queue of the instance the runner just replaced with us (#132829): it only
  68. drains on its own ``_mark_connected``, which never comes; later holds there forward here."""
  69. self._held_inbound_successor = None # we own the queue again: a reverse link would forward in a cycle
  70. predecessor._held_inbound_successor = weakref.ref(self)
  71. held = getattr(predecessor, "_held_inbound_events", None) or []
  72. events = list(held)
  73. held.clear()
  74. for event in events:
  75. self._adopt_held_event(event, where="adopted", schedule=False)
  76. self._schedule_held_inbound_redispatch()
  77. def _adopt_held_event(self, event: MessageEvent, *, where: str, schedule: bool) -> None:
  78. """Hold a predecessor's event here; only the transport ref moves (profile/authz stay as resolved)."""
  79. if getattr(event, "source", None) is not None:
  80. event.source._transport_adapter_ref = weakref.ref(self)
  81. self._hold_inbound_event(event, where=where, schedule=schedule)
  82. def _rehold_from(self, events: list, idx: int, where: str) -> None:
  83. """Re-hold ``events[idx:]`` without rescheduling (drain interrupted / failed / cancelled)."""
  84. for rest in events[idx:]:
  85. self._hold_inbound_event(rest, where=where, schedule=False)
  86. async def _redispatch_held_inbound(self, prior: Optional[asyncio.Task] = None) -> None:
  87. """Drain the hold queue after reconnect or a connected-path hold; ``prior`` (previous
  88. redispatch task) is cancelled+awaited here so ``_mark_connected`` stays synchronous."""
  89. if prior is not asyncio.current_task(): # a self-redispatch must not cancel itself
  90. await cancel_task(prior)
  91. held = getattr(self, "_held_inbound_events", None)
  92. if self._is_permanent_fatal():
  93. if held:
  94. n = len(held)
  95. held.clear()
  96. logger.warning("[Telegram] Redispatch aborted; discarded %d held inbound under non-retryable fatal", n)
  97. return
  98. if not held:
  99. return
  100. # Take ownership atomically; concurrent holds append to the fresh list for a follow-up.
  101. events = list(held)
  102. held.clear()
  103. logger.warning("[Telegram] Redispatching %d held inbound message(s)", len(events))
  104. allow_followup_schedule = True
  105. try:
  106. for idx, event in enumerate(events):
  107. if self._is_permanent_fatal() or self._should_drop_delayed_delivery():
  108. self._rehold_from(events, idx, "redispatch-interrupted")
  109. return
  110. try:
  111. await self.handle_message(event)
  112. except asyncio.CancelledError:
  113. self._rehold_from(events, idx, "redispatch-cancelled")
  114. raise
  115. except Exception:
  116. # Retryable failure: re-hold but do NOT reschedule now (a poison event would
  117. # tight-loop); the next mark_connected/live hold drains.
  118. logger.exception(
  119. "[Telegram] Failed to redispatch held inbound (%d chars); re-holding", len(getattr(event, "text", None) or ""))
  120. self._rehold_from(events, idx, "redispatch-failed")
  121. allow_followup_schedule = False
  122. return
  123. finally:
  124. # Events that arrived mid-drain while still connected need another pass.
  125. if (
  126. allow_followup_schedule
  127. and getattr(self, "_held_inbound_events", None)
  128. and not self._should_drop_delayed_delivery()
  129. and not self._is_permanent_fatal()):
  130. self._schedule_held_inbound_redispatch()