parked_route_refusal.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. """A message a direct platform bot routes to a parked profile is refused out loud, never dropped.
  2. A parked profile comes back on a rescan, so its traffic is temporarily unavailable. Webhooks are
  3. answered 503 for the provider to redeliver (``webhook_ingress``). A direct bot (Telegram's PTB
  4. ``Updater``, Slack Socket Mode, Discord's gateway, Google Chat Pub/Sub) has already confirmed the
  5. update to its platform before the gateway sees it, so the platform never redelivers it (Discord's own
  6. missed-message backfill, when enabled, can still pick it up after recovery). Holding it here would be
  7. a second inbound queue outside the one durable queue, kept for a profile whose store is the one that
  8. cannot be trusted. So the sender is told, in-channel, to resend, and nothing is admitted.
  9. A relay frame is refused the same way (:class:`ParkedProfileRefusal`), and acknowledged: the park
  10. lasts until a rescan or a restart, and the connector drains its per-instance buffer in order,
  11. ack-gated, so a withheld frame would hold back every later frame for every profile on that instance;
  12. a live frame has no redelivery at all. Only transient refusals (a busy store, a drain) withhold it.
  13. A transport that answers only after an awaited dispatch (Google Chat HTTP events) has not acknowledged
  14. the delivery yet: it runs that dispatch in :func:`retryable_ingress` and answers the named, retryable
  15. :class:`ParkedRouteRefused` instead, so the platform redelivers it.
  16. """
  17. import asyncio
  18. import contextvars
  19. import logging
  20. import time
  21. from contextlib import contextmanager
  22. from hermes_state_runtime import RuntimeStoreError
  23. logger = logging.getLogger(__name__)
  24. class ParkedRouteRefused(Exception):
  25. """An unacknowledged delivery routed to a parked profile: nothing was admitted, answer ``status``."""
  26. code, status, headers = 'profile_parked', 503, {'Retry-After': '30'}
  27. def __init__(self):
  28. super().__init__('The routed profile is not being served right now; retry this delivery')
  29. class ParkedProfileRefusal(RuntimeStoreError):
  30. """Producer admission refused because the delivery routes to the parked *profile*: retryable
  31. (``runtime_draining``), but for as long as the park lasts, so an ordered transport acknowledges it
  32. and refuses it to the sender (:func:`refuse_parked_route`) instead of redelivering it."""
  33. def __init__(self, profile):
  34. super().__init__('runtime_draining', field='profile', detail='the routed profile is parked')
  35. self.profile = profile
  36. class _RetryableIngress:
  37. def __init__(self):
  38. self.open, self.refused = True, False
  39. _retryable_ingress = contextvars.ContextVar('parked_route_retryable_ingress', default=None)
  40. @contextmanager
  41. def retryable_ingress():
  42. """Scope an awaited dispatch its transport has not acknowledged. A parked route inside it is only
  43. recorded (``scope.refused``) for the caller to raise :class:`ParkedRouteRefused`; no in-chat notice.
  44. Closed on exit, so a task that outlives the dispatch still refuses in-channel."""
  45. scope = _RetryableIngress()
  46. token = _retryable_ingress.set(scope)
  47. try:
  48. yield scope
  49. finally:
  50. scope.open = False
  51. _retryable_ingress.reset(token)
  52. # Generic on purpose: the chat is not told which profile the route targets or why it is parked.
  53. # Worded as an outage notice Discord's backfill recognises (``_is_down_notice_content``): the reply
  54. # must not mark the never-admitted message answered, so backfill picks it up once it is served again.
  55. PARKED_ROUTE_NOTICE = 'The agent is currently unavailable; please resend your message in a moment.'
  56. # One notice per (profile, platform, chat) per window: a relay reconnect drains its whole backlog
  57. # for a parked profile at once, and N frames must not post N notices in the same chat.
  58. PARKED_NOTICE_WINDOW_S = 60.0
  59. _monotonic = time.monotonic
  60. def _notice_due(adapter, profile, platform, chat_id):
  61. """True (and the window restarts) when this chat was not told about *profile* in the window."""
  62. sent = getattr(adapter, '_parked_notice_sent_at', None)
  63. if not isinstance(sent, dict):
  64. sent = {}
  65. try:
  66. adapter._parked_notice_sent_at = sent
  67. except AttributeError:
  68. return True
  69. now, key = _monotonic(), (profile, platform, chat_id)
  70. for stale in [k for k, at in sent.items() if now - at >= PARKED_NOTICE_WINDOW_S]:
  71. del sent[stale]
  72. if key in sent:
  73. return False
  74. sent[key] = now
  75. return True
  76. def drop_rejected_route(adapter, source):
  77. """Log an inbound event dropped because its explicit profile route targets a profile this
  78. gateway does not serve; when that profile is parked, also refuse it to the sender in-channel."""
  79. from gateway.run_runtime import profile_parked
  80. name = getattr(adapter, 'name', None) or getattr(getattr(source, 'platform', None), 'value', '?')
  81. chat_id = getattr(source, 'chat_id', '?')
  82. profile = getattr(source, '_rejected_route_profile', None)
  83. if adapter is None or not profile_parked(getattr(adapter, 'gateway_runner', None), profile):
  84. logger.warning(
  85. '[%s] Dropping inbound event for %s: explicit profile route targets an unserved profile', name, chat_id)
  86. return
  87. refuse_parked_route(adapter, source, profile)
  88. def refuse_parked_route(adapter, source, profile, reply_to=None):
  89. """Refuse an event routed to the parked *profile*, nothing admitted: answered retryably inside
  90. :func:`retryable_ingress`, else told to the sender in-channel, in reply to *reply_to* (default:
  91. the source's message), off the caller, which may be the transport reader a relay send waits on."""
  92. name = getattr(adapter, 'name', None) or getattr(getattr(source, 'platform', None), 'value', '?')
  93. chat_id = getattr(source, 'chat_id', '?')
  94. platform = getattr(getattr(source, 'platform', None), 'value', None) or '?'
  95. scope = _retryable_ingress.get()
  96. if scope is not None and scope.open:
  97. scope.refused = True
  98. logger.warning(
  99. '[%s] Refusing inbound event for %s: routed profile %r on %s is parked (reason=profile_parked); '
  100. 'nothing was admitted, the delivery is answered retryably', name, chat_id, profile, platform)
  101. return
  102. if not _notice_due(adapter, profile, platform, chat_id):
  103. logger.info(
  104. '[%s] Refusing inbound event for %s: routed profile %r on %s is parked (reason=profile_parked); '
  105. 'nothing was admitted, the chat was already told within %.0fs', name, chat_id, profile, platform,
  106. PARKED_NOTICE_WINDOW_S)
  107. return
  108. logger.warning(
  109. '[%s] Refusing inbound event for %s: routed profile %r on %s is parked (reason=profile_parked); '
  110. 'nothing was admitted, a resend notice to the sender is scheduled', name, chat_id, profile, platform)
  111. try:
  112. task = asyncio.get_running_loop().create_task(
  113. _send_refusal(adapter, source, reply_to or getattr(source, 'message_id', None)))
  114. except RuntimeError:
  115. logger.warning('[%s] Could not refuse the parked-profile event for %s: no running loop', name, chat_id)
  116. return
  117. tasks = getattr(adapter, '_background_tasks', None)
  118. if isinstance(tasks, set):
  119. tasks.add(task)
  120. task.add_done_callback(tasks.discard)
  121. async def _send_refusal(adapter, source, reply_to):
  122. """Reply in the sender's chat and thread, as the runner's own notices do. Marked interim: it is no
  123. turn's answer, so it never seals another turn's open stream (relay Slack) or fills a pending
  124. answer slot (LINE's postback cache)."""
  125. from gateway.run import _interim_metadata
  126. try:
  127. metadata = _interim_metadata(adapter.gateway_runner._thread_metadata_for_source(source))
  128. result = await adapter.send(source.chat_id, PARKED_ROUTE_NOTICE, reply_to=reply_to, metadata=metadata)
  129. if getattr(result, 'success', True) is False:
  130. # Safe fields only: the raw ``error`` can echo tokens or message content.
  131. logger.warning('[%s] The parked-profile refusal to %s was not delivered (error_kind=%s, retryable=%s)',
  132. getattr(adapter, 'name', '?'), getattr(source, 'chat_id', '?'),
  133. getattr(result, 'error_kind', None), getattr(result, 'retryable', None))
  134. except Exception:
  135. logger.warning('[%s] Could not send the parked-profile refusal to %s',
  136. getattr(adapter, 'name', '?'), getattr(source, 'chat_id', '?'), exc_info=True)