signal_rate_limit.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. """Signal attachment rate-limit scheduler: process-wide token-bucket simulator mirroring the per-account
  2. attachment rate limit signal-cli/Signal-Server enforce. Producers (``SignalAdapter.send_multiple_images``
  3. and the ``send_message`` tool's Signal path) call ``acquire(n)`` before an attachment send; on a 429 they
  4. call ``feedback(retry_after, n)`` so the model recalibrates from the server's authoritative hint.
  5. Concurrent calls serialize through an ``asyncio.Lock`` — FIFO fairness across sessions sharing one daemon."""
  6. from __future__ import annotations
  7. import asyncio
  8. import logging
  9. import re
  10. import time
  11. from typing import Any, Optional
  12. from agent.i18n import t
  13. from agent.retry_utils import parse_retry_after_seconds
  14. logger = logging.getLogger(__name__)
  15. SIGNAL_MAX_ATTACHMENTS_PER_MSG = 32 # per-message attachment cap (Signal-{Android,Desktop} source)
  16. SIGNAL_RATE_LIMIT_BUCKET_CAPACITY = 50 # server-side token-bucket capacity for attachments
  17. SIGNAL_RATE_LIMIT_DEFAULT_RETRY_AFTER = 4 # fallback token refill interval for signal-cli < v0.14.3
  18. SIGNAL_RATE_LIMIT_MAX_ATTEMPTS = 2 # initial attempt + 1 retry
  19. SIGNAL_BATCH_PACING_NOTICE_THRESHOLD = 10.0 # estimated wait above this → notify the user
  20. SIGNAL_RPC_ERROR_RATELIMIT = -5 # signal-cli (v0.14.3+) JSON-RPC error code for RateLimitException
  21. class SignalRateLimitError(Exception):
  22. """Raised by ``SignalAdapter._rpc`` for rate-limit responses when ``raise_on_rate_limit=True``.
  23. ``retry_after`` is the server's per-token Retry-After in seconds (signal-cli ≥ v0.14.3) or None."""
  24. def __init__(self, message: str, retry_after: Optional[float] = None) -> None:
  25. super().__init__(message)
  26. self.retry_after = retry_after
  27. class SignalSchedulerError(Exception):
  28. pass
  29. # "Retry after 4 seconds" — libsignal-net's RetryLaterException string form, surfaced when 429s hit during
  30. # attachment upload (signal-cli wraps these as AttachmentInvalidException, so the typed path doesn't fire).
  31. _RETRY_AFTER_RE = re.compile(r"Retry after (\d+(?:\.\d+)?)\s*second", re.IGNORECASE)
  32. def _error_message(err: Any) -> str:
  33. return str(err.get("message", "")) if isinstance(err, dict) else str(err)
  34. def _extract_retry_after_seconds(err: Any) -> Optional[float]:
  35. """Per-token Retry-After from a signal-cli rate-limit error, or None. Sources, in order:
  36. ``error.data.response.results[*].retryAfterSeconds`` (signal-cli ≥ v0.14.3), then "Retry after N
  37. seconds" parsed from the message (RetryLaterException wrapped as AttachmentInvalidException)."""
  38. if isinstance(err, dict):
  39. results = ((err.get("data") or {}).get("response") or {}).get("results") or []
  40. candidates = [parse_retry_after_seconds(r.get("retryAfterSeconds")) for r in results
  41. if isinstance(r, dict) and r.get("retryAfterSeconds")]
  42. if candidates := [c for c in candidates if c is not None]:
  43. return max(candidates)
  44. match = _RETRY_AFTER_RE.search(_error_message(err))
  45. return parse_retry_after_seconds(match.group(1)) if match else None
  46. def _is_signal_rate_limit_error(err: Any) -> bool:
  47. """True if a signal-cli RPC error reflects a rate-limit failure: the typed ``RATELIMIT_ERROR`` code
  48. (≥ v0.14.3), legacy ``[429]`` / ``RateLimitException`` substrings, or libsignal-net's
  49. ``RetryLaterException`` / "Retry after N seconds" leaked through AttachmentInvalidException."""
  50. if isinstance(err, dict) and err.get("code") == SIGNAL_RPC_ERROR_RATELIMIT:
  51. return True
  52. message = _error_message(err)
  53. msg_lower = message.lower()
  54. return "[429]" in message or any(s in msg_lower for s in ("ratelimit", "retrylaterexception", "retry after"))
  55. def _format_wait(seconds: float) -> str:
  56. """Human-friendly wait label for user-facing pacing notices."""
  57. s = max(0.0, seconds)
  58. return t("platform.signal.wait_seconds", count=round(s)) if s < 90 else t("platform.signal.wait_minutes", count=max(1, round(s / 60)))
  59. def _signal_send_timeout(num_attachments: int) -> float:
  60. """HTTP timeout for a Signal ``send`` RPC: signal-cli uploads attachments serially inside the call, so
  61. the default 30s truncates large batches mid-upload (phantom failure). 5s/attachment, 60s floor."""
  62. return 30.0 if num_attachments <= 0 else max(60.0, 5.0 * num_attachments)
  63. class SignalAttachmentScheduler:
  64. """Process-wide token-bucket simulator for Signal attachment sends: up to ``capacity`` tokens (default
  65. 50 = Signal's server bucket), one per attachment, refilling at ``refill_rate``/s (calibrated from the
  66. server's per-token Retry-After once a 429 has been observed; default 1 token / 4s). ``acquire(n)``
  67. calls serialize through an ``asyncio.Lock`` — FIFO across sessions."""
  68. def __init__(self, capacity: float = float(SIGNAL_RATE_LIMIT_BUCKET_CAPACITY),
  69. default_retry_after: float = float(SIGNAL_RATE_LIMIT_DEFAULT_RETRY_AFTER)) -> None:
  70. self.capacity = float(capacity)
  71. self.tokens = float(capacity)
  72. self.refill_rate = 1.0 / float(default_retry_after)
  73. self.last_refill = time.monotonic()
  74. self._lock = asyncio.Lock()
  75. def _projected_tokens(self, now: Optional[float] = None) -> float:
  76. """Tokens the bucket would hold at ``now``, without mutating state."""
  77. elapsed = (time.monotonic() if now is None else now) - self.last_refill
  78. if elapsed > 0 and self.tokens < self.capacity:
  79. return min(self.capacity, self.tokens + elapsed * self.refill_rate)
  80. return self.tokens
  81. def _refill(self) -> None:
  82. now = time.monotonic()
  83. self.tokens = self._projected_tokens(now)
  84. self.last_refill = now
  85. def estimate_wait(self, n: int) -> float:
  86. """Seconds until ``n`` tokens would be available (lock-free, informational — decides whether to emit
  87. a pacing notice *before* a possibly-blocking ``acquire``; races vs. concurrent acquires are benign)."""
  88. deficit = n - self._projected_tokens()
  89. return 0.0 if deficit <= 0 else deficit / self.refill_rate
  90. async def acquire(self, n: int) -> float:
  91. """Block until at least ``n`` tokens are available; return the seconds slept. Does **not** deduct
  92. tokens — the bucket is a read-only model of server-side capacity; call ``report_rpc_duration()``
  93. after the RPC to sync. The lock is released during ``asyncio.sleep`` so other callers interleave,
  94. and the loop re-checks after each sleep in case the deadline was pessimistic. Signal's server is
  95. ground truth and will 429 (→ requeue) if the model drifts."""
  96. if n <= 0:
  97. return 0.0
  98. if n > self.capacity:
  99. raise SignalSchedulerError(f"Signal scheduler was called requesting {n} tokens (max is {self.capacity})")
  100. total_slept, first_pass = 0.0, True
  101. while True:
  102. async with self._lock:
  103. self._refill()
  104. if self.tokens >= n:
  105. if not first_pass or total_slept > 0:
  106. logger.debug("Signal scheduler: tokens sufficient for %d (remaining=%.1f, total_slept=%.1fs)",
  107. n, self.tokens, total_slept)
  108. return total_slept
  109. deficit = n - self.tokens
  110. wait = deficit / self.refill_rate
  111. if first_pass:
  112. logger.info("Signal scheduler: pausing %.1fs for %d tokens (available=%.1f, deficit=%.1f, "
  113. "refill=%.4f/s ≈ %.1fs/token)", wait, n, self.tokens, deficit, self.refill_rate,
  114. 1.0 / self.refill_rate)
  115. first_pass = False
  116. await asyncio.sleep(wait)
  117. total_slept += wait
  118. async def report_rpc_duration(self, rpc_duration: float, n_attachments: int) -> None:
  119. """Deduct ``n_attachments`` tokens for a completed send RPC. No refill is credited for the upload
  120. window: Signal's server checks the bucket at RPC start and resumes refill only after the response,
  121. so crediting it causes cumulative drift that eventually triggers 429s. Advances ``last_refill``."""
  122. if n_attachments <= 0:
  123. return
  124. async with self._lock:
  125. token_before = self.tokens
  126. self.tokens = max(0.0, token_before - float(n_attachments))
  127. self.last_refill = time.monotonic()
  128. logger.log(logging.INFO if rpc_duration > 10 and n_attachments > 5 else logging.DEBUG,
  129. "Signal scheduler: RPC for %d att took %.1fs — tokens %.1f → %.1f (deducted=%d, no upload refill "
  130. "credited, refill=%.4fs⁻¹)", n_attachments, rpc_duration, token_before, self.tokens, n_attachments,
  131. self.refill_rate)
  132. def feedback(self, retry_after: Optional[float], n_attempted: int) -> None:
  133. """Apply server feedback after a 429: empty the bucket and, when ``retry_after`` (per-token refill
  134. window) is present, calibrate ``refill_rate`` from it."""
  135. if retry_after and retry_after > 0 and (new_rate := 1.0 / float(retry_after)) != self.refill_rate:
  136. logger.info("Signal scheduler: calibrating refill_rate to %.4f tokens/sec (server retry_after=%.1fs "
  137. "per token)", new_rate, retry_after)
  138. self.refill_rate = new_rate
  139. self.tokens = 0.0
  140. self.last_refill = time.monotonic()
  141. def state(self) -> dict:
  142. """Current scheduler state for diagnostic logging (read-only, doesn't advance ``last_refill``)."""
  143. return {"tokens": round(self._projected_tokens(), 1), "capacity": int(self.capacity),
  144. "refill_rate": round(self.refill_rate, 4),
  145. "refill_seconds_per_token": round(1.0 / self.refill_rate, 1) if self.refill_rate > 0 else float("inf")}
  146. _scheduler: Optional[SignalAttachmentScheduler] = None
  147. def get_scheduler() -> SignalAttachmentScheduler:
  148. """Return the process-wide scheduler, creating it on first access."""
  149. global _scheduler
  150. if _scheduler is None:
  151. _scheduler = SignalAttachmentScheduler()
  152. logger.info("Signal scheduler: created (capacity=%d tokens, refill=%.4f/s ≈ %.1fs/token)",
  153. int(_scheduler.capacity), _scheduler.refill_rate, 1.0 / _scheduler.refill_rate)
  154. return _scheduler
  155. def _reset_scheduler() -> None:
  156. """Drop the cached scheduler so the next ``get_scheduler`` builds a fresh one. Test-only."""
  157. global _scheduler
  158. _scheduler = None