rate_limit_tracker.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. """Rate limit tracking for inference API responses.
  2. Captures ``x-ratelimit-{limit,remaining,reset}-{requests,tokens}[-1h]``
  3. headers (Nous Portal format, also used by OpenRouter / OpenAI-compatible APIs)
  4. and formats them for the /usage slash command. Reset values are seconds.
  5. """
  6. from __future__ import annotations
  7. import time
  8. from dataclasses import dataclass, field
  9. from typing import Any, Mapping, Optional
  10. # (state attribute, header tag) for the four windows.
  11. _BUCKET_TAGS = (
  12. ("requests_min", "requests"),
  13. ("requests_hour", "requests-1h"),
  14. ("tokens_min", "tokens"),
  15. ("tokens_hour", "tokens-1h"),
  16. )
  17. @dataclass
  18. class RateLimitBucket:
  19. """One rate-limit window (e.g. requests per minute)."""
  20. limit: int = 0
  21. remaining: int = 0
  22. reset_seconds: float = 0.0
  23. captured_at: float = 0.0 # time.time() when this was captured
  24. @property
  25. def used(self) -> int:
  26. return max(0, self.limit - self.remaining)
  27. @property
  28. def usage_pct(self) -> float:
  29. return (self.used / self.limit) * 100.0 if self.limit > 0 else 0.0
  30. @property
  31. def remaining_seconds_now(self) -> float:
  32. """Estimated seconds remaining until reset, adjusted for elapsed time."""
  33. return max(0.0, self.reset_seconds - (time.time() - self.captured_at))
  34. @dataclass
  35. class RateLimitState:
  36. """Full rate-limit state parsed from response headers."""
  37. requests_min: RateLimitBucket = field(default_factory=RateLimitBucket)
  38. requests_hour: RateLimitBucket = field(default_factory=RateLimitBucket)
  39. tokens_min: RateLimitBucket = field(default_factory=RateLimitBucket)
  40. tokens_hour: RateLimitBucket = field(default_factory=RateLimitBucket)
  41. captured_at: float = 0.0 # when the headers were captured
  42. provider: str = ""
  43. @property
  44. def has_data(self) -> bool:
  45. return self.captured_at > 0
  46. @property
  47. def age_seconds(self) -> float:
  48. return time.time() - self.captured_at if self.has_data else float("inf")
  49. def _safe_float(value: Any, default: Any = 0.0) -> Any:
  50. try:
  51. return float(value)
  52. except (TypeError, ValueError):
  53. return default
  54. def _safe_int(value: Any, default: Any = 0) -> Any:
  55. try:
  56. return int(float(value))
  57. except (TypeError, ValueError):
  58. return default
  59. def lower_headers(headers: Optional[Mapping[str, str]]) -> dict[str, str]:
  60. """Lowercase header names (HTTP header names are case-insensitive)."""
  61. return {k.lower(): v for k, v in headers.items()} if headers else {}
  62. def has_rate_limit_headers(lowered: Mapping[str, str]) -> bool:
  63. return any(k.startswith("x-ratelimit-") for k in lowered)
  64. def parse_rate_limit_headers(headers: Mapping[str, str], provider: str = "") -> Optional[RateLimitState]:
  65. """Parse x-ratelimit-* headers into a RateLimitState (None if none present)."""
  66. lowered = lower_headers(headers)
  67. if not has_rate_limit_headers(lowered):
  68. return None
  69. now = time.time()
  70. buckets = {
  71. attr: RateLimitBucket(
  72. limit=_safe_int(lowered.get(f"x-ratelimit-limit-{tag}")),
  73. remaining=_safe_int(lowered.get(f"x-ratelimit-remaining-{tag}")),
  74. reset_seconds=_safe_float(lowered.get(f"x-ratelimit-reset-{tag}")),
  75. captured_at=now,
  76. )
  77. for attr, tag in _BUCKET_TAGS
  78. }
  79. return RateLimitState(captured_at=now, provider=provider, **buckets)
  80. # ── Formatting ──────────────────────────────────────────────────────────
  81. def _fmt_count(n: int) -> str:
  82. """Human-friendly number: 7999856 -> '8.0M', 33599 -> '33.6K', 799 -> '799'."""
  83. if n >= 1_000_000:
  84. return f"{n / 1_000_000:.1f}M"
  85. if n >= 1_000:
  86. return f"{n / 1_000:.1f}K"
  87. return str(n)
  88. def _fmt_seconds(seconds: float) -> str:
  89. """Seconds -> human-friendly duration: '58s', '2m 14s', '58m 57s', '1h 2m'."""
  90. s = max(0, int(seconds))
  91. if s < 60:
  92. return f"{s}s"
  93. if s < 3600:
  94. m, sec = divmod(s, 60)
  95. return f"{m}m {sec}s" if sec else f"{m}m"
  96. h, m = divmod(s, 3600)
  97. m //= 60
  98. return f"{h}h {m}m" if m else f"{h}h"
  99. def _bar(pct: float, width: int = 20) -> str:
  100. """ASCII progress bar: [████████░░░░░░░░░░░░] 40%."""
  101. filled = max(0, min(width, int(pct / 100.0 * width)))
  102. return f"[{'█' * filled}{'░' * (width - filled)}]"
  103. def _bucket_line(label: str, bucket: RateLimitBucket, label_width: int = 14) -> str:
  104. """Format one bucket as a single line."""
  105. if bucket.limit <= 0:
  106. return f" {label:<{label_width}} (no data)"
  107. pct = bucket.usage_pct
  108. used, limit, remaining = map(_fmt_count, (bucket.used, bucket.limit, bucket.remaining))
  109. reset = _fmt_seconds(bucket.remaining_seconds_now)
  110. return f" {label:<{label_width}} {_bar(pct)} {pct:5.1f}% {used}/{limit} used ({remaining} left, resets in {reset})"
  111. def format_rate_limit_display(state: RateLimitState) -> str:
  112. """Format rate limit state for terminal/chat display."""
  113. if not state.has_data:
  114. return "No rate limit data yet — make an API request first."
  115. age = state.age_seconds
  116. freshness = "just now" if age < 5 else f"{int(age)}s ago" if age < 60 else f"{_fmt_seconds(age)} ago"
  117. provider_label = state.provider.title() if state.provider else "Provider"
  118. labeled = [("Requests/min", state.requests_min), ("Requests/hr", state.requests_hour),
  119. ("Tokens/min", state.tokens_min), ("Tokens/hr", state.tokens_hour)]
  120. lines = [f"{provider_label} Rate Limits (captured {freshness}):", ""]
  121. lines += [_bucket_line(label, bucket) for label, bucket in labeled[:2]]
  122. lines += [""] + [_bucket_line(label, bucket) for label, bucket in labeled[2:]]
  123. warnings = [
  124. f" ⚠ {label.lower()} at {bucket.usage_pct:.0f}% — resets in {_fmt_seconds(bucket.remaining_seconds_now)}"
  125. for label, bucket in labeled
  126. if bucket.limit > 0 and bucket.usage_pct >= 80
  127. ]
  128. if warnings:
  129. lines += [""] + warnings
  130. return "\n".join(lines)
  131. def format_rate_limit_compact(state: RateLimitState) -> str:
  132. """One-line compact summary for status bars / gateway messages."""
  133. if not state.has_data:
  134. return "No rate limit data."
  135. # (tag, bucket, count formatter, show reset) — RPM stays raw digits, hourly windows show the reset.
  136. windows = (
  137. ("RPM", state.requests_min, str, False),
  138. ("RPH", state.requests_hour, _fmt_count, True),
  139. ("TPM", state.tokens_min, _fmt_count, False),
  140. ("TPH", state.tokens_hour, _fmt_count, True),
  141. )
  142. return " | ".join(
  143. f"{tag}: {fmt(b.remaining)}/{fmt(b.limit)}" + (f" (resets {_fmt_seconds(b.remaining_seconds_now)})" if reset else "")
  144. for tag, b, fmt, reset in windows if b.limit > 0
  145. )