account_usage_cache.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. """What ``model.options`` knows about subscription usage without waiting on the network.
  2. Usage windows come from a provider API (:func:`agent.account_usage.fetch_account_usage`), so a
  3. picker reads this cache and asks for a background refresh instead of fetching inline. Every fetch
  4. lands here, including the one ``session.usage`` runs after each turn, so a picker is at most one
  5. turn or one refresh interval behind.
  6. Slots are keyed per profile home AND per account: one process may serve many profiles, and a
  7. provider may have a pool of credentials whose usage must not contaminate each other. A fetcher
  8. that can identify its account (a decoded Codex JWT principal) dedupes several credentials of one
  9. account onto a single slot; fetchers that cannot get one slot per credential, never a
  10. provider-wide singleton. The legacy provider slot (``identity_id=None``) remains the single-account
  11. gauge: a per-turn or ``/usage`` fetch still lands there, so non-pooled chips are unchanged.
  12. """
  13. from __future__ import annotations
  14. import threading
  15. import time
  16. from contextvars import copy_context
  17. from typing import TYPE_CHECKING, Any, Iterable, Optional
  18. if TYPE_CHECKING:
  19. from agent.account_usage import AccountUsageSnapshot
  20. # A picker opened again within this long reuses what it has; usage windows move in minutes, not seconds.
  21. REFRESH_AFTER_S = 120.0
  22. # A snapshot older than this renders as state ``unknown`` instead of a confident gauge: a benched
  23. # credential's windows stop updating, and reporting stale numbers as live would mislabel an
  24. # account that may have recovered (or worsened) in the meantime.
  25. SNAPSHOT_STALE_AFTER_S = 15 * 60.0
  26. _lock = threading.Lock()
  27. # slot key (home, provider, identity_id, base_url) → snapshot
  28. _snapshots: dict[tuple[str, str, str, str], AccountUsageSnapshot] = {}
  29. # slot key → monotonic time the stored snapshot's fetch STARTED (late-fetch guard)
  30. _started: dict[tuple[str, str, str, str], float] = {}
  31. # slot key → monotonic time of the last refresh try (success or failure)
  32. _last_try: dict[tuple[str, str, str, str], float] = {}
  33. _inflight: set[tuple[str, str, str, str]] = set()
  34. def _key(provider: str, identity_id: Optional[str] = None, base_url: Optional[str] = None) -> tuple[str, str, str, str]:
  35. from hermes_constants import hermes_home_key
  36. return (
  37. hermes_home_key(),
  38. str(provider or "").strip().lower(),
  39. str(identity_id or "").strip(),
  40. str(base_url or "").strip().rstrip("/"),
  41. )
  42. def has_account_usage(provider: str) -> bool:
  43. """Whether *provider* can report usage windows at all (a built-in fetcher or a plugin hook)."""
  44. from agent.account_usage import _USAGE_FETCHERS
  45. from providers import get_provider_profile
  46. from providers.base import ProviderProfile
  47. slug = str(provider or "").strip().lower()
  48. if slug in _USAGE_FETCHERS:
  49. return True
  50. profile = get_provider_profile(slug) if slug else None
  51. return profile is not None and type(profile).fetch_account_usage is not ProviderProfile.fetch_account_usage
  52. def _identity_id_for(provider: str, entry: Any) -> str:
  53. """Stable non-secret account id of one pool entry, for slot keys and wire ``accounts[].id``.
  54. A trusted decoded identity (Codex JWT principal) wins: several credentials of one account
  55. share a slot, so the account is counted once. Without one, the entry's token fingerprint
  56. stands in — same token ⇒ same account, and a replaced token never reuses the old quota.
  57. A metadata-only row (no token) falls back to the entry id: nothing better is knowable.
  58. """
  59. token = str(getattr(entry, "runtime_api_key", "") or "").strip()
  60. if not token:
  61. return f"entry:{getattr(entry, 'id', '')}"
  62. if provider == "openai-codex":
  63. from agent.credential_pool import _codex_principal_identity
  64. principal = _codex_principal_identity(token)
  65. if principal:
  66. return f"codex:{principal[0]}:{principal[1]}"
  67. import hashlib
  68. return f"fp:{hashlib.sha256(token.encode('utf-8')).hexdigest()[:16]}"
  69. def remember_account_usage(
  70. provider: Optional[str], snapshot: Optional[AccountUsageSnapshot],
  71. *, identity_id: Optional[str] = None, base_url: Optional[str] = None,
  72. started_monotonic: Optional[float] = None,
  73. ) -> None:
  74. """Land one fetch result.
  75. ``identity_id`` given (per-credential fetch): the snapshot is stored under that account's
  76. slot only — one pool entry's usage must never pose as the provider-wide gauge. The snapshot's
  77. own trusted ``identity`` (decoded from the very token it was fetched with) wins over the
  78. caller's id, so a replaced credential cannot repopulate its old account's slot.
  79. ``identity_id`` absent (per-turn / ``/usage`` fetch): stored in the provider's legacy slot —
  80. the single-account gauge — and additionally in the account's identity slot when the fetcher
  81. identified it, so a pooled picker sees per-turn data without waiting on its own refresh.
  82. A snapshot without windows is never stored: a failed or empty fetch must not erase a fresher
  83. snapshot. A fetch that STARTED before the freshest stored one cannot replace it either
  84. (``started_monotonic``), however late its response arrives.
  85. """
  86. if not provider or snapshot is None or not snapshot.windows:
  87. return
  88. started = time.monotonic() if started_monotonic is None else started_monotonic
  89. with _lock:
  90. targets = []
  91. snapshot_identity = getattr(snapshot, "identity", None)
  92. if identity_id is None:
  93. targets.append(_key(provider, None, base_url))
  94. if snapshot_identity:
  95. targets.append(_key(provider, snapshot_identity, None))
  96. else:
  97. targets.append(_key(provider, snapshot_identity or identity_id, None))
  98. for key in targets:
  99. if key in _started and _started[key] > started:
  100. continue # a fresher fetch already landed; this late result must not replace it
  101. _snapshots[key] = snapshot
  102. _started[key] = started
  103. def cached_account_usage(
  104. provider: str, *, identity_id: Optional[str] = None, base_url: Optional[str] = None,
  105. ) -> Optional[AccountUsageSnapshot]:
  106. with _lock:
  107. return _snapshots.get(_key(provider, identity_id, base_url))
  108. def snapshot_is_stale(snapshot: Optional[AccountUsageSnapshot]) -> bool:
  109. """True when the snapshot is too old to render as a live gauge (the picker shows state
  110. ``unknown`` instead of trusting its percentages)."""
  111. from agent.account_usage import AccountUsageSnapshot as _Snapshot
  112. if not isinstance(snapshot, _Snapshot):
  113. return True
  114. try:
  115. age = time.time() - snapshot.fetched_at.timestamp()
  116. except (AttributeError, TypeError, ValueError, OSError, OverflowError):
  117. return True
  118. return age < 0 or age > SNAPSHOT_STALE_AFTER_S
  119. def _claim_refresh(key: tuple[str, str, str, str], now: float) -> bool:
  120. """Take the refresh right for *key* unless one is inflight or tried too recently.
  121. A failed or empty fetch still counts as a try, so a dead credential (or a provider with no
  122. usage API) isn't re-asked on every picker open."""
  123. with _lock:
  124. if key in _inflight or now - _last_try.get(key, float("-inf")) < REFRESH_AFTER_S:
  125. return False
  126. _inflight.add(key)
  127. _last_try[key] = now
  128. return True
  129. def _release_refresh(key: tuple[str, str, str, str]) -> None:
  130. with _lock:
  131. _inflight.discard(key)
  132. def refresh_account_usage_async(providers: Iterable[str]) -> list[threading.Thread]:
  133. """Background-refresh the provider-wide (legacy) slot for each provider not tried within
  134. ``REFRESH_AFTER_S``. Returns the started threads (daemon; callers never join them)."""
  135. started: list[threading.Thread] = []
  136. now = time.monotonic()
  137. for provider in dict.fromkeys(providers):
  138. key = _key(provider)
  139. if not _claim_refresh(key, now):
  140. continue
  141. thread = threading.Thread(
  142. target=copy_context().run, args=(_refresh_legacy, provider, key),
  143. name="hermes-account-usage-refresh", daemon=True)
  144. thread.start()
  145. started.append(thread)
  146. return started
  147. def _refresh_legacy(provider: str, key: tuple[str, str, str, str]) -> None:
  148. from agent.account_usage import fetch_account_usage
  149. try:
  150. fetch_account_usage(provider) # remembers its own result
  151. finally:
  152. _release_refresh(key)
  153. def refresh_account_usage_entries_async(requests: Iterable[dict]) -> list[threading.Thread]:
  154. """Background-refresh per-credential usage slots. *requests* entries are
  155. ``{"provider", "identity_id", "base_url", "api_key"}`` — the live credentials to read.
  156. One worker thread fetches all due entries serially (no thread storm for a 10-credential
  157. pool), read-only: a 401 or any other failure is reported, never repaired by rotating or
  158. refreshing the credential. Throttled per account slot, failures included."""
  159. due: list[dict] = []
  160. now = time.monotonic()
  161. for request in requests:
  162. provider = str(request.get("provider") or "").strip().lower()
  163. identity_id = str(request.get("identity_id") or "").strip()
  164. if not provider or not identity_id or not str(request.get("api_key") or "").strip():
  165. continue
  166. if not _claim_refresh(_key(provider, identity_id), now):
  167. continue
  168. due.append(request)
  169. if not due:
  170. return []
  171. thread = threading.Thread(
  172. target=copy_context().run, args=(_refresh_entries, list(due)),
  173. name="hermes-account-usage-refresh", daemon=True)
  174. thread.start()
  175. return [thread]
  176. def _refresh_entries(requests: list[dict]) -> None:
  177. from agent.account_usage import fetch_account_usage
  178. for request in requests:
  179. provider = request["provider"]
  180. key = _key(provider, request["identity_id"])
  181. try:
  182. fetch_account_usage(
  183. provider, base_url=request.get("base_url"), api_key=request.get("api_key"),
  184. read_only=True, identity_id=request["identity_id"],
  185. )
  186. finally:
  187. _release_refresh(key)