model_metadata_memo_bounds.py 1.8 KB

123456789101112131415161718192021222324252627282930313233343536
  1. """Memory bounds for ``agent.model_metadata``'s per-credential in-process memos.
  2. The memos are keyed by a credential fingerprint, and minted keys (Nous agent keys, OAuth access
  3. tokens) rotate, so each rotation adds a key the process never reads again. Only entries no reader
  4. can use are shed: expired ones, or the oldest past a count far above any real working set.
  5. """
  6. import threading
  7. import time
  8. from typing import Any
  9. from agent.cache_bounds import drop_expired, evict_oldest
  10. # Same bound as ``agent.codex_service_tiers._MAX_CATALOGS`` for the same token-fingerprint keys.
  11. CODEX_CACHE_MAX = 128
  12. # Every writer of a paired memo (value + stamp, context + max-context) holds this from sweep to publish:
  13. # a sweep pops two tables, and a refresh landing between the pops would keep half an entry.
  14. MEMO_LOCK = threading.Lock()
  15. def remember_endpoint_models(memo_key: tuple[str, str], cache: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
  16. """Publish a ``/models`` catalog, sweeping expired ones (the reader refetches those anyway)."""
  17. from agent import model_metadata as mm # facade seam: tests swap the memo dicts there
  18. with MEMO_LOCK:
  19. now = time.time()
  20. drop_expired(mm._endpoint_model_metadata_cache, mm._endpoint_model_metadata_cache_time, mm._ENDPOINT_MODEL_CACHE_TTL, now)
  21. mm._endpoint_model_metadata_cache[memo_key] = cache
  22. mm._endpoint_model_metadata_cache_time[memo_key] = now
  23. return cache
  24. def bound_codex_caches(cache_key: str) -> None:
  25. """Before storing ``cache_key``, evict the stalest Codex token entry from both context memos.
  26. Caller holds ``MEMO_LOCK`` through its publish."""
  27. from agent import model_metadata as mm # facade seam: tests swap the memo dicts there
  28. evict_oldest(mm._codex_oauth_context_cache, cache_key, lambda entry: entry[1], CODEX_CACHE_MAX, mm._codex_oauth_max_context_cache)