model_metadata_token_memo.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. """Storage and release for the per-message token-estimate memo (``agent.model_metadata``).
  2. Each entry pins every string its fingerprint holds (the pin keeps the ``id()`` key sound), so the memo is
  3. bounded two ways, both least-recently-used first so a conversation that is still running never pays:
  4. * a retained-bytes budget: pinned string bytes plus the fingerprint structure around them;
  5. * an entry count, the same 4,096 the memo always had.
  6. Both bounds only govern IDLE entries (below), so they are set to hold no more than main's 4,096-entry memo
  7. ever did: what a running conversation needs is protected by the active window, never by the budget.
  8. Neither bound ever evicts an entry looked up in the last ``_ACTIVE_WINDOW_S`` seconds: a conversation larger
  9. than the budget (image-heavy history) overshoots it while it runs instead of missing on every message, and the
  10. overshoot is shed by the first insert after it goes idle. The protected entries pin strings the running
  11. conversations hold anyway, so the overshoot costs only fingerprint structure.
  12. ``close()``, compaction and the compressor telemetry also drop entries they know are dead
  13. (``forget_message_token_memo``). Eviction never changes an estimate: a later lookup recomputes the same value.
  14. """
  15. from __future__ import annotations
  16. import sys
  17. import threading
  18. import time
  19. from collections import OrderedDict
  20. from typing import Any
  21. # fingerprint -> (pins, text tokens, image count). Order is recency: a hit moves its entry to the end, and an
  22. # OrderedDict evicts from the front in O(1) where a plain dict rescans its deleted slots on every eviction.
  23. _MSG_TOKENS_CACHE: OrderedDict[Any, tuple[list, int, int]] = OrderedDict()
  24. _MSG_TOKENS_CACHE_MAX = 4096
  25. _MSG_TOKENS_CACHE_MAX_BYTES = 16 * 1024 * 1024
  26. # Measured on real histories: the fingerprint tuple, its id int and the pin slot cost ~120 B per string leaf.
  27. _LEAF_OVERHEAD_BYTES = 120
  28. # An entry looked up this recently is never evicted (see module docstring).
  29. _ACTIVE_WINDOW_S = 60.0
  30. _clock = time.monotonic
  31. _lock = threading.Lock()
  32. _retained_bytes = 0
  33. class MemoPins(list):
  34. """An entry's pinned strings, stamped with the clock time of the entry's last lookup."""
  35. __slots__ = ("touched",)
  36. def lookup_message_tokens(key: Any) -> tuple[list, int, int] | None:
  37. """Return the entry for ``key`` (or None), marking it most recently used.
  38. The stamp and reorder share eviction's lock: a hit landing between the evictor's idle check of the
  39. oldest entry and its pop would otherwise move that entry to the end, so the pop took a different,
  40. active one."""
  41. with _lock:
  42. entry = _MSG_TOKENS_CACHE.get(key)
  43. if entry is not None:
  44. entry[0].touched = _clock()
  45. _MSG_TOKENS_CACHE.move_to_end(key)
  46. return entry
  47. def _entry_bytes(entry: tuple[list, int, int]) -> int:
  48. pins = entry[0]
  49. return sum(map(sys.getsizeof, pins)) + _LEAF_OVERHEAD_BYTES * len(pins)
  50. def remember_message_tokens(key: Any, entry: tuple[MemoPins, int, int]) -> None:
  51. """Insert ``entry`` as most recent, then evict least-recently-used entries past either bound, stopping at
  52. the first one looked up inside the active window (everything after it is more recent)."""
  53. global _retained_bytes
  54. size = _entry_bytes(entry)
  55. if size > _MSG_TOKENS_CACHE_MAX_BYTES:
  56. return # one message larger than the whole budget is recomputed, never allowed to flush the memo
  57. cache = _MSG_TOKENS_CACHE
  58. now = _clock()
  59. entry[0].touched = now
  60. idle_before = now - _ACTIVE_WINDOW_S
  61. with _lock:
  62. if not cache: # an emptied memo (tests clear it) restarts the count
  63. _retained_bytes = 0
  64. previous = cache.pop(key, None)
  65. if previous is not None:
  66. _retained_bytes -= _entry_bytes(previous)
  67. cache[key] = entry
  68. _retained_bytes += size
  69. while len(cache) > 1 and (len(cache) > _MSG_TOKENS_CACHE_MAX or _retained_bytes > _MSG_TOKENS_CACHE_MAX_BYTES):
  70. if next(iter(cache.values()))[0].touched >= idle_before:
  71. break
  72. _retained_bytes -= _entry_bytes(cache.popitem(last=False)[1])
  73. def retained_bytes() -> int:
  74. """Current retained-bytes estimate of the memo (what the budget is compared against)."""
  75. return _retained_bytes
  76. def _long_string_ids(messages: Any) -> set[int]:
  77. ids: set[int] = set()
  78. stack = [messages]
  79. while stack:
  80. value = stack.pop()
  81. if type(value) is dict:
  82. stack.extend(value.values())
  83. elif type(value) is list or type(value) is tuple:
  84. stack.extend(value)
  85. elif type(value) is str and len(value) > 64: # short values ("assistant", tool names) are shared
  86. ids.add(id(value))
  87. return ids
  88. def forget_message_token_memo(messages: Any, *, keep: Any = None) -> int:
  89. """Evict memo entries pinning any long string value of ``messages`` that ``keep`` does not also hold;
  90. return how many.
  91. Eviction never changes an estimate: a later lookup recomputes the same value.
  92. """
  93. global _retained_bytes
  94. owned = _long_string_ids(messages)
  95. if owned and keep is not None:
  96. owned -= _long_string_ids(keep)
  97. if not owned:
  98. return 0
  99. with _lock:
  100. stale = [key for key, (pins, _text, _images) in list(_MSG_TOKENS_CACHE.items())
  101. if any(id(pin) in owned for pin in pins)]
  102. for key in stale:
  103. _retained_bytes -= _entry_bytes(_MSG_TOKENS_CACHE.pop(key))
  104. return len(stale)