prompt_cache_boundary.py 2.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758
  1. """Builder-declared stable prefixes for Anthropic prompt caching.
  2. Skill/webhook/cron builders concatenate a large static scaffold with a small volatile
  3. tail into one user-message string. Only the builder knows where the tail begins, so it
  4. registers the stable prefix here and the cache planner places a breakpoint at that
  5. boundary. Re-parsing marker strings at request time is deliberately avoided: markers
  6. can legitimately appear inside skill bodies, and any delimiter heuristic then shrinks
  7. the cached prefix or absorbs volatile bytes. Process-local by design: any miss falls
  8. back to the whole-message policy.
  9. """
  10. import threading
  11. from collections import OrderedDict
  12. from typing import Optional
  13. # A couple dozen active scaffolds is generous for one gateway process.
  14. _MAX_ENTRIES = 32
  15. # Entries hold whole expanded skill bodies, so also bound total retained chars.
  16. # The newest entry is always kept so one oversized scaffold still gets a boundary.
  17. _MAX_CHARS = 4 * 1024 * 1024
  18. _lock = threading.Lock()
  19. _prefixes: OrderedDict[str, None] = OrderedDict()
  20. def register_stable_prefix(prefix: str) -> None:
  21. """Record ``prefix`` as the stable scaffold of a just-built message."""
  22. if not prefix:
  23. return
  24. with _lock:
  25. _prefixes[prefix] = None
  26. _prefixes.move_to_end(prefix)
  27. while len(_prefixes) > _MAX_ENTRIES:
  28. _prefixes.popitem(last=False)
  29. while len(_prefixes) > 1 and sum(map(len, _prefixes)) > _MAX_CHARS:
  30. _prefixes.popitem(last=False)
  31. def find_stable_prefix(content: str) -> Optional[str]:
  32. """Longest registered *proper* prefix of ``content`` with a non-whitespace tail.
  33. The tail must be non-whitespace so the split never yields an empty text
  34. block (Anthropic rejects it with HTTP 400). A hit refreshes the entry's LRU
  35. position so a scaffold fired every minute by cron is not evicted by a
  36. burst of one-off skill invocations.
  37. """
  38. with _lock:
  39. best: Optional[str] = None
  40. for prefix in _prefixes:
  41. if (
  42. content.startswith(prefix)
  43. and content[len(prefix):].strip()
  44. and (best is None or len(prefix) > len(best))
  45. ):
  46. best = prefix
  47. if best is not None:
  48. _prefixes.move_to_end(best) # after the scan: never mutate mid-iteration
  49. return best