prompt_caching.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. """Anthropic prompt caching strategy — pure functions, no AIAgent dependency.
  2. Default layout: 4 cache_control breakpoints — the static system prefix, the end of the
  3. system prompt, and the last 2 non-system messages (without a static prefix: one system
  4. breakpoint plus the last 3 messages). All markers share one TTL (5m or 1h).
  5. """
  6. import copy
  7. from dataclasses import dataclass
  8. from typing import Any, Dict, List
  9. from agent.prompt_cache_boundary import find_stable_prefix
  10. @dataclass(frozen=True)
  11. class PromptCachePlan:
  12. """Request-local message and tool sections with their cache markers."""
  13. messages: list[dict[str, Any]]
  14. tools: list[dict[str, Any]]
  15. def envelope_tool_part_cache_markers_supported(provider: str | None, base_url: str | None) -> bool:
  16. """Whether the envelope-layout route honors part-level markers on role:tool.
  17. OpenRouter/Nous Portal relocate a part-level ``cache_control`` onto the ``tool_result``
  18. block; LiteLLM-style proxies copy parts verbatim, so it lands at ``tool_result.content[0]``
  19. (non-retryable 400). There, role:tool carries no part markers and the budget reallocates.
  20. """
  21. from agent.agent_runtime_helpers import _is_litellm_route
  22. return not _is_litellm_route((provider or "").strip().lower(), base_url or "")
  23. def is_one_shot_request(messages: list[dict[str, Any]] | None, tools: list[dict[str, Any]] | None) -> bool:
  24. """True for a request no later call can read back from cache: no tools, no assistant/tool
  25. turns, at most one user turn (compression summaries, titles, MoA synthesis). A marker there
  26. only buys a cache write (1.25x input at 5m, 2x at 1h) that nothing ever reads. A caller that
  27. already decorated the request (a MoA advisor's first iteration, whose prefix later
  28. iterations replay) decided it will reuse it, so that decision wins over the shape."""
  29. if tools:
  30. return False
  31. msgs = [m for m in messages or [] if isinstance(m, dict)]
  32. if any("cache_control" in m or _has_part_marker(m.get("content")) for m in msgs):
  33. return False
  34. roles = [m.get("role") for m in msgs]
  35. return roles.count("user") <= 1 and not any(r in ("assistant", "tool") for r in roles)
  36. def _text_part(text: str, cache_marker: dict | None = None) -> dict:
  37. part: dict = {"type": "text", "text": text}
  38. if cache_marker is not None:
  39. part["cache_control"] = cache_marker
  40. return part
  41. def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool = False,
  42. tool_part_markers: bool = True) -> None:
  43. """Add cache_control to a single message, handling all format variations."""
  44. role = msg.get("role", "")
  45. content = msg.get("content")
  46. if role == "tool" and not native_anthropic and not tool_part_markers:
  47. # LiteLLM-style envelope: a part marker → tool_result.content[0] → non-retryable 400.
  48. return
  49. if (role == "tool" and native_anthropic) or content is None or content == "":
  50. # Native role:tool: top-level marker, the adapter moves it inside tool_result. Empty
  51. # content: no part can carry it, and OpenRouter rejects a top-level marker on role:tool
  52. # (silent hang) and ignores it on empty assistant turns — skip those on the envelope.
  53. if not (role in ("tool", "assistant") and not native_anthropic):
  54. msg["cache_control"] = cache_marker
  55. elif isinstance(content, str):
  56. # Builder-declared boundary: the scaffold carries the breakpoint and the volatile tail
  57. # rides unmarked. Request-local only — the stored message stays a string.
  58. msg["content"] = _scaffold_parts(msg, cache_marker) or [_text_part(content, cache_marker)]
  59. elif isinstance(content, list) and content and isinstance(content[-1], dict):
  60. content[-1]["cache_control"] = cache_marker
  61. def _scaffold_parts(msg: dict, cache_marker: dict | None = None) -> list | None:
  62. """``[scaffold, tail]`` parts of a string user message with a registered scaffold, the
  63. marker (if any) on the scaffold; ``find_stable_prefix`` keeps the tail non-blank."""
  64. content = msg.get("content")
  65. prefix = find_stable_prefix(content) if msg.get("role") == "user" and isinstance(content, str) else None
  66. return None if prefix is None else [_text_part(prefix, cache_marker), _text_part(content[len(prefix):])]
  67. def _split_unmarked_scaffolds(messages: list[dict[str, Any]]) -> None:
  68. """Send every registered scaffold message as [scaffold, tail] parts, marked or not (in place).
  69. Provider caches key on content, and two text parts are different content from one string:
  70. a skill message split while it held a breakpoint must keep the same parts after the rolling
  71. marker window moves past it, or the next request re-writes the whole scaffold (#81867).
  72. Each hit refreshes the scaffold's registry LRU slot, so a scaffold still in history stays
  73. registered and the failover stripper can still recognise its split.
  74. """
  75. for i, msg in enumerate(messages):
  76. if isinstance(msg, dict) and (parts := _scaffold_parts(msg)):
  77. messages[i] = {**msg, "content": parts}
  78. def _can_carry_marker(msg: dict, native_anthropic: bool, tool_part_markers: bool = True) -> bool:
  79. """True if a marker on this message is actually honored by the provider.
  80. Native Anthropic honors every message; the envelope layout only honors markers inside
  81. content parts (empty content wastes a breakpoint) and ``tool_part_markers=False`` excludes
  82. role:tool too (400). Must agree with :func:`_apply_cache_marker` (marks the LAST part).
  83. """
  84. if native_anthropic:
  85. return True
  86. if msg.get("role") == "tool" and not tool_part_markers:
  87. return False
  88. content = msg.get("content")
  89. return isinstance(content[-1], dict) if isinstance(content, list) and content else isinstance(content, str) and content != ""
  90. def _build_marker(ttl: str) -> dict[str, str]:
  91. """Build a cache_control marker dict for the given TTL ('5m' or '1h')."""
  92. return {"type": "ephemeral", "ttl": "1h"} if ttl == "1h" else {"type": "ephemeral"}
  93. # Alibaba-family providers (Qwen routes): five-minute context cache, 1h tier rejected. Shared
  94. # with agent_runtime_helpers.anthropic_prompt_cache_policy so the opt-in and the TTL clamp
  95. # never desync. Do NOT narrow this set to extend a TTL — narrowing DISABLES caching.
  96. ALIBABA_FAMILY_PROVIDERS = frozenset({"opencode", "opencode-go", "opencode-zen", "alibaba"})
  97. # 1h-tier ALLOW-list: only routes wire-measured to retain a 1h marker. Other opencode routes
  98. # are UNMEASURED, not known-bad (opencode-go's `ephemeral_5m_input_tokens` label is not
  99. # evidence of the retention window).
  100. MEASURED_1H_PROVIDERS = frozenset({"opencode-go"})
  101. # Models measured to ignore the 1h tier on a MEASURED_1H_PROVIDERS route; consulted only
  102. # there (the same model on its own endpoint is a separate route).
  103. NO_1H_TIER_MODELS = frozenset({"minimax-m2.5"})
  104. def _flat_model(model: str) -> str:
  105. """Bare model id, tolerating aggregator prefixes (``vendor/model``)."""
  106. return (model or "").strip().rsplit("/", 1)[-1].lower()
  107. def is_qwen_model(model: str) -> bool:
  108. """True when ``model`` names a Qwen-family model (shared with anthropic_prompt_cache_policy)."""
  109. return "qwen" in (model or "").lower()
  110. # ``prompt_caching.cache_ttl: auto`` picks the tier by who paces the session. The 1h tier
  111. # writes at 2x base vs 1.25x for 5m and only pays off when turns are more than five minutes
  112. # apart — a person stepping away and coming back. Machine-paced sessions call every few
  113. # seconds until they finish and never collect the retention, so they stay on 5m.
  114. # Measured on one install (2 days of per-call logs): 63% of interactive cache-write tokens
  115. # were cold re-writes after a 5–60 min idle gap (1h saves ~42% of write cost there), while
  116. # 1h on subagents/cron would have cost ~49% more.
  117. AUTO_CACHE_TTL = "auto"
  118. MACHINE_PACED_SOURCES = frozenset({"subagent", "cron", "oneshot", "webhook", "kanban", "api", "tool", "batch"})
  119. def auto_cache_ttl_for_source(source: str | None) -> str:
  120. """The tier ``auto`` resolves to for a session source: ``5m`` for machine-paced sources,
  121. ``1h`` for everything a human types into (cli, tui, desktop, messaging platforms)."""
  122. return "5m" if (source or "").strip().lower() in MACHINE_PACED_SOURCES else "1h"
  123. def effective_cache_ttl(ttl: str | None, *, model: str = "", provider: str = "") -> str:
  124. """Clamp a requested cache TTL to what the destination route supports (``None`` → ``5m``).
  125. Qwen/Alibaba routes drop ``1h`` (→ ``5m``) except on ``MEASURED_1H_PROVIDERS`` minus
  126. ``NO_1H_TIER_MODELS``; that check runs BEFORE the generic Qwen clamp, which would swallow it.
  127. """
  128. if ttl != "1h":
  129. return ttl or "5m"
  130. provider_lower = (provider or "").lower()
  131. if provider_lower in MEASURED_1H_PROVIDERS:
  132. return "5m" if _flat_model(model) in NO_1H_TIER_MODELS else "1h"
  133. return "5m" if is_qwen_model(model) or provider_lower in ALIBABA_FAMILY_PROVIDERS else "1h"
  134. def _apply_system_cache_markers(
  135. message: dict, cache_marker: dict, static_system_prefix: str | None, *,
  136. native_anthropic: bool, mark_suffix: bool = True, fallback_to_whole: bool = True,
  137. ) -> int:
  138. """Mark the static system prefix (and optionally the full prompt); returns markers applied.
  139. The stored system prompt stays one string, split only in the request. ``mark_suffix=False``
  140. is the tool-cache-plan layout (suffix budget spent on the tools array); ``fallback_to_whole=
  141. False`` marks nothing when the split is impossible. When the prompt IS the prefix the whole
  142. message is one block — never an empty text block (400).
  143. """
  144. content = message.get("content")
  145. if isinstance(static_system_prefix, str) and static_system_prefix and isinstance(content, str) and content.startswith(static_system_prefix):
  146. suffix = content[len(static_system_prefix):]
  147. if suffix.strip():
  148. message["content"] = [_text_part(static_system_prefix, cache_marker),
  149. _text_part(suffix, cache_marker if mark_suffix else None)]
  150. return 2 if mark_suffix else 1
  151. elif not fallback_to_whole:
  152. return 0
  153. _apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
  154. return 1
  155. def _has_part_marker(content: Any) -> bool:
  156. return isinstance(content, list) and any(isinstance(part, dict) and "cache_control" in part for part in content)
  157. def strip_anthropic_cache_control(api_messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  158. """Remove ``cache_control`` markers and undo decoration-produced list shapes (in place).
  159. Used before re-decorating after a mid-turn failover. Flattening to a string is restricted
  160. to the exact shapes :func:`apply_anthropic_cache_control` produces from string content
  161. (single text part, two-part system split, two-part skill split) so the ``""``-join is
  162. byte-exact. Marker removal is copy-on-write on part dicts: parts can alias caller-held
  163. lists and stripping must never rewrite the stored transcript.
  164. """
  165. for msg in api_messages:
  166. if not isinstance(msg, dict):
  167. continue
  168. msg.pop("cache_control", None)
  169. content = msg.get("content")
  170. if not isinstance(content, list):
  171. continue
  172. role = msg.get("role")
  173. user_pair = role == "user" and len(content) == 2 and all(isinstance(p, dict) for p in content)
  174. # The skill split is the only decoration marking the FIRST part of a user message, so the
  175. # marked shape identifies it even after the prefix registry evicted the entry.
  176. first_part_marked = user_pair and "cache_control" in content[0] and "cache_control" not in content[1]
  177. if _has_part_marker(content):
  178. content = msg["content"] = [
  179. {k: v for k, v in part.items() if k != "cache_control"}
  180. if isinstance(part, dict) and "cache_control" in part else part for part in content
  181. ]
  182. plain_text_parts = content and all(
  183. isinstance(part, dict) and part.get("type", "text") == "text"
  184. and isinstance(part.get("text"), str) and set(part.keys()) <= {"type", "text"} for part in content
  185. )
  186. if not plain_text_parts:
  187. continue
  188. joined = "".join(part["text"] for part in content)
  189. # An UNMARKED split (outside the marker window) only the registry tells from a genuine
  190. # two-part user message.
  191. skill_split = first_part_marked or (user_pair and find_stable_prefix(joined) == content[0]["text"])
  192. if len(content) == 1 or (role == "system" and len(content) == 2) or skill_split:
  193. msg["content"] = joined
  194. return api_messages
  195. def _strip_prior_decoration(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  196. """Re-canonicalise planner output before re-planning (in place on the list, copy-on-write per message).
  197. Every list-content message is stripped, not just marked ones: an UNMARKED skill split (outside
  198. the previous request's marker window) left as two parts would get the new breakpoint on its
  199. volatile tail instead of the scaffold. A shallow copy suffices because the strip is
  200. copy-on-write on content parts.
  201. """
  202. for i, msg in enumerate(messages):
  203. if isinstance(msg, dict) and ("cache_control" in msg or isinstance(msg.get("content"), list)):
  204. messages[i] = strip_anthropic_cache_control([dict(msg)])[0]
  205. return messages
  206. def strip_anthropic_tool_cache_control(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
  207. """Return copied tools without request-local Anthropic cache markers."""
  208. cleaned = copy.deepcopy(tools or [])
  209. for tool in cleaned:
  210. if isinstance(tool, dict):
  211. tool.pop("cache_control", None)
  212. return cleaned
  213. def _count_cache_markers(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> int:
  214. """Count the wire-visible cache markers in a request-local plan."""
  215. parts = [p for m in messages if isinstance(m, dict) and isinstance(m.get("content"), list) for p in m["content"]]
  216. return sum(1 for item in [*messages, *parts, *tools] if isinstance(item, dict) and "cache_control" in item)
  217. def _completed_transaction_endpoint_indexes(messages: list[dict[str, Any]], *, native_anthropic: bool) -> list[int]:
  218. """Select legal ends of completed tool runs and ordinary turns."""
  219. def _tool_run_end(start: int) -> int:
  220. end = start
  221. while end < len(messages) and isinstance(messages[end], dict) and messages[end].get("role") == "tool":
  222. end += 1
  223. return end
  224. endpoints: list[int] = []
  225. index = 0
  226. while index < len(messages):
  227. message = messages[index]
  228. if not isinstance(message, dict) or message.get("role") == "system":
  229. index += 1
  230. continue
  231. role = message.get("role")
  232. if role == "assistant" and message.get("tool_calls"):
  233. result_end = _tool_run_end(index + 1)
  234. if result_end > index + 1 and _can_carry_marker(messages[result_end - 1], native_anthropic):
  235. endpoints.append(result_end - 1)
  236. index = result_end
  237. continue
  238. if role == "tool":
  239. index = _tool_run_end(index)
  240. continue
  241. open_turn = (role == "user" and index + 1 < len(messages)) or (
  242. role == "assistant" and message.get("content") in (None, ""))
  243. if not open_turn and _can_carry_marker(message, native_anthropic):
  244. endpoints.append(index)
  245. index += 1
  246. return endpoints
  247. def build_prompt_cache_plan(
  248. api_messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None, *,
  249. cache_ttl: str = "5m", native_anthropic: bool = False, static_system_prefix: str | None = None,
  250. direct_native_tool_cache: bool = False, tool_part_markers: bool = True,
  251. ) -> PromptCachePlan:
  252. """Build copy-on-write cache sections for one resolved request destination
  253. (``tool_part_markers=False`` keeps markers off role:tool parts on LiteLLM-style routes)."""
  254. planned_tools = strip_anthropic_tool_cache_control(tools)
  255. if not direct_native_tool_cache or not planned_tools:
  256. # apply_anthropic_cache_control re-canonicalises prior decoration itself.
  257. planned_messages = apply_anthropic_cache_control(
  258. list(api_messages or []), cache_ttl=cache_ttl, native_anthropic=native_anthropic,
  259. static_system_prefix=static_system_prefix, tool_part_markers=tool_part_markers)
  260. return PromptCachePlan(messages=planned_messages, tools=planned_tools)
  261. messages = _strip_prior_decoration(list(api_messages or []))
  262. marker = _build_marker(cache_ttl)
  263. if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
  264. # Tool-cache layout: only the static prefix carries a system-side marker; the
  265. # volatile suffix's budget is spent on the tools array.
  266. messages[0] = copy.deepcopy(messages[0])
  267. _apply_system_cache_markers(messages[0], marker, static_system_prefix,
  268. native_anthropic=True, mark_suffix=False, fallback_to_whole=False)
  269. planned_tools[-1]["cache_control"] = dict(marker)
  270. for endpoint in _completed_transaction_endpoint_indexes(messages, native_anthropic=True)[-2:]:
  271. messages[endpoint] = copy.deepcopy(messages[endpoint])
  272. _apply_cache_marker(messages[endpoint], marker, native_anthropic=True)
  273. _split_unmarked_scaffolds(messages)
  274. return PromptCachePlan(messages=messages, tools=planned_tools)
  275. def apply_anthropic_cache_control(
  276. api_messages: list[dict[str, Any]], cache_ttl: str = "5m", native_anthropic: bool = False,
  277. static_system_prefix: str | None = None, tool_part_markers: bool = True,
  278. ) -> list[dict[str, Any]]:
  279. """Apply Anthropic cache-control markers to API messages.
  280. With a matching ``static_system_prefix`` the prefix and full system prompt each get a
  281. marker and the remaining two go to the latest cacheable non-system messages; otherwise
  282. the legacy system-and-3 layout applies. Idempotent: prior decoration is stripped from a
  283. per-message copy first, so a second call (or messages a prior call already planned) can never
  284. accumulate past 4 markers (#90971). Returns a shallow list copy with deep copies of modified
  285. messages.
  286. """
  287. if not api_messages:
  288. return api_messages
  289. messages = _strip_prior_decoration(list(api_messages))
  290. marker = _build_marker(cache_ttl)
  291. breakpoints_used = 0
  292. if messages[0].get("role") == "system":
  293. messages[0] = copy.deepcopy(messages[0])
  294. breakpoints_used = _apply_system_cache_markers(messages[0], marker, static_system_prefix,
  295. native_anthropic=native_anthropic)
  296. non_sys = [i for i, m in enumerate(messages) if m.get("role") != "system"
  297. and _can_carry_marker(m, native_anthropic=native_anthropic, tool_part_markers=tool_part_markers)]
  298. for idx in non_sys[-(4 - breakpoints_used):]:
  299. messages[idx] = copy.deepcopy(messages[idx])
  300. _apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic, tool_part_markers=tool_part_markers)
  301. _split_unmarked_scaffolds(messages)
  302. return messages