| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377 |
- """Anthropic prompt caching strategy — pure functions, no AIAgent dependency.
- Default layout: 4 cache_control breakpoints — the static system prefix, the end of the
- system prompt, and the last 2 non-system messages (without a static prefix: one system
- breakpoint plus the last 3 messages). All markers share one TTL (5m or 1h).
- """
- import copy
- from dataclasses import dataclass
- from typing import Any, Dict, List
- from agent.prompt_cache_boundary import find_stable_prefix
- @dataclass(frozen=True)
- class PromptCachePlan:
- """Request-local message and tool sections with their cache markers."""
- messages: list[dict[str, Any]]
- tools: list[dict[str, Any]]
- def envelope_tool_part_cache_markers_supported(provider: str | None, base_url: str | None) -> bool:
- """Whether the envelope-layout route honors part-level markers on role:tool.
- OpenRouter/Nous Portal relocate a part-level ``cache_control`` onto the ``tool_result``
- block; LiteLLM-style proxies copy parts verbatim, so it lands at ``tool_result.content[0]``
- (non-retryable 400). There, role:tool carries no part markers and the budget reallocates.
- """
- from agent.agent_runtime_helpers import _is_litellm_route
- return not _is_litellm_route((provider or "").strip().lower(), base_url or "")
- def is_one_shot_request(messages: list[dict[str, Any]] | None, tools: list[dict[str, Any]] | None) -> bool:
- """True for a request no later call can read back from cache: no tools, no assistant/tool
- turns, at most one user turn (compression summaries, titles, MoA synthesis). A marker there
- only buys a cache write (1.25x input at 5m, 2x at 1h) that nothing ever reads. A caller that
- already decorated the request (a MoA advisor's first iteration, whose prefix later
- iterations replay) decided it will reuse it, so that decision wins over the shape."""
- if tools:
- return False
- msgs = [m for m in messages or [] if isinstance(m, dict)]
- if any("cache_control" in m or _has_part_marker(m.get("content")) for m in msgs):
- return False
- roles = [m.get("role") for m in msgs]
- return roles.count("user") <= 1 and not any(r in ("assistant", "tool") for r in roles)
- def _text_part(text: str, cache_marker: dict | None = None) -> dict:
- part: dict = {"type": "text", "text": text}
- if cache_marker is not None:
- part["cache_control"] = cache_marker
- return part
- def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool = False,
- tool_part_markers: bool = True) -> None:
- """Add cache_control to a single message, handling all format variations."""
- role = msg.get("role", "")
- content = msg.get("content")
- if role == "tool" and not native_anthropic and not tool_part_markers:
- # LiteLLM-style envelope: a part marker → tool_result.content[0] → non-retryable 400.
- return
- if (role == "tool" and native_anthropic) or content is None or content == "":
- # Native role:tool: top-level marker, the adapter moves it inside tool_result. Empty
- # content: no part can carry it, and OpenRouter rejects a top-level marker on role:tool
- # (silent hang) and ignores it on empty assistant turns — skip those on the envelope.
- if not (role in ("tool", "assistant") and not native_anthropic):
- msg["cache_control"] = cache_marker
- elif isinstance(content, str):
- # Builder-declared boundary: the scaffold carries the breakpoint and the volatile tail
- # rides unmarked. Request-local only — the stored message stays a string.
- msg["content"] = _scaffold_parts(msg, cache_marker) or [_text_part(content, cache_marker)]
- elif isinstance(content, list) and content and isinstance(content[-1], dict):
- content[-1]["cache_control"] = cache_marker
- def _scaffold_parts(msg: dict, cache_marker: dict | None = None) -> list | None:
- """``[scaffold, tail]`` parts of a string user message with a registered scaffold, the
- marker (if any) on the scaffold; ``find_stable_prefix`` keeps the tail non-blank."""
- content = msg.get("content")
- prefix = find_stable_prefix(content) if msg.get("role") == "user" and isinstance(content, str) else None
- return None if prefix is None else [_text_part(prefix, cache_marker), _text_part(content[len(prefix):])]
- def _split_unmarked_scaffolds(messages: list[dict[str, Any]]) -> None:
- """Send every registered scaffold message as [scaffold, tail] parts, marked or not (in place).
- Provider caches key on content, and two text parts are different content from one string:
- a skill message split while it held a breakpoint must keep the same parts after the rolling
- marker window moves past it, or the next request re-writes the whole scaffold (#81867).
- Each hit refreshes the scaffold's registry LRU slot, so a scaffold still in history stays
- registered and the failover stripper can still recognise its split.
- """
- for i, msg in enumerate(messages):
- if isinstance(msg, dict) and (parts := _scaffold_parts(msg)):
- messages[i] = {**msg, "content": parts}
- def _can_carry_marker(msg: dict, native_anthropic: bool, tool_part_markers: bool = True) -> bool:
- """True if a marker on this message is actually honored by the provider.
- Native Anthropic honors every message; the envelope layout only honors markers inside
- content parts (empty content wastes a breakpoint) and ``tool_part_markers=False`` excludes
- role:tool too (400). Must agree with :func:`_apply_cache_marker` (marks the LAST part).
- """
- if native_anthropic:
- return True
- if msg.get("role") == "tool" and not tool_part_markers:
- return False
- content = msg.get("content")
- return isinstance(content[-1], dict) if isinstance(content, list) and content else isinstance(content, str) and content != ""
- def _build_marker(ttl: str) -> dict[str, str]:
- """Build a cache_control marker dict for the given TTL ('5m' or '1h')."""
- return {"type": "ephemeral", "ttl": "1h"} if ttl == "1h" else {"type": "ephemeral"}
- # Alibaba-family providers (Qwen routes): five-minute context cache, 1h tier rejected. Shared
- # with agent_runtime_helpers.anthropic_prompt_cache_policy so the opt-in and the TTL clamp
- # never desync. Do NOT narrow this set to extend a TTL — narrowing DISABLES caching.
- ALIBABA_FAMILY_PROVIDERS = frozenset({"opencode", "opencode-go", "opencode-zen", "alibaba"})
- # 1h-tier ALLOW-list: only routes wire-measured to retain a 1h marker. Other opencode routes
- # are UNMEASURED, not known-bad (opencode-go's `ephemeral_5m_input_tokens` label is not
- # evidence of the retention window).
- MEASURED_1H_PROVIDERS = frozenset({"opencode-go"})
- # Models measured to ignore the 1h tier on a MEASURED_1H_PROVIDERS route; consulted only
- # there (the same model on its own endpoint is a separate route).
- NO_1H_TIER_MODELS = frozenset({"minimax-m2.5"})
- def _flat_model(model: str) -> str:
- """Bare model id, tolerating aggregator prefixes (``vendor/model``)."""
- return (model or "").strip().rsplit("/", 1)[-1].lower()
- def is_qwen_model(model: str) -> bool:
- """True when ``model`` names a Qwen-family model (shared with anthropic_prompt_cache_policy)."""
- return "qwen" in (model or "").lower()
- # ``prompt_caching.cache_ttl: auto`` picks the tier by who paces the session. The 1h tier
- # writes at 2x base vs 1.25x for 5m and only pays off when turns are more than five minutes
- # apart — a person stepping away and coming back. Machine-paced sessions call every few
- # seconds until they finish and never collect the retention, so they stay on 5m.
- # Measured on one install (2 days of per-call logs): 63% of interactive cache-write tokens
- # were cold re-writes after a 5–60 min idle gap (1h saves ~42% of write cost there), while
- # 1h on subagents/cron would have cost ~49% more.
- AUTO_CACHE_TTL = "auto"
- MACHINE_PACED_SOURCES = frozenset({"subagent", "cron", "oneshot", "webhook", "kanban", "api", "tool", "batch"})
- def auto_cache_ttl_for_source(source: str | None) -> str:
- """The tier ``auto`` resolves to for a session source: ``5m`` for machine-paced sources,
- ``1h`` for everything a human types into (cli, tui, desktop, messaging platforms)."""
- return "5m" if (source or "").strip().lower() in MACHINE_PACED_SOURCES else "1h"
- def effective_cache_ttl(ttl: str | None, *, model: str = "", provider: str = "") -> str:
- """Clamp a requested cache TTL to what the destination route supports (``None`` → ``5m``).
- Qwen/Alibaba routes drop ``1h`` (→ ``5m``) except on ``MEASURED_1H_PROVIDERS`` minus
- ``NO_1H_TIER_MODELS``; that check runs BEFORE the generic Qwen clamp, which would swallow it.
- """
- if ttl != "1h":
- return ttl or "5m"
- provider_lower = (provider or "").lower()
- if provider_lower in MEASURED_1H_PROVIDERS:
- return "5m" if _flat_model(model) in NO_1H_TIER_MODELS else "1h"
- return "5m" if is_qwen_model(model) or provider_lower in ALIBABA_FAMILY_PROVIDERS else "1h"
- def _apply_system_cache_markers(
- message: dict, cache_marker: dict, static_system_prefix: str | None, *,
- native_anthropic: bool, mark_suffix: bool = True, fallback_to_whole: bool = True,
- ) -> int:
- """Mark the static system prefix (and optionally the full prompt); returns markers applied.
- The stored system prompt stays one string, split only in the request. ``mark_suffix=False``
- is the tool-cache-plan layout (suffix budget spent on the tools array); ``fallback_to_whole=
- False`` marks nothing when the split is impossible. When the prompt IS the prefix the whole
- message is one block — never an empty text block (400).
- """
- content = message.get("content")
- if isinstance(static_system_prefix, str) and static_system_prefix and isinstance(content, str) and content.startswith(static_system_prefix):
- suffix = content[len(static_system_prefix):]
- if suffix.strip():
- message["content"] = [_text_part(static_system_prefix, cache_marker),
- _text_part(suffix, cache_marker if mark_suffix else None)]
- return 2 if mark_suffix else 1
- elif not fallback_to_whole:
- return 0
- _apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
- return 1
- def _has_part_marker(content: Any) -> bool:
- return isinstance(content, list) and any(isinstance(part, dict) and "cache_control" in part for part in content)
- def strip_anthropic_cache_control(api_messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Remove ``cache_control`` markers and undo decoration-produced list shapes (in place).
- Used before re-decorating after a mid-turn failover. Flattening to a string is restricted
- to the exact shapes :func:`apply_anthropic_cache_control` produces from string content
- (single text part, two-part system split, two-part skill split) so the ``""``-join is
- byte-exact. Marker removal is copy-on-write on part dicts: parts can alias caller-held
- lists and stripping must never rewrite the stored transcript.
- """
- for msg in api_messages:
- if not isinstance(msg, dict):
- continue
- msg.pop("cache_control", None)
- content = msg.get("content")
- if not isinstance(content, list):
- continue
- role = msg.get("role")
- user_pair = role == "user" and len(content) == 2 and all(isinstance(p, dict) for p in content)
- # The skill split is the only decoration marking the FIRST part of a user message, so the
- # marked shape identifies it even after the prefix registry evicted the entry.
- first_part_marked = user_pair and "cache_control" in content[0] and "cache_control" not in content[1]
- if _has_part_marker(content):
- content = msg["content"] = [
- {k: v for k, v in part.items() if k != "cache_control"}
- if isinstance(part, dict) and "cache_control" in part else part for part in content
- ]
- plain_text_parts = content and all(
- isinstance(part, dict) and part.get("type", "text") == "text"
- and isinstance(part.get("text"), str) and set(part.keys()) <= {"type", "text"} for part in content
- )
- if not plain_text_parts:
- continue
- joined = "".join(part["text"] for part in content)
- # An UNMARKED split (outside the marker window) only the registry tells from a genuine
- # two-part user message.
- skill_split = first_part_marked or (user_pair and find_stable_prefix(joined) == content[0]["text"])
- if len(content) == 1 or (role == "system" and len(content) == 2) or skill_split:
- msg["content"] = joined
- return api_messages
- def _strip_prior_decoration(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Re-canonicalise planner output before re-planning (in place on the list, copy-on-write per message).
- Every list-content message is stripped, not just marked ones: an UNMARKED skill split (outside
- the previous request's marker window) left as two parts would get the new breakpoint on its
- volatile tail instead of the scaffold. A shallow copy suffices because the strip is
- copy-on-write on content parts.
- """
- for i, msg in enumerate(messages):
- if isinstance(msg, dict) and ("cache_control" in msg or isinstance(msg.get("content"), list)):
- messages[i] = strip_anthropic_cache_control([dict(msg)])[0]
- return messages
- def strip_anthropic_tool_cache_control(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
- """Return copied tools without request-local Anthropic cache markers."""
- cleaned = copy.deepcopy(tools or [])
- for tool in cleaned:
- if isinstance(tool, dict):
- tool.pop("cache_control", None)
- return cleaned
- def _count_cache_markers(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> int:
- """Count the wire-visible cache markers in a request-local plan."""
- parts = [p for m in messages if isinstance(m, dict) and isinstance(m.get("content"), list) for p in m["content"]]
- return sum(1 for item in [*messages, *parts, *tools] if isinstance(item, dict) and "cache_control" in item)
- def _completed_transaction_endpoint_indexes(messages: list[dict[str, Any]], *, native_anthropic: bool) -> list[int]:
- """Select legal ends of completed tool runs and ordinary turns."""
- def _tool_run_end(start: int) -> int:
- end = start
- while end < len(messages) and isinstance(messages[end], dict) and messages[end].get("role") == "tool":
- end += 1
- return end
- endpoints: list[int] = []
- index = 0
- while index < len(messages):
- message = messages[index]
- if not isinstance(message, dict) or message.get("role") == "system":
- index += 1
- continue
- role = message.get("role")
- if role == "assistant" and message.get("tool_calls"):
- result_end = _tool_run_end(index + 1)
- if result_end > index + 1 and _can_carry_marker(messages[result_end - 1], native_anthropic):
- endpoints.append(result_end - 1)
- index = result_end
- continue
- if role == "tool":
- index = _tool_run_end(index)
- continue
- open_turn = (role == "user" and index + 1 < len(messages)) or (
- role == "assistant" and message.get("content") in (None, ""))
- if not open_turn and _can_carry_marker(message, native_anthropic):
- endpoints.append(index)
- index += 1
- return endpoints
- def build_prompt_cache_plan(
- api_messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None, *,
- cache_ttl: str = "5m", native_anthropic: bool = False, static_system_prefix: str | None = None,
- direct_native_tool_cache: bool = False, tool_part_markers: bool = True,
- ) -> PromptCachePlan:
- """Build copy-on-write cache sections for one resolved request destination
- (``tool_part_markers=False`` keeps markers off role:tool parts on LiteLLM-style routes)."""
- planned_tools = strip_anthropic_tool_cache_control(tools)
- if not direct_native_tool_cache or not planned_tools:
- # apply_anthropic_cache_control re-canonicalises prior decoration itself.
- planned_messages = apply_anthropic_cache_control(
- list(api_messages or []), cache_ttl=cache_ttl, native_anthropic=native_anthropic,
- static_system_prefix=static_system_prefix, tool_part_markers=tool_part_markers)
- return PromptCachePlan(messages=planned_messages, tools=planned_tools)
- messages = _strip_prior_decoration(list(api_messages or []))
- marker = _build_marker(cache_ttl)
- if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
- # Tool-cache layout: only the static prefix carries a system-side marker; the
- # volatile suffix's budget is spent on the tools array.
- messages[0] = copy.deepcopy(messages[0])
- _apply_system_cache_markers(messages[0], marker, static_system_prefix,
- native_anthropic=True, mark_suffix=False, fallback_to_whole=False)
- planned_tools[-1]["cache_control"] = dict(marker)
- for endpoint in _completed_transaction_endpoint_indexes(messages, native_anthropic=True)[-2:]:
- messages[endpoint] = copy.deepcopy(messages[endpoint])
- _apply_cache_marker(messages[endpoint], marker, native_anthropic=True)
- _split_unmarked_scaffolds(messages)
- return PromptCachePlan(messages=messages, tools=planned_tools)
- def apply_anthropic_cache_control(
- api_messages: list[dict[str, Any]], cache_ttl: str = "5m", native_anthropic: bool = False,
- static_system_prefix: str | None = None, tool_part_markers: bool = True,
- ) -> list[dict[str, Any]]:
- """Apply Anthropic cache-control markers to API messages.
- With a matching ``static_system_prefix`` the prefix and full system prompt each get a
- marker and the remaining two go to the latest cacheable non-system messages; otherwise
- the legacy system-and-3 layout applies. Idempotent: prior decoration is stripped from a
- per-message copy first, so a second call (or messages a prior call already planned) can never
- accumulate past 4 markers (#90971). Returns a shallow list copy with deep copies of modified
- messages.
- """
- if not api_messages:
- return api_messages
- messages = _strip_prior_decoration(list(api_messages))
- marker = _build_marker(cache_ttl)
- breakpoints_used = 0
- if messages[0].get("role") == "system":
- messages[0] = copy.deepcopy(messages[0])
- breakpoints_used = _apply_system_cache_markers(messages[0], marker, static_system_prefix,
- native_anthropic=native_anthropic)
- non_sys = [i for i, m in enumerate(messages) if m.get("role") != "system"
- and _can_carry_marker(m, native_anthropic=native_anthropic, tool_part_markers=tool_part_markers)]
- for idx in non_sys[-(4 - breakpoints_used):]:
- messages[idx] = copy.deepcopy(messages[idx])
- _apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic, tool_part_markers=tool_part_markers)
- _split_unmarked_scaffolds(messages)
- return messages
|