prompt_cache_scope.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. """Rotation-stable logical cache scope for prompt_cache_key derivation.
  2. Legacy compression rotation mints a new physical ``session_id`` mid-conversation, moving it
  3. into a fresh cache bucket. ``resolve_prompt_cache_scope()`` maps the physical id to the ROOT
  4. of its compression lineage — NOT ``get_conversation_root`` (the Portal-attribution walk),
  5. which would collapse /branch children and delegate trees into one id. ``/new`` starts a
  6. fresh scope; fork children (branch, delegate, tool-tagged) are isolated. Hosts minting one id
  7. per RESPONSE declare the conversation via ``gateway_session_key``, which wins over the lineage
  8. walk and is hashed to ``gwk_<sha256[:24]>`` (it embeds platform/chat/user identifiers).
  9. """
  10. import hashlib
  11. import logging
  12. from typing import Any, Optional
  13. from utils import base_url_hostname
  14. logger = logging.getLogger(__name__)
  15. _MEMO_ATTR = "_prompt_cache_scope_memo"
  16. _DECLARED_SCOPE_PREFIX = "gwk_"
  17. def _lineage_root(session_id: str, session_db: Any) -> Optional[str]:
  18. """Compression-lineage root of *session_id*, or None (tolerates test-double results)."""
  19. if session_db is None:
  20. return None
  21. try:
  22. lineage = session_db.get_compression_lineage(session_id)
  23. except Exception:
  24. logger.debug("prompt-cache scope lineage walk failed", exc_info=True)
  25. return None
  26. if isinstance(lineage, (list, tuple)) and lineage:
  27. root = lineage[0]
  28. if isinstance(root, str) and root:
  29. return root
  30. return None
  31. def _agent_source(
  32. agent: Any, session_id: str, session_db: Any, row_source: Optional[str] = None
  33. ) -> str:
  34. """The ``sessions.source`` this agent's conversation is recorded under.
  35. ``row_source``: the row's value if already read (``""`` = read, none; ``None`` = look it
  36. up). Before the row lands, use the SAME resolver persistence uses, not ``agent.platform``:
  37. they diverge under ``HERMES_SESSION_SOURCE`` and the declared scope is memoized at once,
  38. so both sides of a ``/new`` would otherwise hash the same scope.
  39. """
  40. if row_source is None and session_id and session_db is not None:
  41. try:
  42. row = session_db.get_session(session_id)
  43. except Exception:
  44. logger.debug("declared-scope source lookup failed", exc_info=True)
  45. row = None
  46. row_source = str(row.get("source") or "").strip() if row else ""
  47. if row_source:
  48. return row_source
  49. platform = getattr(agent, "platform", None)
  50. from agent.session_source import session_source_for
  51. return str(session_source_for(platform)).strip()
  52. def _conversation_generation(session_key: str, source: str, session_db: Any) -> str:
  53. """Durable generation for *session_key*'s current conversation (``""`` if none).
  54. The declared key survives ``/new``, so hashing it alone would reuse one scope across
  55. conversations. The counter advances with each reset boundary, independent of prunable
  56. rows and wall-clock; compression does not advance it.
  57. The declared key names a chat and deliberately survives `/new` and policy resets. See #79017, #86733.
  58. """
  59. reader = getattr(session_db, "latest_conversation_boundary", None)
  60. if not callable(reader):
  61. return ""
  62. generation = reader(session_key, source)
  63. return "" if generation is None else str(int(generation))
  64. def declared_conversation_scope(agent: Any) -> Optional[str]:
  65. """Host-declared logical conversation scope (``gwk_<sha256[:24]>``), or None.
  66. Hashes ``(source, gateway_session_key, generation)``. None (fall back to the physical id)
  67. when no key is declared, for a background-review fork (``_persist_disabled``), for an
  68. explicit fork child, and on any DB error (fail closed rather than merge a fork onto its
  69. parent's key).
  70. The one sanctioned exception is a same-model cache-parity fork (#109964): its whole purpose
  71. is prefix parity with the parent, yet ``_persist_disabled`` + ``_session_db=None`` made both
  72. resolvers key it into a different bucket (one cold ~full-context request per review).
  73. ``build_cache_parity_fork`` stamps the parent's ALREADY-RESOLVED scope as
  74. ``_inherited_cache_scope`` (no DB access from the fork). Only a ``gwk_`` value is a declared
  75. scope; a physical lineage root stays out of the affinity header so the fork publishes
  76. exactly what its parent publishes (None → consumers fall back to the conversation root).
  77. """
  78. inherited = getattr(agent, "_inherited_cache_scope", None)
  79. if isinstance(inherited, str) and inherited.startswith(_DECLARED_SCOPE_PREFIX):
  80. return inherited
  81. key = str(getattr(agent, "_gateway_session_key", "") or "").strip()
  82. if not key or getattr(agent, "_persist_disabled", False):
  83. return None
  84. sid = str(getattr(agent, "session_id", None) or "")
  85. db = getattr(agent, "_session_db", None)
  86. generation = ""
  87. row_source: Optional[str] = None
  88. if sid and db is not None:
  89. try:
  90. # One read for both halves of the row identity (fork verdict + source).
  91. # One read for both halves of the row's identity: the fork verdict and the source the peer
  92. # queries match on live on the same ``sessions`` row, and asking for them separately read it
  93. # twice per resolution (@teknium1 on #98811). A SessionDB without the combined view keeps the
  94. # original call, so nothing that predates it — including the doubles that certify the
  95. # fail-closed contract below — changes behaviour.
  96. identity = getattr(db, "declared_scope_identity", None)
  97. if callable(identity):
  98. is_fork, row_source = identity(sid)
  99. else:
  100. is_fork = db.is_explicit_fork_child(sid)
  101. if is_fork:
  102. return None
  103. except Exception:
  104. logger.debug("declared-scope fork check failed", exc_info=True)
  105. return None
  106. source = _agent_source(agent, sid, db, row_source)
  107. if db is not None:
  108. try:
  109. generation = _conversation_generation(key, source, db)
  110. except Exception:
  111. logger.debug("declared-scope generation read failed", exc_info=True)
  112. return None
  113. # Same identity tuple the peer queries use: same key under different sources must not collapse.
  114. carrier = f"{source}|{key}|{generation}"
  115. digest = hashlib.sha256(carrier.encode("utf-8", errors="replace")).hexdigest()[:24]
  116. return f"{_DECLARED_SCOPE_PREFIX}{digest}"
  117. def resolve_prompt_cache_scope(agent: Any) -> str:
  118. """Rotation-stable cache-scope id: the inherited parent scope of a same-model cache-parity
  119. fork, else the declared scope, else the compression-lineage root of ``agent.session_id``
  120. (the physical id without ancestry/DB). Memoized on the agent."""
  121. inherited = getattr(agent, "_inherited_cache_scope", None)
  122. if isinstance(inherited, str) and inherited:
  123. return _apply_fork_tag(agent, inherited)
  124. sid = str(getattr(agent, "session_id", None) or "")
  125. if not sid:
  126. return ""
  127. db = getattr(agent, "_session_db", None)
  128. # DB presence is part of the key: an agent that gains a DB handle later must re-resolve.
  129. key = (sid, db is not None)
  130. memo = getattr(agent, _MEMO_ATTR, None)
  131. if isinstance(memo, tuple) and len(memo) == 2 and memo[0] == key:
  132. return _apply_fork_tag(agent, memo[1])
  133. root = declared_conversation_scope(agent) or _lineage_root(sid, db)
  134. scope = root or sid
  135. # Memoize on success, with no DB, or when the agent never persists a row. A failed/empty
  136. # walk on a persisting agent is NOT memoized: the physical id is right for now (row not
  137. # yet persisted) but would stay wrong for the whole segment once it lands.
  138. if root is not None or db is None or getattr(agent, "_persist_disabled", False):
  139. try:
  140. setattr(agent, _MEMO_ATTR, (key, scope))
  141. except Exception:
  142. pass # frozen/slotted doubles: resolution works, just unmemoized
  143. return _apply_fork_tag(agent, scope)
  144. # ``<scope>::<tag>`` — double colon so gateway session keys that already contain single
  145. # colons are never mistaken for a fork scope.
  146. FORK_SCOPE_SEPARATOR = "::"
  147. # Aggregator slugs that front xAI Grok (OpenRouter and friends).
  148. GROK_AGGREGATOR_MODEL_PREFIXES = ("x-ai/grok-", "xai/grok-")
  149. def is_slot_keyed_cache_route(provider: Any, model: Any, base_url: Any = "") -> bool:
  150. """True when the cache key selects ONE server-side slot per conversation.
  151. xAI (``xai``/``xai-oauth``, ``api.x.ai``, or Grok via OpenRouter ``x-ai/grok-*``) pins its
  152. prompt cache to the server picked by ``x-grok-conv-id`` / ``prompt_cache_key``: two divergent
  153. request streams under one key evict each other. Anthropic, DeepSeek and Gemini caches are
  154. content-addressed and OpenAI's ``prompt_cache_key`` only routes over a prefix match, so the
  155. shared scope (#109964) stays a win there. Only the OpenRouter profile reads ``cache_scope_id``
  156. for the sticky header; other aggregators fronting Grok change only the body key.
  157. """
  158. if str(provider or "").strip().lower() in {"xai", "xai-oauth"}:
  159. return True
  160. if base_url_hostname(str(base_url or "")) == "api.x.ai":
  161. return True
  162. return str(model or "").strip().lower().startswith(GROK_AGGREGATOR_MODEL_PREFIXES)
  163. def is_fork_cache_scope(scope: Any) -> bool:
  164. """True when *scope* is a fork-derived scope (``<scope>::<tag>``)."""
  165. return isinstance(scope, str) and FORK_SCOPE_SEPARATOR in scope
  166. def _apply_fork_tag(agent: Any, scope: str) -> str:
  167. """``<scope>::<tag>`` for a tagged fork on a slot-keyed route, once the fork's OWN compaction
  168. committed: before it the fork extends the parent's prefix (warm read, #109964); after it the
  169. rewritten stream would evict the parent's xAI slot. Read per call, so fallbacks re-evaluate."""
  170. tag = getattr(agent, "_prompt_cache_fork_tag", None)
  171. if not scope or not isinstance(tag, str) or not tag:
  172. return scope
  173. if getattr(getattr(agent, "context_compressor", None), "compression_count", 0) < 1:
  174. return scope
  175. if not is_slot_keyed_cache_route(
  176. getattr(agent, "provider", ""), getattr(agent, "model", ""), getattr(agent, "base_url", ""),
  177. ):
  178. return scope
  179. return f"{scope}{FORK_SCOPE_SEPARATOR}{tag}"
  180. def declared_conversation_scope_safe(agent: Any) -> Optional[str]:
  181. """Never-raising variant of :func:`declared_conversation_scope`."""
  182. try:
  183. return declared_conversation_scope(agent)
  184. except Exception:
  185. logger.debug("declared conversation scope resolution failed", exc_info=True)
  186. return None
  187. def resolve_prompt_cache_scope_safe(agent: Any) -> Optional[str]:
  188. """Never-raising variant of :func:`resolve_prompt_cache_scope` (None = use the physical id).
  189. At turn_context an exception inside ``set_runtime_main(...)`` would skip the whole binding.
  190. Returns None on any failure (or when there is no scope). Consumers treat None/empty as "fall back to the
  191. physical session_id", so a resolution failure degrades to pre-#79017 behavior instead of blocking the
  192. caller — important at turn_context's call site, where an exception raised inside the
  193. ``set_runtime_main(...)`` argument list would otherwise skip the whole runtime binding, not just the
  194. cache scope.
  195. """
  196. try:
  197. return resolve_prompt_cache_scope(agent) or None
  198. except Exception:
  199. logger.debug("prompt-cache scope resolution failed", exc_info=True)
  200. return None