usage_anchor.py 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. """Usage-anchored token accounting: the provider's real ``usage.prompt_tokens`` is the only
  2. authoritative context size; the local ``bytes/4`` estimate covers ONLY messages appended since.
  3. An anchor = provider usage at capture + a snapshot of the transcript position it priced:
  4. ``base_count`` (len(messages) at capture; the reply is not yet appended and is covered by
  5. ``completion_tokens``, so the delta walk skips an assistant row at that index), ``base_last_role``
  6. and ``base_last_fp`` (content fingerprint of the last priced message; compaction, splices and
  7. rewinds replace it → anchor fails closed → full estimation until the next real reading).
  8. The fingerprint (not ``id()``) is the identity: the gateway re-reads the transcript from the DB
  9. every turn and a resumed session runs in a fresh process, so object identity is never stable
  10. across the surfaces where the estimate mattered most (#99421, #104462). The anchor also persists
  11. on the session row (``model_config._usage_anchor``) so a restarted process can restore it; a
  12. restored anchor is honored only while the durable transcript still matches its fingerprint.
  13. """
  14. from __future__ import annotations
  15. import hashlib
  16. import json
  17. import logging
  18. from typing import Any, Dict, List, Optional
  19. logger = logging.getLogger(__name__)
  20. USAGE_ANCHOR_MODEL_CONFIG_KEY = "_usage_anchor"
  21. # Identity of a priced message = the provider-visible fields that round-trip the session DB
  22. # byte-for-byte. Display/persistence metadata (timestamps, row ids, display kinds) is rewritten
  23. # on reload and would only ever fail the match closed.
  24. _FINGERPRINT_KEYS = ("role", "content", "api_content", "tool_call_id", "tool_calls")
  25. def message_fingerprint(msg: Any) -> Optional[str]:
  26. """Stable digest of one transcript message over its provider-visible, persisted fields."""
  27. if not isinstance(msg, dict):
  28. return None
  29. payload = {k: msg.get(k) for k in _FINGERPRINT_KEYS if msg.get(k) is not None}
  30. try:
  31. raw = json.dumps(payload, sort_keys=True, default=str, ensure_ascii=True, separators=(",", ":"))
  32. except (TypeError, ValueError):
  33. raw = repr(sorted(payload.items()))
  34. return hashlib.sha256(raw.encode("utf-8", "replace")).hexdigest()
  35. def _priced_prefix_fingerprint(messages: list[dict[str, Any]], base_count: int) -> Optional[str]:
  36. """Stable digest of the whole provider-priced prefix.
  37. The last priced message fingerprint proves only that one row survived at
  38. ``base_count - 1``. A compaction can preserve that row while rewriting the
  39. earlier prefix, so the anchor must also bind to the full priced prefix it
  40. represents.
  41. """
  42. if base_count <= 0 or len(messages) < base_count:
  43. return None
  44. fps = []
  45. for msg in messages[:base_count]:
  46. fp = message_fingerprint(msg)
  47. if not fp:
  48. return None
  49. role = msg.get("role") if isinstance(msg, dict) else None
  50. fps.append((role, fp))
  51. raw = json.dumps(fps, separators=(",", ":"), ensure_ascii=True)
  52. return hashlib.sha256(raw.encode("utf-8", "replace")).hexdigest()
  53. def capture_usage_anchor(prompt_tokens: Any, completion_tokens: Any, messages: list[dict[str, Any]]) -> Optional[dict[str, Any]]:
  54. """Build a usage anchor from provider-reported usage, or None when usage is unusable."""
  55. try:
  56. pt = int(prompt_tokens or 0)
  57. ct = int(completion_tokens or 0)
  58. except (TypeError, ValueError):
  59. return None
  60. if pt <= 0 or not isinstance(messages, list) or not messages:
  61. return None # some endpoints omit usage — caller keeps its anchor
  62. last = messages[-1]
  63. return {
  64. "prompt_tokens": pt,
  65. "completion_tokens": max(0, ct),
  66. "base_count": len(messages),
  67. "base_last_role": last.get("role") if isinstance(last, dict) else None,
  68. "base_last_fp": message_fingerprint(last),
  69. "base_prefix_fp": _priced_prefix_fingerprint(messages, len(messages)),
  70. }
  71. def _anchor_matches(messages: list[dict[str, Any]], anchor: dict[str, Any]) -> bool:
  72. try:
  73. base_count = int(anchor.get("base_count") or 0)
  74. except (TypeError, ValueError):
  75. return False
  76. if base_count <= 0 or len(messages) < base_count:
  77. return False
  78. base_msg = messages[base_count - 1]
  79. if not isinstance(base_msg, dict) or base_msg.get("role") != anchor.get("base_last_role"):
  80. return False
  81. fp = anchor.get("base_last_fp")
  82. if not isinstance(fp, str) or not fp or message_fingerprint(base_msg) != fp:
  83. return False
  84. prefix_fp = anchor.get("base_prefix_fp")
  85. return (
  86. isinstance(prefix_fp, str)
  87. and bool(prefix_fp)
  88. and _priced_prefix_fingerprint(messages, base_count) == prefix_fp
  89. )
  90. def anchored_context_tokens(messages: list[dict[str, Any]], anchor: Optional[dict[str, Any]], *, charge_stale_thinking: bool = True) -> Optional[int]:
  91. """Anchored prompt+completion tokens plus a rough estimate of ONLY the messages appended since;
  92. None when the anchor is missing or stale. The anchored response's own reply is skipped (already
  93. in completion_tokens). ``charge_stale_thinking`` is forwarded to the delta estimate."""
  94. if not isinstance(anchor, dict) or not isinstance(messages, list) or not _anchor_matches(messages, anchor):
  95. return None
  96. from agent.model_metadata import estimate_messages_tokens_rough
  97. total = int(anchor["prompt_tokens"]) + int(anchor.get("completion_tokens") or 0)
  98. delta = messages[int(anchor["base_count"]):]
  99. if delta and isinstance(delta[0], dict) and delta[0].get("role") == "assistant":
  100. delta = delta[1:]
  101. if delta:
  102. total += estimate_messages_tokens_rough(delta, charge_stale_thinking=charge_stale_thinking)
  103. return total
  104. def _serialize(anchor: Any) -> Optional[dict[str, Any]]:
  105. if not isinstance(anchor, dict):
  106. return None
  107. try:
  108. pt, ct, base_count = (int(anchor.get(k) or 0) for k in ("prompt_tokens", "completion_tokens", "base_count"))
  109. except (TypeError, ValueError):
  110. return None
  111. fp, role = anchor.get("base_last_fp"), anchor.get("base_last_role")
  112. prefix_fp = anchor.get("base_prefix_fp")
  113. if (
  114. pt <= 0
  115. or base_count <= 0
  116. or not isinstance(fp, str)
  117. or not fp
  118. or not isinstance(prefix_fp, str)
  119. or not prefix_fp
  120. ):
  121. return None
  122. return {"prompt_tokens": pt, "completion_tokens": max(0, ct), "base_count": base_count,
  123. "base_last_role": role if isinstance(role, str) else None, "base_last_fp": fp,
  124. "base_prefix_fp": prefix_fp}
  125. def persist_usage_anchor(agent: Any, anchor: Optional[dict[str, Any]]) -> None:
  126. """Write (or clear, ``None``) the session row's anchor blob. Best-effort: the row may not exist yet."""
  127. if getattr(agent, "_persist_disabled", False):
  128. return
  129. session_id = getattr(agent, "session_id", None)
  130. patcher = getattr(getattr(agent, "_session_db", None), "patch_session_model_config", None)
  131. if not session_id or not callable(patcher):
  132. return
  133. try:
  134. patcher(session_id, {USAGE_ANCHOR_MODEL_CONFIG_KEY: _serialize(anchor)})
  135. except Exception:
  136. logger.debug("usage anchor persist failed", exc_info=True)
  137. def set_usage_anchor(agent: Any, anchor: Optional[dict[str, Any]], *, turn_base: bool = False) -> None:
  138. """Install ``anchor`` on the agent (``None`` clears) and mirror it to the session row."""
  139. agent._usage_anchor = anchor
  140. if turn_base or anchor is None:
  141. agent._turn_base_usage_anchor = anchor
  142. persist_usage_anchor(agent, anchor)
  143. def restore_usage_anchor(agent: Any, conversation_history: Optional[list[dict[str, Any]]]) -> None:
  144. """On a resumed session, adopt the persisted anchor when ``conversation_history`` still carries
  145. the priced prefix; otherwise clear the stale blob so it can never suppress compression."""
  146. if getattr(agent, "_usage_anchor", None) is not None or getattr(agent, "_persist_disabled", False):
  147. return
  148. session_id = getattr(agent, "session_id", None)
  149. getter = getattr(getattr(agent, "_session_db", None), "get_session_model_config_value", None)
  150. if not session_id or not callable(getter) or not isinstance(conversation_history, list):
  151. return
  152. try:
  153. anchor = _serialize(getter(session_id, USAGE_ANCHOR_MODEL_CONFIG_KEY, None))
  154. except Exception:
  155. logger.debug("usage anchor load failed", exc_info=True)
  156. return
  157. if anchor is None:
  158. return
  159. if _anchor_matches(conversation_history, anchor):
  160. agent._usage_anchor = anchor
  161. else:
  162. persist_usage_anchor(agent, None)
  163. def persisted_anchor_tokens(session_db: Any, session_id: Any, messages: Any) -> Optional[int]:
  164. """Anchored token figure from the session row's persisted anchor, for callers without a live
  165. agent (gateway hygiene); None when absent, unreadable, or stale against ``messages``."""
  166. getter = getattr(session_db, "get_session_model_config_value", None)
  167. if not session_id or not callable(getter) or not isinstance(messages, list):
  168. return None
  169. try:
  170. anchor = _serialize(getter(session_id, USAGE_ANCHOR_MODEL_CONFIG_KEY, None))
  171. except Exception:
  172. logger.debug("usage anchor load failed", exc_info=True)
  173. return None
  174. return anchored_context_tokens(messages, anchor) if anchor else None