turn_usage.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. """Per-response usage accounting for the conversation turn loop.
  2. After every successful model API call, ``record_response_usage`` folds ``response.usage``
  3. into: the context compressor (``update_from_response`` + the compression-budget rearm
  4. latch), the usage anchor for display/compression math, per-session token/cost counters,
  5. the state.db token-delta queue, and the observability log line. MoA sessions additionally
  6. fold advisor fan-out usage into the reported counts and price the aggregator at its REAL
  7. model/provider. Logger name stays ``agent.conversation_loop`` for caplog parity.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. from contextlib import suppress
  12. from dataclasses import dataclass
  13. from typing import Any, Dict, List
  14. from agent.image_token_cost import calibrate_from_usage
  15. from agent.usage_anchor import capture_usage_anchor, set_usage_anchor
  16. from agent.usage_pricing import estimate_usage_cost, normalize_usage, with_served_service_tier
  17. logger = logging.getLogger("agent.conversation_loop")
  18. def _agent_session_source(agent: Any) -> str:
  19. """The surface the agent's own row create would stamp (``_ensure_db_session``), so an
  20. accounting guard that wins the row-creation race never mints an anonymous session."""
  21. from agent.session_source import session_source_for
  22. return session_source_for(getattr(agent, "platform", None))
  23. @dataclass
  24. class ResponseUsageOutcome:
  25. """``compression_attempts`` is the (possibly rearmed-to-zero) budget counter;
  26. ``rearmed`` tells the loop to also clear its preflight-block latch."""
  27. compression_attempts: int
  28. rearmed: bool = False
  29. def _loop_mod():
  30. """Lazy ``agent.conversation_loop`` import (avoids an import cycle)."""
  31. import agent.conversation_loop as _cl
  32. return _cl
  33. def _fold_moa_usage(agent, canonical_usage):
  34. """MoA: fold advisor fan-out usage into REPORTED token counts (only aggregator usage is
  35. returned, so advisor spend would be invisible) and flush the full-turn trace when
  36. ``moa.save_traces`` is on. Returns ``(client, canonical_usage, advisor_cost)``."""
  37. _moa_ref_cost = None
  38. _moa_client = getattr(agent, "client", None)
  39. if _moa_client is not None and hasattr(_moa_client, "consume_reference_usage"):
  40. try:
  41. _ref_usage, _moa_ref_cost = _moa_client.consume_reference_usage()
  42. if _ref_usage is not None:
  43. canonical_usage = canonical_usage + _ref_usage
  44. except Exception as _moa_acct_exc: # pragma: no cover - defensive
  45. logger.debug("MoA reference usage accounting failed: %s", _moa_acct_exc)
  46. if _moa_client is not None and hasattr(_moa_client, "consume_and_save_trace"):
  47. try:
  48. # Streaming path: pass the streamed acting text so the trace is self-contained.
  49. _agg_streamed_text = getattr(agent, "_current_streamed_assistant_text", "") or ""
  50. _moa_client.consume_and_save_trace(
  51. agent.session_id, aggregator_output_fallback=_agg_streamed_text or None
  52. )
  53. except Exception as _moa_trace_exc: # pragma: no cover - defensive
  54. logger.debug("MoA trace flush failed: %s", _moa_trace_exc)
  55. return _moa_client, canonical_usage, _moa_ref_cost
  56. def record_response_usage(
  57. agent: Any, response: Any, *, messages: list[dict[str, Any]], api_call_count: int,
  58. api_duration: float, compression_attempts: int, max_compression_attempts: int,
  59. ) -> ResponseUsageOutcome:
  60. """Fold ``response.usage`` into compressor, anchors, session counters, state.db
  61. and the API-call log line (see module docstring). No-usage responses only
  62. consume a pending compaction verdict. Returns the loop-visible outcome."""
  63. rearmed = False
  64. compressor = agent.context_compressor
  65. # Count every completed provider attempt, including providers that omit usage.
  66. # Token/cost accounting below stays gated on real usage, but the request itself
  67. # must remain observable.
  68. agent.session_api_calls += 1
  69. if not (hasattr(response, 'usage') and response.usage):
  70. if getattr(compressor, "awaiting_real_usage_after_compression", False):
  71. # No usage -> cannot adjudicate the prior compaction; consume the
  72. # pending verdict so later readings aren't charged to it and
  73. # preflight deferral isn't latched indefinitely.
  74. compressor.update_from_response({})
  75. _note_usage_less = getattr(compressor, "note_usage_less_response", None)
  76. if callable(_note_usage_less):
  77. _note_usage_less()
  78. logger.info(
  79. "API call #%d: model=%s provider=%s in=? out=? total=? latency=%.1fs usage=unavailable",
  80. agent.session_api_calls, agent.model, agent.provider or "unknown", api_duration,
  81. )
  82. return ResponseUsageOutcome(compression_attempts=compression_attempts, rearmed=rearmed)
  83. canonical_usage = with_served_service_tier(
  84. normalize_usage(response.usage, provider=agent.provider, api_mode=agent.api_mode), response)
  85. # Aggregator-only usage kept for pricing: advisor tokens are priced at each advisor's
  86. # OWN model rate and added as dollars below.
  87. aggregator_usage = canonical_usage
  88. _moa_client, canonical_usage, _moa_ref_cost = _fold_moa_usage(agent, canonical_usage)
  89. prompt_tokens = canonical_usage.prompt_tokens
  90. completion_tokens = canonical_usage.output_tokens
  91. total_tokens = canonical_usage.total_tokens
  92. # Gemini long decode sums every hop's re-sent prefix into prompt_tokens (billing, session totals);
  93. # the context actually in the window is the first hop's prompt: compaction and the meter read that.
  94. _initial_prompt = getattr(response.usage, "initial_prompt_tokens", None)
  95. context_prompt_tokens = _initial_prompt if isinstance(_initial_prompt, int) and _initial_prompt > 0 else None
  96. # The canonical buckets follow the same split, so engines reading them never see cache_read > prompt.
  97. _initial_cached = getattr(response.usage, "initial_cached_tokens", 0) if context_prompt_tokens else None
  98. context_cache_read = _initial_cached if isinstance(_initial_cached, int) else None
  99. # Canonical token + cache buckets for context engines; legacy keys stay for back-compat.
  100. usage_dict = {
  101. "prompt_tokens": context_prompt_tokens or prompt_tokens,
  102. "completion_tokens": completion_tokens,
  103. "total_tokens": (context_prompt_tokens + completion_tokens) if context_prompt_tokens else total_tokens,
  104. "input_tokens": (context_prompt_tokens - context_cache_read) if context_cache_read is not None else canonical_usage.input_tokens,
  105. "output_tokens": canonical_usage.output_tokens,
  106. "cache_read_tokens": canonical_usage.cache_read_tokens if context_cache_read is None else context_cache_read,
  107. "cache_write_tokens": canonical_usage.cache_write_tokens,
  108. "reasoning_tokens": canonical_usage.reasoning_tokens,
  109. }
  110. # Capture the boundary latch before update_from_response() consumes it: only the real
  111. # prompt count right after a compaction rearms the budget.
  112. _completed_compaction_pending = bool(
  113. getattr(compressor, "_verify_compaction_cleared_threshold", False)
  114. )
  115. compressor.update_from_response(usage_dict)
  116. # Usage-anchored accounting: snapshot exact provider usage against the durable
  117. # transcript (main-loop ONLY; MoA uses pre-fold aggregator usage). The display meter
  118. # anchors on the turn's FIRST response: later same-turn responses inflate
  119. # prompt_tokens with replayed thinking. Display-only; compression math uses real usage.
  120. # The provider just priced this request exactly: if the delta since the previous anchor
  121. # introduced images, the residual is their real per-image cost (learned before re-anchoring).
  122. calibrate_from_usage(agent, messages, context_prompt_tokens or aggregator_usage.prompt_tokens)
  123. _new_anchor = capture_usage_anchor(
  124. context_prompt_tokens or aggregator_usage.prompt_tokens, aggregator_usage.output_tokens, messages
  125. )
  126. if _new_anchor is not None:
  127. set_usage_anchor(agent, _new_anchor, turn_base=api_call_count == 1)
  128. _compression_threshold = int(getattr(compressor, "threshold_tokens", 0) or 0)
  129. if _loop_mod()._should_rearm_compression_budget(
  130. compression_attempts, completed_compaction_pending=_completed_compaction_pending,
  131. prompt_tokens=usage_dict["prompt_tokens"], threshold_tokens=_compression_threshold,
  132. ):
  133. logger.info(
  134. "Compression budget rearmed after provider-confirmed "
  135. "recovery: prompt=%s < threshold=%s (attempts were %s/%s)",
  136. f"{usage_dict['prompt_tokens']:,}",
  137. f"{_compression_threshold:,}",
  138. compression_attempts,
  139. max_compression_attempts,
  140. )
  141. compression_attempts = 0
  142. # Confirmed recovery also clears the loop's stale insufficient-progress verdict
  143. # (``_preflight_compression_blocked``), else a later pressure spike grows unchecked.
  144. rearmed = True
  145. # Stash canonical usage for on_turn_complete(); keep the latest call's.
  146. agent._last_turn_usage = dict(usage_dict)
  147. # The parent's CURRENT prompt size for headroom math (delegate summary budgets): the
  148. # aggregator's own prompt, never the MoA-folded total (advisor prompts are not in this context).
  149. agent._last_prompt_size_tokens = int(context_prompt_tokens or aggregator_usage.prompt_tokens or 0)
  150. # Persist only provider-confirmed context lengths, not probe tiers.
  151. if getattr(compressor, "_context_probed", False):
  152. ctx = compressor.context_length
  153. if getattr(compressor, "_context_probe_persistable", False):
  154. from agent.model_metadata import save_provider_context_length
  155. save_provider_context_length(agent.model, agent.base_url, ctx, agent.provider)
  156. agent._safe_print(f"{agent.log_prefix}💾 Cached context length: {ctx:,} tokens for {agent.model}")
  157. compressor._context_probed = False
  158. compressor._context_probe_persistable = False
  159. agent.session_prompt_tokens += prompt_tokens
  160. agent.session_completion_tokens += completion_tokens
  161. agent.session_total_tokens += total_tokens
  162. agent.session_input_tokens += canonical_usage.input_tokens
  163. agent.session_output_tokens += canonical_usage.output_tokens
  164. agent.session_cache_read_tokens += canonical_usage.cache_read_tokens
  165. agent.session_cache_write_tokens += canonical_usage.cache_write_tokens
  166. agent.session_reasoning_tokens += canonical_usage.reasoning_tokens
  167. # Rolling history for status-bar averages (last 10).
  168. with suppress(Exception):
  169. hist = getattr(agent, "_api_latency_history", None)
  170. if hist is not None:
  171. hist.append(float(api_duration))
  172. ohist = getattr(agent, "_api_output_history", None)
  173. if ohist is not None:
  174. ohist.append(int(canonical_usage.output_tokens or 0))
  175. _cache_pct = ""
  176. if canonical_usage.cache_read_tokens and prompt_tokens:
  177. _cache_pct = f" cache={canonical_usage.cache_read_tokens}/{prompt_tokens} ({100*canonical_usage.cache_read_tokens/prompt_tokens:.0f}%)"
  178. # write= is the money (cache writes cost 50x a read); id= is what a provider needs to look the
  179. # request up; upstream= is who actually served it when the route reports that (OpenRouter's
  180. # `provider`). Diagnosing the 1,393-agent run's cache misses took a DB join and a live probe
  181. # because none of the three were on this line.
  182. if canonical_usage.cache_write_tokens:
  183. _cache_pct += f" write={canonical_usage.cache_write_tokens}"
  184. _rid = getattr(response, "id", None)
  185. _ident = f" id={_rid}" if isinstance(_rid, str) and _rid else ""
  186. _upstream = getattr(response, "provider", None)
  187. if isinstance(_upstream, str) and _upstream:
  188. _ident += f" upstream={_upstream}"
  189. logger.info(
  190. "API call #%d: model=%s provider=%s in=%d out=%d total=%d latency=%.1fs%s%s",
  191. agent.session_api_calls, agent.model, agent.provider or "unknown",
  192. prompt_tokens, completion_tokens, total_tokens,
  193. api_duration, _cache_pct, _ident,
  194. )
  195. # nous.anthropic_wire=auto: the session's wire is decided once, from this first response.
  196. if agent.session_api_calls == 1 and (agent.provider or "") == "nous":
  197. with suppress(Exception):
  198. from agent.nous_wire import maybe_switch_wire_after_first_response
  199. maybe_switch_wire_after_first_response(agent, response, agent.session_api_calls)
  200. # MoA: agent.model/provider are the virtual preset/"moa" with no pricing entry, silently
  201. # dropping aggregator spend. Price at the REAL model/provider from the aggregator slot.
  202. _agg_cost_model, _agg_cost_provider, _agg_cost_base_url = agent.model, agent.provider, agent.base_url
  203. _agg_slot = getattr(_moa_client, "last_aggregator_slot", None) if _moa_client is not None else None
  204. if _agg_slot and _agg_slot.get("model"):
  205. _agg_cost_model = _agg_slot["model"]
  206. _agg_cost_provider = _agg_slot.get("provider") or agent.provider
  207. _agg_cost_base_url = _agg_slot.get("base_url") or agent.base_url
  208. cost_result = estimate_usage_cost(
  209. _agg_cost_model, aggregator_usage, provider=_agg_cost_provider,
  210. base_url=_agg_cost_base_url, api_key=getattr(agent, "api_key", ""),
  211. )
  212. # Cost delta = aggregator + MoA advisor cost (already priced per-advisor at each
  213. # advisor's own model rate), so state.db's estimated_cost_usd matches the folded
  214. # token counts.
  215. _cost_delta = None
  216. if cost_result.amount_usd is not None:
  217. _cost_delta = float(cost_result.amount_usd)
  218. agent.session_estimated_cost_usd += _cost_delta
  219. if _moa_ref_cost is not None:
  220. try:
  221. _moa_cost = float(_moa_ref_cost)
  222. except (TypeError, ValueError): # pragma: no cover - defensive
  223. _moa_cost = None
  224. if _moa_cost is not None:
  225. agent.session_estimated_cost_usd += _moa_cost
  226. _cost_delta = (_cost_delta or 0.0) + _moa_cost
  227. agent.session_cost_status = cost_result.status
  228. agent.session_cost_source = cost_result.source
  229. # Persist per-call token deltas for any session_id so non-CLI runs can't lose
  230. # accounting; gateway/session-store writes use absolute totals and safely overwrite
  231. # these deltas. Enqueued, not written (a cold state.db UPDATE here stalled the tool
  232. # loop); drained at finalize via _persist_session.
  233. if agent._session_db and agent.session_id:
  234. try:
  235. # Ensure the row exists: under concurrent SQLite load the initial
  236. # _ensure_db_session() may fail, and UPDATE on a missing row affects 0 rows.
  237. if not agent._session_db_created:
  238. agent._ensure_db_session()
  239. agent._session_db.queue_token_counts(
  240. agent.session_id,
  241. source=_agent_session_source(agent),
  242. input_tokens=canonical_usage.input_tokens,
  243. output_tokens=canonical_usage.output_tokens,
  244. cache_read_tokens=canonical_usage.cache_read_tokens,
  245. cache_write_tokens=canonical_usage.cache_write_tokens,
  246. reasoning_tokens=canonical_usage.reasoning_tokens,
  247. estimated_cost_usd=_cost_delta,
  248. cost_status=cost_result.status,
  249. cost_source=cost_result.source,
  250. billing_provider=agent.provider,
  251. billing_base_url=agent.base_url,
  252. billing_mode="subscription_included"
  253. if cost_result.status == "included" else None,
  254. model=agent.model,
  255. api_call_count=1,
  256. task=getattr(agent, "_turn_route_task", "") or "",
  257. )
  258. except Exception as e: # silent loss here undercounts analytics
  259. logger.debug(
  260. "Token persistence failed (session=%s, tokens=%d): %s",
  261. agent.session_id, total_tokens, e,
  262. )
  263. if agent.verbose_logging:
  264. logging.debug(f"Token usage: prompt={usage_dict['prompt_tokens']:,}, completion={usage_dict['completion_tokens']:,}, total={usage_dict['total_tokens']:,}")
  265. # Report cache stats for any provider that returns ``prompt_tokens_details.cached_tokens``,
  266. # not only when we inject cache_control markers.
  267. cached = canonical_usage.cache_read_tokens
  268. written = canonical_usage.cache_write_tokens
  269. prompt = usage_dict["prompt_tokens"]
  270. if (cached or written) and not agent.quiet_mode:
  271. hit_pct = (cached / prompt * 100) if prompt > 0 else 0
  272. agent._vprint(
  273. f"{agent.log_prefix} 💾 Cache: "
  274. f"{cached:,}/{prompt:,} tokens "
  275. f"({hit_pct:.0f}% hit, {written:,} written)"
  276. )
  277. return ResponseUsageOutcome(compression_attempts=compression_attempts, rearmed=rearmed)