shared_metrics_efficiency.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363
  1. """Efficiency shared metrics: what one user turn costs, what users throw away, and where Hermes
  2. spends tokens it did not have to (spilled tool output, idle tool schemas, prompt-cache breaks).
  3. Per-turn state lives on the relay runtime's task/session objects (see ``TurnCost`` and
  4. ``SessionEfficiency``); the agent-side entry points below run on the agent thread and are no-ops
  5. unless shared metrics are enabled. Provider/model always go through the catalog helpers.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import re
  10. from collections import deque
  11. from dataclasses import dataclass, field
  12. from typing import Any, Iterable
  13. from . import shared_metrics_contract as contract
  14. from .shared_metrics_contract import _bucket, _non_negative_number
  15. logger = logging.getLogger(__name__)
  16. _TOKEN_THRESHOLDS = (
  17. (2_000, "lt_2k"), (10_000, "2k_to_10k"), (50_000, "10k_to_50k"), (200_000, "50k_to_200k"),
  18. (1_000_000, "200k_to_1m"),
  19. )
  20. _ACTIVITY_THRESHOLDS = (
  21. (1, "0"), (2, "1"), (3, "2"), (6, "3_to_5"), (11, "6_to_10"), (26, "11_to_25"), (51, "26_to_50"),
  22. (101, "51_to_100"),
  23. )
  24. _OUTPUT_SIZE_THRESHOLDS = (
  25. (1_000, "lt_1k"), (10_000, "1k_to_10k"), (50_000, "10k_to_50k"), (100_000, "50k_to_100k"),
  26. (500_000, "100k_to_500k"),
  27. )
  28. _SCHEMA_TOKEN_THRESHOLDS = (
  29. (1, "0"), (2_000, "lt_2k"), (5_000, "2k_to_5k"), (10_000, "5k_to_10k"), (20_000, "10k_to_20k"),
  30. (40_000, "20k_to_40k"),
  31. )
  32. _TASK_COST_OUTCOMES = {"success": "completed", "cancelled": "interrupted"}
  33. # Anthropic's default prompt-cache TTL: a cold read after this idle gap is expiry, not a bug.
  34. CACHE_TTL_S = 300.0
  35. # Per-session memory of spent turns an /undo or /retry can still reach.
  36. _SPENT_TURNS_KEPT = 64
  37. _PENDING_OUTPUTS_ATTR = "_shared_metrics_tool_outputs"
  38. _PENDING_OUTPUTS_MAX = 256
  39. # The one notice tools/tool_output_truncate.py writes when terminal, execute_code or MCP output was
  40. # cut down inside the tool, before Hermes ever saw the full text.
  41. _TOOL_TRUNCATION_NOTICE = re.compile(r"\[[A-Z_ ]{1,32} TRUNCATED - [\d,]{1,20} chars omitted out of ([\d,]{1,20}) total\]")
  42. _NOTICE_MIN_CHARS = 1_000
  43. def tokens_bucket(tokens: int | None) -> str:
  44. return "unknown" if tokens is None else _bucket(max(0, tokens), _TOKEN_THRESHOLDS, "gte_1m")
  45. def activity_bucket(count: int) -> str:
  46. return _bucket(max(0, int(count)), _ACTIVITY_THRESHOLDS, "gte_101")
  47. def output_size_bucket(size: int | None) -> str:
  48. return "unknown" if size is None else _bucket(max(0, size), _OUTPUT_SIZE_THRESHOLDS, "gte_500k")
  49. def schema_tokens_bucket(tokens: int) -> str:
  50. return _bucket(max(0, int(tokens)), _SCHEMA_TOKEN_THRESHOLDS, "gte_40k")
  51. def _int(value: Any) -> int | None:
  52. number = _non_negative_number(value)
  53. return None if number is None else int(number)
  54. def call_tokens(usage: Any) -> int | None:
  55. """Prompt (cache reads/writes included) plus completion tokens of one call; None without usage."""
  56. if not isinstance(usage, dict):
  57. return None
  58. total = _int(usage.get("total_tokens"))
  59. if total is not None:
  60. return total
  61. parts = [_int(usage.get(key)) for key in (
  62. "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens")]
  63. return sum(p for p in parts if p is not None) if any(p is not None for p in parts) else None
  64. def toolset_metric_name(toolset: Any) -> str:
  65. """A shipped toolset's own name; MCP servers, plugins and user toolsets read ``custom``."""
  66. return toolset if isinstance(toolset, str) and toolset in contract.BUILTIN_TOOLSET_NAMES else "custom"
  67. def _route(route: dict[str, str] | None) -> dict[str, str]:
  68. from .shared_metrics_model import model_route
  69. return dict(route) if route else model_route(None, None)
  70. # ---- per-turn cost and waste (runtime state; caller holds session.lock) ------------------------
  71. @dataclass
  72. class TurnCost:
  73. """Token spend of one task; ``user_turn`` marks a turn a user message started (pre_llm_call),
  74. which Hermes-owned forks sharing the session id never fire."""
  75. user_turn: bool = False
  76. tokens: int = 0
  77. known: bool = False
  78. route: dict[str, str] | None = None
  79. def add_call(self, route: dict[str, str], usage: Any) -> None:
  80. self.route = route
  81. tokens = call_tokens(usage)
  82. if tokens is not None:
  83. self.tokens += tokens
  84. self.known = True
  85. def bucket(self) -> str:
  86. return tokens_bucket(self.tokens if self.known else None)
  87. @dataclass
  88. class _SpentTurn:
  89. route: dict[str, str]
  90. tokens_bucket: str
  91. wasted: bool = False
  92. def _wasted_row(reason: str, turn: _SpentTurn) -> tuple[str, dict[str, str]]:
  93. return contract.WASTED_TOKENS_MARK, {**turn.route, "reason": reason, "tokens_bucket": turn.tokens_bucket}
  94. @dataclass
  95. class ToolUsage:
  96. """What the session paid to carry tool definitions, and which shipped toolsets it touched."""
  97. observed: bool = False
  98. surface: str = "unknown"
  99. enabled_toolsets: frozenset[str] = frozenset()
  100. enabled_count: int = 0
  101. schema_tokens: int = 0
  102. names: tuple[str, ...] | None = None
  103. used: set[str] = field(default_factory=set)
  104. def observe(self, names: tuple[str, ...], toolsets: frozenset[str], enabled_count: int, schema_tokens: int) -> bool:
  105. """Record the tools one request sent; True when a later request changed the array."""
  106. changed = self.names is not None and names != self.names
  107. self.observed, self.names = True, names
  108. self.enabled_toolsets = self.enabled_toolsets | toolsets
  109. self.enabled_count, self.schema_tokens = max(self.enabled_count, enabled_count), schema_tokens
  110. return changed
  111. def absorb(self, other: ToolUsage) -> None:
  112. if not other.observed:
  113. return
  114. self.observed, self.surface = True, other.surface
  115. self.enabled_toolsets = self.enabled_toolsets | other.enabled_toolsets
  116. self.enabled_count = max(self.enabled_count, other.enabled_count)
  117. self.schema_tokens = max(self.schema_tokens, other.schema_tokens)
  118. self.used |= other.used
  119. def rows(self) -> list[tuple[str, dict[str, str]]]:
  120. if not self.observed:
  121. return []
  122. rows = [(contract.TOOL_OVERHEAD_MARK, {
  123. "enabled_tool_count_bucket": contract.size_bucket(self.enabled_count),
  124. "execution_surface": self.surface,
  125. "tool_schema_tokens_bucket": schema_tokens_bucket(self.schema_tokens),
  126. })]
  127. rows.extend(
  128. (contract.TOOL_ENABLED_UNUSED_MARK, {"toolset": name, "used": "yes" if name in self.used else "no"})
  129. for name in sorted(self.enabled_toolsets)
  130. )
  131. return rows
  132. @dataclass
  133. class SessionEfficiency:
  134. spent: deque[_SpentTurn] = field(default_factory=lambda: deque(maxlen=_SPENT_TURNS_KEPT))
  135. tools: ToolUsage = field(default_factory=ToolUsage)
  136. tools_key: tuple[int, int, int] | None = None
  137. cache_read: int = 0
  138. cache_route: dict[str, str] | None = None
  139. cache_ended_ns: int = 0
  140. def finish_turn(
  141. self, cost: TurnCost, terminal: dict[str, str], *, route: dict[str, str] | None,
  142. model_calls: int, tool_calls: int,
  143. ) -> list[tuple[str, dict[str, str]]]:
  144. """task_cost for one user turn the user saw end; an interrupt also counts its tokens wasted."""
  145. turn = _SpentTurn(_route(cost.route or route), cost.bucket())
  146. rows = [(contract.TASK_COST_MARK, {
  147. **turn.route,
  148. "api_calls_bucket": activity_bucket(model_calls),
  149. "outcome": _TASK_COST_OUTCOMES.get(terminal.get("outcome", ""), "failed"),
  150. "tokens_bucket": turn.tokens_bucket,
  151. "tool_calls_bucket": activity_bucket(tool_calls),
  152. })]
  153. if terminal.get("end_reason") == "user_cancelled":
  154. turn.wasted = True
  155. rows.append(_wasted_row("interrupt", turn))
  156. self.spent.append(turn)
  157. return rows
  158. def discard_turns(self, reason: str, turns: int, fallback_route: dict[str, str]) -> list[tuple[str, dict[str, str]]]:
  159. """An /undo or /retry threw away the newest ``turns`` turns: one row per turn not already
  160. counted (an interrupted turn the user then undoes was wasted once, at the interrupt)."""
  161. rows = []
  162. for _ in range(max(1, min(int(turns), _SPENT_TURNS_KEPT))):
  163. turn = self.spent.pop() if self.spent else _SpentTurn(_route(fallback_route), "unknown")
  164. if not turn.wasted:
  165. rows.append(_wasted_row(reason, turn))
  166. return rows
  167. def observe_cache(
  168. self, route: dict[str, str], usage: Any, started_ns: int, ended_ns: int, *, expected: bool,
  169. ) -> str | None:
  170. """A cold primary read after a warm one on the same route is a break Hermes did not announce
  171. (``expected``: Hermes already counted the cause); a gap past the cache TTL is plain expiry."""
  172. read = _int(usage.get("cache_read_tokens")) if isinstance(usage, dict) else None
  173. if read is None:
  174. return None
  175. prev_read, prev_route, prev_ended = self.cache_read, self.cache_route, self.cache_ended_ns
  176. self.cache_read, self.cache_route, self.cache_ended_ns = read, route, ended_ns
  177. if expected or read or not prev_read or prev_route != route:
  178. return None
  179. idle_s = (started_ns - prev_ended) / 1e9
  180. return "cache_expired" if idle_s >= CACHE_TTL_S else "provider_reported_miss"
  181. def cache_break_row(cause: str, route: dict[str, str] | None) -> tuple[str, dict[str, str]]:
  182. return contract.CACHE_BREAK_MARK, {**_route(route), "cause": cause}
  183. # ---- agent-side entry points (agent thread; never raise) ---------------------------------------
  184. def _internal_agent(agent: Any) -> bool:
  185. """Hermes-owned forks (background review, curator) share the user's session id."""
  186. return getattr(agent, "_memory_write_origin", None) == "background_review"
  187. def _agent_route(agent: Any) -> dict[str, str]:
  188. from .shared_metrics_model import model_route
  189. return model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
  190. def _enabled() -> bool:
  191. from .relay_shared_metrics import enabled
  192. return enabled()
  193. def note_tool_result(agent: Any, tool_name: Any, tool_call_id: Any, raw: Any, persisted: Any) -> None:
  194. """Remember one committed tool result until its batch's turn budget has run."""
  195. try:
  196. if _internal_agent(agent) or not _enabled():
  197. return
  198. pending = agent.__dict__.setdefault(_PENDING_OUTPUTS_ATTR, {})
  199. if len(pending) >= _PENDING_OUTPUTS_MAX:
  200. return
  201. size, cut_in_tool = _result_size(raw)
  202. pending[str(tool_call_id or id(persisted))] = [
  203. contract.tool_metric_name({"tool_name": tool_name}), size, cut_in_tool or persisted is not raw,
  204. ]
  205. except Exception:
  206. logger.debug("Shared-metrics tool output not noted", exc_info=True)
  207. def _result_size(result: Any) -> tuple[int | None, bool]:
  208. """(characters the tool produced, whether the tool itself already truncated them)."""
  209. if isinstance(result, dict): # multimodal envelope: its text parts are what can spill
  210. parts = result.get("content") or []
  211. return sum(len(p.get("text") or "") for p in parts if isinstance(p, dict) and p.get("type") == "text"), False
  212. if not isinstance(result, str):
  213. return None, False
  214. notice = _TOOL_TRUNCATION_NOTICE.search(result) if len(result) >= _NOTICE_MIN_CHARS else None
  215. if notice is None:
  216. return len(result), False
  217. return max(len(result), int(notice.group(1).replace(",", ""))), True
  218. def record_tool_batch(agent: Any, batch: list[dict], contents_before: list[Any]) -> None:
  219. """One row per tool result of the batch: truncated when the per-result cap or the per-turn
  220. budget replaced it (the budget runs after the cap, so a result is judged once, here)."""
  221. try:
  222. pending = agent.__dict__.pop(_PENDING_OUTPUTS_ATTR, None)
  223. if not pending or not _enabled():
  224. return
  225. for message, before in zip(batch, contents_before):
  226. entry = pending.get(str(message.get("tool_call_id") or ""))
  227. if entry is not None and message.get("content") is not before:
  228. entry[2] = True
  229. from .relay_shared_metrics import record_process_mark
  230. for tool, size, truncated in pending.values():
  231. record_process_mark(contract.TOOL_OUTPUT_TRUNCATION_MARK, {
  232. "original_size_bucket": output_size_bucket(size), "tool": tool, "truncated": "yes" if truncated else "no",
  233. })
  234. except Exception:
  235. logger.debug("Shared-metrics tool output truncation not recorded", exc_info=True)
  236. def observe_request_tools(agent: Any, tools_for_api: Any) -> None:
  237. """The tool definitions one primary request carries (evidence for trimming default toolsets)."""
  238. try:
  239. if getattr(agent, "_persist_disabled", False) or _internal_agent(agent) or not _enabled():
  240. return
  241. from .relay_shared_metrics import record_session_tools
  242. record_session_tools(str(getattr(agent, "session_id", "") or ""), agent, tools_for_api or [])
  243. except Exception:
  244. logger.debug("Shared-metrics tool overhead not observed", exc_info=True)
  245. def tool_snapshot(agent: Any, tools_for_api: list) -> tuple[tuple[str, ...], frozenset[str], int, int]:
  246. """(sent tool names, enabled shipped toolsets, enabled tool count, estimated schema tokens)."""
  247. from agent.model_metadata import _estimate_tools_tokens_rough
  248. from model_tools import get_toolset_for_tool
  249. names = tuple(_tool_names(tools_for_api))
  250. enabled = list(_tool_names(getattr(agent, "tools", None) or tools_for_api))
  251. extra = getattr(agent, "valid_tool_names", None) or ()
  252. enabled_names = set(enabled) | {n for n in extra if isinstance(n, str)}
  253. toolsets = frozenset(toolset_metric_name(get_toolset_for_tool(n)) for n in enabled_names)
  254. return names, toolsets, len(enabled_names), _estimate_tools_tokens_rough(tools_for_api) if tools_for_api else 0
  255. def _tool_names(tools: Iterable[Any]) -> Iterable[str]:
  256. for tool in tools or ():
  257. fn = tool.get("function") if isinstance(tool, dict) else None
  258. name = fn.get("name") if isinstance(fn, dict) else (tool.get("name") if isinstance(tool, dict) else None)
  259. if isinstance(name, str):
  260. yield name
  261. def record_cache_break(agent: Any, cause: str) -> None:
  262. """Hermes itself invalidated the conversation's cached prefix (compression, a rebuilt prompt)."""
  263. try:
  264. if cause not in contract.CACHE_BREAK_CAUSES or getattr(agent, "_persist_disabled", False):
  265. return
  266. if _internal_agent(agent) or not _enabled():
  267. return
  268. from .relay_shared_metrics import record_known_cache_break
  269. record_known_cache_break(cause, _agent_route(agent), str(getattr(agent, "session_id", "") or ""))
  270. except Exception:
  271. logger.debug("Shared-metrics cache break not recorded", exc_info=True)
  272. def record_prompt_rebuild(agent: Any, stored_prompt: Any, stored_state: str, rebuilt: Any) -> None:
  273. """A continuing conversation rebuilt its system prompt instead of replaying the stored bytes."""
  274. if stored_state in ("null", "empty"):
  275. record_cache_break(agent, "system_prompt_rebuild")
  276. return
  277. if stored_state != "stale_runtime" or not stored_prompt or rebuilt == stored_prompt:
  278. return
  279. try:
  280. from agent.surface_switch import identity_line_value
  281. switched = any(
  282. (stored := identity_line_value(stored_prompt, label)) and stored != str(getattr(agent, attr, "") or "").strip()
  283. for label, attr in (("Model", "model"), ("Provider", "provider"))
  284. )
  285. except Exception:
  286. switched = False
  287. record_cache_break(agent, "model_switch" if switched else "system_prompt_rebuild")