| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363 |
- """Efficiency shared metrics: what one user turn costs, what users throw away, and where Hermes
- spends tokens it did not have to (spilled tool output, idle tool schemas, prompt-cache breaks).
- Per-turn state lives on the relay runtime's task/session objects (see ``TurnCost`` and
- ``SessionEfficiency``); the agent-side entry points below run on the agent thread and are no-ops
- unless shared metrics are enabled. Provider/model always go through the catalog helpers.
- """
- from __future__ import annotations
- import logging
- import re
- from collections import deque
- from dataclasses import dataclass, field
- from typing import Any, Iterable
- from . import shared_metrics_contract as contract
- from .shared_metrics_contract import _bucket, _non_negative_number
- logger = logging.getLogger(__name__)
- _TOKEN_THRESHOLDS = (
- (2_000, "lt_2k"), (10_000, "2k_to_10k"), (50_000, "10k_to_50k"), (200_000, "50k_to_200k"),
- (1_000_000, "200k_to_1m"),
- )
- _ACTIVITY_THRESHOLDS = (
- (1, "0"), (2, "1"), (3, "2"), (6, "3_to_5"), (11, "6_to_10"), (26, "11_to_25"), (51, "26_to_50"),
- (101, "51_to_100"),
- )
- _OUTPUT_SIZE_THRESHOLDS = (
- (1_000, "lt_1k"), (10_000, "1k_to_10k"), (50_000, "10k_to_50k"), (100_000, "50k_to_100k"),
- (500_000, "100k_to_500k"),
- )
- _SCHEMA_TOKEN_THRESHOLDS = (
- (1, "0"), (2_000, "lt_2k"), (5_000, "2k_to_5k"), (10_000, "5k_to_10k"), (20_000, "10k_to_20k"),
- (40_000, "20k_to_40k"),
- )
- _TASK_COST_OUTCOMES = {"success": "completed", "cancelled": "interrupted"}
- # Anthropic's default prompt-cache TTL: a cold read after this idle gap is expiry, not a bug.
- CACHE_TTL_S = 300.0
- # Per-session memory of spent turns an /undo or /retry can still reach.
- _SPENT_TURNS_KEPT = 64
- _PENDING_OUTPUTS_ATTR = "_shared_metrics_tool_outputs"
- _PENDING_OUTPUTS_MAX = 256
- # The one notice tools/tool_output_truncate.py writes when terminal, execute_code or MCP output was
- # cut down inside the tool, before Hermes ever saw the full text.
- _TOOL_TRUNCATION_NOTICE = re.compile(r"\[[A-Z_ ]{1,32} TRUNCATED - [\d,]{1,20} chars omitted out of ([\d,]{1,20}) total\]")
- _NOTICE_MIN_CHARS = 1_000
- def tokens_bucket(tokens: int | None) -> str:
- return "unknown" if tokens is None else _bucket(max(0, tokens), _TOKEN_THRESHOLDS, "gte_1m")
- def activity_bucket(count: int) -> str:
- return _bucket(max(0, int(count)), _ACTIVITY_THRESHOLDS, "gte_101")
- def output_size_bucket(size: int | None) -> str:
- return "unknown" if size is None else _bucket(max(0, size), _OUTPUT_SIZE_THRESHOLDS, "gte_500k")
- def schema_tokens_bucket(tokens: int) -> str:
- return _bucket(max(0, int(tokens)), _SCHEMA_TOKEN_THRESHOLDS, "gte_40k")
- def _int(value: Any) -> int | None:
- number = _non_negative_number(value)
- return None if number is None else int(number)
- def call_tokens(usage: Any) -> int | None:
- """Prompt (cache reads/writes included) plus completion tokens of one call; None without usage."""
- if not isinstance(usage, dict):
- return None
- total = _int(usage.get("total_tokens"))
- if total is not None:
- return total
- parts = [_int(usage.get(key)) for key in (
- "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens")]
- return sum(p for p in parts if p is not None) if any(p is not None for p in parts) else None
- def toolset_metric_name(toolset: Any) -> str:
- """A shipped toolset's own name; MCP servers, plugins and user toolsets read ``custom``."""
- return toolset if isinstance(toolset, str) and toolset in contract.BUILTIN_TOOLSET_NAMES else "custom"
- def _route(route: dict[str, str] | None) -> dict[str, str]:
- from .shared_metrics_model import model_route
- return dict(route) if route else model_route(None, None)
- # ---- per-turn cost and waste (runtime state; caller holds session.lock) ------------------------
- @dataclass
- class TurnCost:
- """Token spend of one task; ``user_turn`` marks a turn a user message started (pre_llm_call),
- which Hermes-owned forks sharing the session id never fire."""
- user_turn: bool = False
- tokens: int = 0
- known: bool = False
- route: dict[str, str] | None = None
- def add_call(self, route: dict[str, str], usage: Any) -> None:
- self.route = route
- tokens = call_tokens(usage)
- if tokens is not None:
- self.tokens += tokens
- self.known = True
- def bucket(self) -> str:
- return tokens_bucket(self.tokens if self.known else None)
- @dataclass
- class _SpentTurn:
- route: dict[str, str]
- tokens_bucket: str
- wasted: bool = False
- def _wasted_row(reason: str, turn: _SpentTurn) -> tuple[str, dict[str, str]]:
- return contract.WASTED_TOKENS_MARK, {**turn.route, "reason": reason, "tokens_bucket": turn.tokens_bucket}
- @dataclass
- class ToolUsage:
- """What the session paid to carry tool definitions, and which shipped toolsets it touched."""
- observed: bool = False
- surface: str = "unknown"
- enabled_toolsets: frozenset[str] = frozenset()
- enabled_count: int = 0
- schema_tokens: int = 0
- names: tuple[str, ...] | None = None
- used: set[str] = field(default_factory=set)
- def observe(self, names: tuple[str, ...], toolsets: frozenset[str], enabled_count: int, schema_tokens: int) -> bool:
- """Record the tools one request sent; True when a later request changed the array."""
- changed = self.names is not None and names != self.names
- self.observed, self.names = True, names
- self.enabled_toolsets = self.enabled_toolsets | toolsets
- self.enabled_count, self.schema_tokens = max(self.enabled_count, enabled_count), schema_tokens
- return changed
- def absorb(self, other: ToolUsage) -> None:
- if not other.observed:
- return
- self.observed, self.surface = True, other.surface
- self.enabled_toolsets = self.enabled_toolsets | other.enabled_toolsets
- self.enabled_count = max(self.enabled_count, other.enabled_count)
- self.schema_tokens = max(self.schema_tokens, other.schema_tokens)
- self.used |= other.used
- def rows(self) -> list[tuple[str, dict[str, str]]]:
- if not self.observed:
- return []
- rows = [(contract.TOOL_OVERHEAD_MARK, {
- "enabled_tool_count_bucket": contract.size_bucket(self.enabled_count),
- "execution_surface": self.surface,
- "tool_schema_tokens_bucket": schema_tokens_bucket(self.schema_tokens),
- })]
- rows.extend(
- (contract.TOOL_ENABLED_UNUSED_MARK, {"toolset": name, "used": "yes" if name in self.used else "no"})
- for name in sorted(self.enabled_toolsets)
- )
- return rows
- @dataclass
- class SessionEfficiency:
- spent: deque[_SpentTurn] = field(default_factory=lambda: deque(maxlen=_SPENT_TURNS_KEPT))
- tools: ToolUsage = field(default_factory=ToolUsage)
- tools_key: tuple[int, int, int] | None = None
- cache_read: int = 0
- cache_route: dict[str, str] | None = None
- cache_ended_ns: int = 0
- def finish_turn(
- self, cost: TurnCost, terminal: dict[str, str], *, route: dict[str, str] | None,
- model_calls: int, tool_calls: int,
- ) -> list[tuple[str, dict[str, str]]]:
- """task_cost for one user turn the user saw end; an interrupt also counts its tokens wasted."""
- turn = _SpentTurn(_route(cost.route or route), cost.bucket())
- rows = [(contract.TASK_COST_MARK, {
- **turn.route,
- "api_calls_bucket": activity_bucket(model_calls),
- "outcome": _TASK_COST_OUTCOMES.get(terminal.get("outcome", ""), "failed"),
- "tokens_bucket": turn.tokens_bucket,
- "tool_calls_bucket": activity_bucket(tool_calls),
- })]
- if terminal.get("end_reason") == "user_cancelled":
- turn.wasted = True
- rows.append(_wasted_row("interrupt", turn))
- self.spent.append(turn)
- return rows
- def discard_turns(self, reason: str, turns: int, fallback_route: dict[str, str]) -> list[tuple[str, dict[str, str]]]:
- """An /undo or /retry threw away the newest ``turns`` turns: one row per turn not already
- counted (an interrupted turn the user then undoes was wasted once, at the interrupt)."""
- rows = []
- for _ in range(max(1, min(int(turns), _SPENT_TURNS_KEPT))):
- turn = self.spent.pop() if self.spent else _SpentTurn(_route(fallback_route), "unknown")
- if not turn.wasted:
- rows.append(_wasted_row(reason, turn))
- return rows
- def observe_cache(
- self, route: dict[str, str], usage: Any, started_ns: int, ended_ns: int, *, expected: bool,
- ) -> str | None:
- """A cold primary read after a warm one on the same route is a break Hermes did not announce
- (``expected``: Hermes already counted the cause); a gap past the cache TTL is plain expiry."""
- read = _int(usage.get("cache_read_tokens")) if isinstance(usage, dict) else None
- if read is None:
- return None
- prev_read, prev_route, prev_ended = self.cache_read, self.cache_route, self.cache_ended_ns
- self.cache_read, self.cache_route, self.cache_ended_ns = read, route, ended_ns
- if expected or read or not prev_read or prev_route != route:
- return None
- idle_s = (started_ns - prev_ended) / 1e9
- return "cache_expired" if idle_s >= CACHE_TTL_S else "provider_reported_miss"
- def cache_break_row(cause: str, route: dict[str, str] | None) -> tuple[str, dict[str, str]]:
- return contract.CACHE_BREAK_MARK, {**_route(route), "cause": cause}
- # ---- agent-side entry points (agent thread; never raise) ---------------------------------------
- def _internal_agent(agent: Any) -> bool:
- """Hermes-owned forks (background review, curator) share the user's session id."""
- return getattr(agent, "_memory_write_origin", None) == "background_review"
- def _agent_route(agent: Any) -> dict[str, str]:
- from .shared_metrics_model import model_route
- return model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
- def _enabled() -> bool:
- from .relay_shared_metrics import enabled
- return enabled()
- def note_tool_result(agent: Any, tool_name: Any, tool_call_id: Any, raw: Any, persisted: Any) -> None:
- """Remember one committed tool result until its batch's turn budget has run."""
- try:
- if _internal_agent(agent) or not _enabled():
- return
- pending = agent.__dict__.setdefault(_PENDING_OUTPUTS_ATTR, {})
- if len(pending) >= _PENDING_OUTPUTS_MAX:
- return
- size, cut_in_tool = _result_size(raw)
- pending[str(tool_call_id or id(persisted))] = [
- contract.tool_metric_name({"tool_name": tool_name}), size, cut_in_tool or persisted is not raw,
- ]
- except Exception:
- logger.debug("Shared-metrics tool output not noted", exc_info=True)
- def _result_size(result: Any) -> tuple[int | None, bool]:
- """(characters the tool produced, whether the tool itself already truncated them)."""
- if isinstance(result, dict): # multimodal envelope: its text parts are what can spill
- parts = result.get("content") or []
- return sum(len(p.get("text") or "") for p in parts if isinstance(p, dict) and p.get("type") == "text"), False
- if not isinstance(result, str):
- return None, False
- notice = _TOOL_TRUNCATION_NOTICE.search(result) if len(result) >= _NOTICE_MIN_CHARS else None
- if notice is None:
- return len(result), False
- return max(len(result), int(notice.group(1).replace(",", ""))), True
- def record_tool_batch(agent: Any, batch: list[dict], contents_before: list[Any]) -> None:
- """One row per tool result of the batch: truncated when the per-result cap or the per-turn
- budget replaced it (the budget runs after the cap, so a result is judged once, here)."""
- try:
- pending = agent.__dict__.pop(_PENDING_OUTPUTS_ATTR, None)
- if not pending or not _enabled():
- return
- for message, before in zip(batch, contents_before):
- entry = pending.get(str(message.get("tool_call_id") or ""))
- if entry is not None and message.get("content") is not before:
- entry[2] = True
- from .relay_shared_metrics import record_process_mark
- for tool, size, truncated in pending.values():
- record_process_mark(contract.TOOL_OUTPUT_TRUNCATION_MARK, {
- "original_size_bucket": output_size_bucket(size), "tool": tool, "truncated": "yes" if truncated else "no",
- })
- except Exception:
- logger.debug("Shared-metrics tool output truncation not recorded", exc_info=True)
- def observe_request_tools(agent: Any, tools_for_api: Any) -> None:
- """The tool definitions one primary request carries (evidence for trimming default toolsets)."""
- try:
- if getattr(agent, "_persist_disabled", False) or _internal_agent(agent) or not _enabled():
- return
- from .relay_shared_metrics import record_session_tools
- record_session_tools(str(getattr(agent, "session_id", "") or ""), agent, tools_for_api or [])
- except Exception:
- logger.debug("Shared-metrics tool overhead not observed", exc_info=True)
- def tool_snapshot(agent: Any, tools_for_api: list) -> tuple[tuple[str, ...], frozenset[str], int, int]:
- """(sent tool names, enabled shipped toolsets, enabled tool count, estimated schema tokens)."""
- from agent.model_metadata import _estimate_tools_tokens_rough
- from model_tools import get_toolset_for_tool
- names = tuple(_tool_names(tools_for_api))
- enabled = list(_tool_names(getattr(agent, "tools", None) or tools_for_api))
- extra = getattr(agent, "valid_tool_names", None) or ()
- enabled_names = set(enabled) | {n for n in extra if isinstance(n, str)}
- toolsets = frozenset(toolset_metric_name(get_toolset_for_tool(n)) for n in enabled_names)
- return names, toolsets, len(enabled_names), _estimate_tools_tokens_rough(tools_for_api) if tools_for_api else 0
- def _tool_names(tools: Iterable[Any]) -> Iterable[str]:
- for tool in tools or ():
- fn = tool.get("function") if isinstance(tool, dict) else None
- name = fn.get("name") if isinstance(fn, dict) else (tool.get("name") if isinstance(tool, dict) else None)
- if isinstance(name, str):
- yield name
- def record_cache_break(agent: Any, cause: str) -> None:
- """Hermes itself invalidated the conversation's cached prefix (compression, a rebuilt prompt)."""
- try:
- if cause not in contract.CACHE_BREAK_CAUSES or getattr(agent, "_persist_disabled", False):
- return
- if _internal_agent(agent) or not _enabled():
- return
- from .relay_shared_metrics import record_known_cache_break
- record_known_cache_break(cause, _agent_route(agent), str(getattr(agent, "session_id", "") or ""))
- except Exception:
- logger.debug("Shared-metrics cache break not recorded", exc_info=True)
- def record_prompt_rebuild(agent: Any, stored_prompt: Any, stored_state: str, rebuilt: Any) -> None:
- """A continuing conversation rebuilt its system prompt instead of replaying the stored bytes."""
- if stored_state in ("null", "empty"):
- record_cache_break(agent, "system_prompt_rebuild")
- return
- if stored_state != "stale_runtime" or not stored_prompt or rebuilt == stored_prompt:
- return
- try:
- from agent.surface_switch import identity_line_value
- switched = any(
- (stored := identity_line_value(stored_prompt, label)) and stored != str(getattr(agent, attr, "") or "").strip()
- for label, attr in (("Model", "model"), ("Provider", "provider"))
- )
- except Exception:
- switched = False
- record_cache_break(agent, "model_switch" if switched else "system_prompt_rebuild")
|