shared_metrics_model.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222
  1. """Per-model quality, friction and context-pressure shared metrics.
  2. Tool-call quality is counted where the agent validates the calls a model emitted (every call,
  3. clean ones as ``issue=none``, so rates have a denominator). Friction and context peaks are
  4. attributed to the model that produced the turn, which the relay runtime tracks per metrics session
  5. in :class:`ModelSessionState`. Provider/model always go through ``model_call_fields`` (the catalog
  6. helpers), so custom endpoints and loopback servers read ``custom``.
  7. """
  8. from __future__ import annotations
  9. import json
  10. import logging
  11. from dataclasses import dataclass
  12. from typing import Any, Iterable
  13. from . import shared_metrics_contract as contract
  14. from . import shared_metrics_fields as fields_
  15. logger = logging.getLogger(__name__)
  16. # A session that ends this soon after a failed turn reads as the user giving up on the model.
  17. QUICK_ABANDON_NS = 60 * 1_000_000_000
  18. # Lower bound inclusive, so 128k (128_000 and 131_072) and 1M windows land in one bucket each.
  19. _WINDOW_THRESHOLDS = (
  20. (32_000, "lt_32k"), (128_000, "32k_to_128k"), (256_000, "128k_to_256k"), (1_000_000, "256k_to_1m"),
  21. )
  22. # The rejections Hermes answers with a forced (overflow-triggered) compression.
  23. _LIMIT_ERROR_CLASSES = frozenset({"context_overflow", "payload_too_large"})
  24. # Unattended runs have no user to interrupt or abandon them.
  25. _UNATTENDED_ENTRYPOINTS = frozenset({"background", "batch", "delegated", "scheduled_task"})
  26. def model_route(provider: Any, model: Any) -> dict[str, str]:
  27. return contract.model_call_fields({"provider": provider, "model": model})
  28. def window_bucket(context_length: Any) -> str:
  29. length = contract._non_negative_number(context_length)
  30. return contract._bucket(length, _WINDOW_THRESHOLDS, "gte_1m") if length else "unknown"
  31. def friction_fields(signal: str, route: dict[str, str] | None) -> dict[str, str] | None:
  32. if signal not in contract.FRICTION_SIGNALS:
  33. return None
  34. return {**(route or model_route(None, None)), "signal": signal}
  35. def _prompt_tokens(usage: Any) -> float | None:
  36. if not isinstance(usage, dict):
  37. return None
  38. tokens = contract._non_negative_number(usage.get("prompt_tokens"))
  39. return tokens if tokens is not None else contract._non_negative_number(usage.get("input_tokens"))
  40. @dataclass
  41. class ModelSessionState:
  42. """Per-session model attribution the runtime keeps between hooks (caller holds session.lock)."""
  43. last_route: dict[str, str] | None = None
  44. peak_route: dict[str, str] | None = None
  45. peak_tokens: float = -1.0
  46. peak_window: float | None = None
  47. limit_hit: bool = False
  48. failed_turn_ns: int = 0
  49. failed_route: dict[str, str] | None = None
  50. def observe_call(self, route: dict[str, str], usage: Any, context_length: Any) -> None:
  51. """One finished primary call: remember its model and the fullest context seen."""
  52. self.last_route = route
  53. tokens, window = _prompt_tokens(usage), contract._non_negative_number(context_length)
  54. if self.peak_route is None:
  55. self.peak_route = route
  56. if tokens is None or not window:
  57. return
  58. if self.peak_window is None or tokens / window > self.peak_tokens / self.peak_window:
  59. self.peak_route, self.peak_tokens, self.peak_window = route, tokens, window
  60. def observe_error(self, route: dict[str, str], error_class: str) -> None:
  61. """A failed primary call still names the model: a session whose every call overflowed
  62. is exactly the one the context peak row must report."""
  63. self.last_route = route
  64. if self.peak_route is None:
  65. self.peak_route = route
  66. if error_class in _LIMIT_ERROR_CLASSES:
  67. self.limit_hit = True
  68. def observe_turn(self, outcome: str, route: dict[str, str] | None, now_ns: int) -> None:
  69. """Only the trailing turn matters for abandonment: a later success clears it."""
  70. failed = outcome == "failed"
  71. self.failed_turn_ns = now_ns if failed else 0
  72. self.failed_route = (route or self.last_route) if failed else None
  73. def quick_abandon_route(self, now_ns: int) -> dict[str, str] | None:
  74. if self.failed_turn_ns and now_ns - self.failed_turn_ns <= QUICK_ABANDON_NS:
  75. return self.failed_route or self.last_route
  76. return None
  77. def context_peak_fields(self) -> dict[str, str] | None:
  78. """One row per session that reached a primary model; None when no call was made."""
  79. if self.peak_route is None:
  80. return None
  81. known = self.peak_window is not None
  82. return {
  83. **self.peak_route,
  84. "limit_hit": "yes" if self.limit_hit else "no",
  85. "peak_fill_bucket": (
  86. fields_.context_fill_bucket(self.peak_tokens, self.peak_window) if known else "unknown"
  87. ),
  88. "window_bucket": window_bucket(self.peak_window) if known else "unknown",
  89. }
  90. def attended(start_fields: dict[str, str] | None) -> bool:
  91. return bool(start_fields) and start_fields.get("entrypoint") not in _UNATTENDED_ENTRYPOINTS
  92. # ---- tool-call quality -----------------------------------------------------------------------
  93. def _required_params(tools: Iterable[Any] | None) -> dict[str, tuple[str, ...]]:
  94. required: dict[str, tuple[str, ...]] = {}
  95. for tool in tools or ():
  96. fn = tool.get("function") if isinstance(tool, dict) else None
  97. if not isinstance(fn, dict) or not isinstance(fn.get("name"), str):
  98. continue
  99. params = fn.get("parameters")
  100. names = params.get("required") if isinstance(params, dict) else None
  101. required[fn["name"]] = tuple(n for n in names if isinstance(n, str)) if isinstance(names, list) else ()
  102. return required
  103. def tool_call_issue(
  104. name: Any, raw_args: Any, *, valid_names: Any, required: dict[str, tuple[str, ...]], repaired: bool,
  105. ) -> str:
  106. """The first problem with one emitted call, most severe first; ``none`` when clean.
  107. Empty arguments are only an issue when the tool declares required parameters: for a
  108. parameterless tool Hermes's normalization to ``{}`` is exactly what the model meant.
  109. """
  110. if name not in valid_names:
  111. return "unknown_tool"
  112. needed = required.get(name, ())
  113. if isinstance(raw_args, (dict, list)):
  114. parsed = raw_args
  115. elif raw_args is None or not str(raw_args).strip():
  116. return "empty_arguments" if needed else ("repaired" if repaired else "none")
  117. else:
  118. try:
  119. parsed = json.loads(str(raw_args))
  120. except (TypeError, ValueError):
  121. return "invalid_json"
  122. if not isinstance(parsed, dict) or any(key not in parsed for key in needed):
  123. return "schema_mismatch"
  124. return "repaired" if repaired else "none"
  125. def record_tool_call_quality(agent: Any, tool_calls: Iterable[Any], repaired_ids: frozenset[int] | set[int]) -> None:
  126. """Count every tool call one primary response emitted, before Hermes normalizes it.
  127. ``repaired_ids`` holds ``id(tc)`` of calls whose name Hermes auto-repaired; streamed calls
  128. whose argument JSON was repaired carry ``function.args_repaired``. No-op unless enabled.
  129. """
  130. try:
  131. from .relay_shared_metrics import enabled, record_process_mark
  132. if not tool_calls or not enabled():
  133. return
  134. from .shared_metrics_signals import tool_unavailable_fields
  135. from .shared_metrics_connectors import tool_batch_fields
  136. route = model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
  137. valid_names = getattr(agent, "valid_tool_names", None) or frozenset()
  138. required = _required_params(getattr(agent, "tools", None))
  139. tool_calls = list(tool_calls)
  140. if (batch := tool_batch_fields(route, tool_calls)) is not None:
  141. record_process_mark(contract.TOOL_BATCH_MARK, batch)
  142. for tc in tool_calls:
  143. fn = tc.function
  144. issue = tool_call_issue(
  145. fn.name, fn.arguments, valid_names=valid_names, required=required,
  146. repaired=id(tc) in repaired_ids or getattr(fn, "args_repaired", False) is True,
  147. )
  148. record_process_mark(contract.MODEL_TOOL_QUALITY_MARK, {**route, "call_role": "primary", "issue": issue})
  149. if (unavailable := tool_unavailable_fields(agent, fn.name, issue, route)) is not None:
  150. record_process_mark(contract.TOOL_UNAVAILABLE_MARK, unavailable)
  151. except Exception:
  152. logger.debug("Shared-metrics tool-call quality not recorded", exc_info=True)
  153. # ---- friction --------------------------------------------------------------------------------
  154. def record_model_friction(
  155. signal: str, *, session_id: Any = None, agent: Any = None, provider: Any = None, model: Any = None,
  156. hermes_home: Any = None, turns: int = 1,
  157. ) -> None:
  158. """Count one user friction action against the model that produced the session's last turn.
  159. ``turns``: how many turns an /undo removed (their tokens count as wasted).
  160. Falls back to ``agent`` (or ``provider``/``model``) when this process never saw the session's
  161. turns (restart, remote compute host). ``hermes_home`` binds the owning profile for callers that
  162. run outside it (multiplexed gateway, Desktop sessions of a secondary profile).
  163. """
  164. token = None
  165. try:
  166. if hermes_home:
  167. from hermes_constants import set_hermes_home_override
  168. token = set_hermes_home_override(str(hermes_home))
  169. from .relay_shared_metrics import enabled, record_session_friction
  170. if signal not in contract.FRICTION_SIGNALS or not enabled():
  171. return
  172. if agent is not None:
  173. provider, model = getattr(agent, "provider", provider), getattr(agent, "model", model)
  174. session_id = getattr(agent, "session_id", None) or session_id
  175. record_session_friction(signal, str(session_id or ""), model_route(provider, model), turns)
  176. except Exception:
  177. logger.debug("Shared-metrics %s friction not recorded", signal, exc_info=True)
  178. finally:
  179. if token is not None:
  180. from hermes_constants import reset_hermes_home_override
  181. reset_hermes_home_override(token)