shared_metrics_fields.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298
  1. """Bounded field builders for the decision-data shared metrics.
  2. Sessions, install milestones, setup completion, token volumes, compression, model switches,
  3. fallbacks, slash commands and extension installs. Every builder takes RAW runtime values and
  4. returns only closed-enum, bucketed or public-catalog dimensions (see shared_metrics_contract).
  5. """
  6. from __future__ import annotations
  7. from typing import Any
  8. from . import shared_metrics_catalog as catalog
  9. from . import shared_metrics_contract as contract
  10. from .shared_metrics_contract import (
  11. MODEL_IDENTIFIER_MAX_LENGTH, PROVIDER_IDENTIFIER_MAX_LENGTH, _bucket, _metric_identifier, _norm,
  12. )
  13. from .shared_metrics_signals import user_created_skill
  14. _MINUTE_MS = 60_000
  15. _SESSION_DURATION_THRESHOLDS = (
  16. (_MINUTE_MS, "lt_1m"), (5 * _MINUTE_MS, "1m_to_5m"), (30 * _MINUTE_MS, "5m_to_30m"),
  17. (120 * _MINUTE_MS, "30m_to_2h"), (480 * _MINUTE_MS, "2h_to_8h"),
  18. )
  19. _HOUR_S = 3_600
  20. _INSTALL_AGE_THRESHOLDS = (
  21. (_HOUR_S, "lt_1h"), (24 * _HOUR_S, "1h_to_1d"), (7 * 24 * _HOUR_S, "1d_to_7d"),
  22. (30 * 24 * _HOUR_S, "7d_to_30d"), (90 * 24 * _HOUR_S, "30d_to_90d"),
  23. )
  24. _TTFT_THRESHOLDS = (
  25. (0.5, "lt_500ms"), (1.0, "500ms_to_1s"), (2.0, "1s_to_2s"), (5.0, "2s_to_5s"), (15.0, "5s_to_15s"),
  26. )
  27. _CONTEXT_FILL_THRESHOLDS = ((50, "lt_50"), (75, "50_to_75"), (90, "75_to_90"), (100, "90_to_100"))
  28. # A session this long is the "real use" signal (vs a one-shot try-out).
  29. LONG_SESSION_TURNS = 6
  30. def _number(value: Any) -> float | None:
  31. return contract._non_negative_number(value)
  32. def provider_identifier(value: Any) -> str:
  33. return catalog.provider_metric_name(value)
  34. def session_fields(
  35. start_fields: dict[str, str], *, turns: int, failed_turns: int, last_outcome: str, active_ms: int,
  36. ) -> dict[str, str]:
  37. """One closed session: where it ran, how many turns, how long it was active, how it ended."""
  38. return {
  39. "active_duration_bucket": _bucket(max(0, active_ms), _SESSION_DURATION_THRESHOLDS, "gte_8h"),
  40. "entrypoint": start_fields.get("entrypoint", "unknown"),
  41. "execution_surface": start_fields.get("execution_surface", "unknown"),
  42. "failed_turn_count_bucket": contract.count_bucket(failed_turns),
  43. "last_outcome": last_outcome if last_outcome in contract.TASK_OUTCOMES else "unknown",
  44. "platform": start_fields.get("platform", "none"),
  45. "turn_count_bucket": contract.size_bucket(turns),
  46. }
  47. def install_age_bucket(age_seconds: Any) -> str:
  48. age = _number(age_seconds)
  49. return "unknown" if age is None else _bucket(age, _INSTALL_AGE_THRESHOLDS, "gte_90d")
  50. def ttft_bucket(event: dict[str, Any]) -> str:
  51. """Time to first streamed chunk for one primary call; ``not_streamed`` when nothing streamed."""
  52. started, first = _number(event.get("started_at")), _number(event.get("first_chunk_at"))
  53. if started is None:
  54. return "unknown"
  55. if first is None:
  56. return "not_streamed"
  57. return _bucket(max(0.0, first - started), _TTFT_THRESHOLDS, "gte_15s")
  58. def context_fill_bucket(tokens_before: Any, context_length: Any) -> str:
  59. tokens, length = _number(tokens_before), _number(context_length)
  60. if tokens is None or not length:
  61. return "unknown"
  62. return _bucket(100 * tokens / length, _CONTEXT_FILL_THRESHOLDS, "gte_100")
  63. _USAGE_KEYS = {
  64. "input": "input_tokens", "output": "output_tokens", "cache_read": "cache_read_tokens",
  65. "cache_write": "cache_write_tokens", "reasoning": "reasoning_tokens",
  66. }
  67. def model_token_fields(
  68. usage: Any, *, model: Any, provider: Any, call_role: str, aux_task: Any = None,
  69. ) -> dict[str, Any] | None:
  70. """Token-usage mark data (dimensions + integer amounts); None when there is no usage."""
  71. if not isinstance(usage, dict):
  72. return None
  73. amounts = {}
  74. for token_type, key in _USAGE_KEYS.items():
  75. value = usage.get(key)
  76. amounts[token_type] = value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0
  77. if not any(amounts.values()):
  78. return None
  79. return {
  80. "aux_task": catalog.aux_task_metric_name(aux_task) if call_role == "auxiliary" else "none",
  81. "call_role": call_role,
  82. "model": catalog.model_metric_name(
  83. model, provider_identifier(provider), max_length=MODEL_IDENTIFIER_MAX_LENGTH,
  84. ),
  85. "provider": provider_identifier(provider),
  86. **amounts,
  87. }
  88. def setup_completed_fields(*, surface: Any, provider: Any) -> dict[str, str]:
  89. value = _norm(surface)
  90. return {
  91. "provider": provider_identifier(provider) if provider else "none",
  92. "surface": value if value in contract.SETUP_SURFACES else "other",
  93. }
  94. _COMPRESSION_TRIGGERS = {
  95. **dict.fromkeys(("auto", "threshold", "preflight", "auto_threshold"), "auto"),
  96. # The attempt log's precise automatic triggers stay one coarse bucket here.
  97. **dict.fromkeys(
  98. ("idle", "turn_start_threshold", "engine_preflight", "pre_api", "post_tool", "gateway_hygiene"), "auto",
  99. ),
  100. **dict.fromkeys(("overflow", "context_overflow", "overflow_error", "error"), "overflow"),
  101. **dict.fromkeys(("manual", "user", "command", "slash"), "manual"),
  102. }
  103. _COMPRESSION_OUTCOMES = {
  104. **dict.fromkeys(("success", "ok", "compressed", "completed"), "success"),
  105. **dict.fromkeys(("skipped", "noop", "not_needed"), "skipped"),
  106. }
  107. # On a committed attempt, a benched summary model led to a deterministic summary;
  108. # the existing ``feasibility_skip`` class counts that fallback.
  109. _COMPRESSION_FAILURE_CLASS_ALIASES = {"summary_model_benched": "feasibility_skip"}
  110. def compression_failure_class(outcome: str, failure_class: Any) -> str:
  111. """The attempt log's class, collapsed onto the closed set: ``exception:<Type>`` / ``rollback:<Type>``
  112. keep only the prefix (the type name may be a plugin's). A success reads ``none`` unless it committed
  113. through a fallback (``feasibility_skip``, ``summary_generation_failed``, ``aux_model_fallback``, ...),
  114. which keeps that class so a fallback commit never reads as a clean one."""
  115. value = _norm(failure_class if isinstance(failure_class, str) else None).split(":", 1)[0]
  116. if outcome == "success":
  117. value = _COMPRESSION_FAILURE_CLASS_ALIASES.get(value, value)
  118. if value in {"", "none"}:
  119. return "none"
  120. if not value:
  121. return "unknown"
  122. return value if value in contract.COMPRESSION_FAILURE_CLASSES - {"none", "unknown"} else "other"
  123. def compression_fields(
  124. *, trigger: Any, outcome: Any, tokens_before: Any, context_length: Any, failure_class: Any = None,
  125. ) -> dict[str, str]:
  126. outcome_value = _COMPRESSION_OUTCOMES.get(_norm(outcome), "failed")
  127. return {
  128. "context_fill_bucket": context_fill_bucket(tokens_before, context_length),
  129. "failure_class": compression_failure_class(outcome_value, failure_class),
  130. "outcome": outcome_value,
  131. "trigger": _COMPRESSION_TRIGGERS.get(_norm(trigger), "other"),
  132. }
  133. def model_switch_fields(*, from_provider: Any, to_provider: Any, surface: Any) -> dict[str, str]:
  134. return {
  135. "execution_surface": contract.execution_surface({"execution_surface": surface}),
  136. "from_provider": provider_identifier(from_provider),
  137. "to_provider": provider_identifier(to_provider),
  138. }
  139. def fallback_fields(*, from_provider: Any, to_provider: Any, reason: Any) -> dict[str, str]:
  140. value = getattr(reason, "value", reason)
  141. return {
  142. "error_class": contract.model_error_class({"reason": value}),
  143. "from_provider": provider_identifier(from_provider),
  144. "to_provider": provider_identifier(to_provider),
  145. }
  146. def slash_command_fields(*, command: Any, surface: Any) -> dict[str, str]:
  147. return {
  148. "command": catalog.slash_command_metric_name(command),
  149. "execution_surface": contract.execution_surface({"execution_surface": surface}),
  150. }
  151. def extension_install_fields(
  152. *, kind: Any, source: Any, name: Any, outcome: Any,
  153. # ---- iuf c1 ----
  154. failure_class: Any = None, registry: Any = None, error: Any = None,
  155. # ---- end iuf c1 ----
  156. ) -> dict[str, str] | None:
  157. kind_value = _norm(kind)
  158. if kind_value not in contract.EXTENSION_KINDS:
  159. return None
  160. source_value = _norm(source)
  161. outcome_value = "success" if _norm(outcome) in {"success", "ok", "installed"} else "failed"
  162. return {
  163. "kind": kind_value,
  164. "name": catalog.extension_metric_name(kind_value, name),
  165. "outcome": outcome_value,
  166. "source": source_value if source_value in contract.EXTENSION_SOURCES else "other",
  167. # ---- iuf c1 ----
  168. "failure_class": extension_failure_class(kind_value, outcome_value, failure_class, error),
  169. "registry": extension_registry(kind_value, registry),
  170. # ---- end iuf c1 ----
  171. }
  172. # ---- iuf c1 ----
  173. # Exceptions from these top-level packages are transport failures, whatever their class name.
  174. _NETWORK_MODULES = frozenset({"aiohttp", "h11", "h2", "httpcore", "httpx", "requests", "ssl", "urllib3", "websockets"})
  175. def tagged_failure_class(error: object) -> str | None:
  176. """The ``failure_class`` a raise site set, read statically: an attribute hook on a third-party
  177. exception must not raise from a classifier and replace the user's error."""
  178. import inspect
  179. tagged = inspect.getattr_static(error, "failure_class", None)
  180. return tagged if isinstance(tagged, str) and tagged else None
  181. def exception_failure_class(error: BaseException) -> str:
  182. """Why a raised install failed, from the exception alone: the class its raise site tagged
  183. (``failure_class`` attribute), else its TYPE. Never reads the message, and never runs the
  184. exception's own attribute hooks (a raising ``__getattr__`` or property reads by type)."""
  185. tagged = tagged_failure_class(error)
  186. if tagged:
  187. return tagged
  188. import socket
  189. import urllib.error
  190. if isinstance(error, PermissionError):
  191. return "permission"
  192. if isinstance(error, (ConnectionError, TimeoutError, socket.gaierror, urllib.error.URLError)) or any(
  193. (getattr(cls, "__module__", "") or "").split(".", 1)[0] in _NETWORK_MODULES for cls in type(error).__mro__):
  194. return "network"
  195. if isinstance(error, OSError):
  196. return "filesystem_error"
  197. return f"exception:{type(error).__name__}"
  198. def extension_failure_class(kind: str, outcome: str, failure_class: Any, error: Any = None) -> str:
  199. """``none`` on success; else the caller's class (or the raised error's), collapsed onto the kind's
  200. closed set: ``exception:<Type>`` keeps only its prefix, anything unlisted reads ``other``."""
  201. if outcome == "success":
  202. return "none"
  203. value = failure_class if isinstance(failure_class, str) and failure_class else None
  204. if value is None and isinstance(error, BaseException):
  205. value = exception_failure_class(error)
  206. value = _norm(value).split(":", 1)[0]
  207. return value if value in contract.EXTENSION_KIND_FAILURE_CLASSES[kind] - {"none"} else "other"
  208. def extension_registry(kind: str, registry: Any) -> str:
  209. """The skills-hub adapter id for a skill row; plugin and MCP rows are ``none``."""
  210. if kind != "skill":
  211. return "none"
  212. # The hub's own alias table (`--source skills.sh`); already imported by the install it records.
  213. from tools.skills_hub_install import _SOURCE_ID_ALIASES
  214. value = _norm(registry) or "none"
  215. value = _SOURCE_ID_ALIASES.get(value, value)
  216. return value if value in contract.EXTENSION_REGISTRIES else "other"
  217. # ---- end iuf c1 ----
  218. def _long_session(d: dict[str, str]) -> bool:
  219. return d.get("turn_count_bucket") not in {"0", "1", "2", "3_to_5"}
  220. # metric -> [(milestone, predicate over its dimensions)]
  221. _MILESTONE_RULES = {
  222. contract.TASK_STARTED_METRIC: (
  223. ("first_task_started", lambda d: True),
  224. ("first_gateway_message", lambda d: d.get("platform") not in {None, "none"}),
  225. ("first_scheduled_task", lambda d: d.get("entrypoint") == "scheduled_task"),
  226. ),
  227. contract.TASK_FINISHED_METRIC: (("first_task_success", lambda d: d.get("outcome") == "success"),),
  228. contract.TOOL_USAGE_METRIC: (
  229. ("first_tool_success", lambda d: d.get("outcome") == "success"),
  230. ("first_mcp_tool_success", lambda d: d.get("outcome") == "success" and d.get("tool_name") == "mcp"),
  231. ("first_delegation", lambda d: d.get("outcome") == "success" and d.get("tool_name") == "delegate_task"),
  232. ),
  233. contract.SKILL_LIFECYCLE_METRIC: (("first_skill_created", user_created_skill),),
  234. contract.SKILL_LOAD_METRIC: (("first_skill_reused", lambda d: d.get("reuse_state") == "reused"),),
  235. contract.SESSION_METRIC: (("first_long_session", _long_session),),
  236. contract.SETUP_COMPLETED_METRIC: (("setup_completed", lambda d: True),),
  237. }
  238. def milestones_for(metric_name: str, dimensions: dict[str, str]) -> tuple[str, ...]:
  239. """Install milestones a recorded counter reaches (each is latched once per install by the store)."""
  240. return tuple(name for name, reached in _MILESTONE_RULES.get(metric_name, ()) if reached(dimensions))