shared_metrics_contract.py 102 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999
  1. """Bounded product contract for the first Hermes shared-metrics slice."""
  2. from __future__ import annotations
  3. from math import isfinite
  4. from typing import Any
  5. from agent.error_classifier import FailoverReason
  6. from agent.relay_runtime import (
  7. LOGICAL_LLM_SCOPE,
  8. RUNTIME_INSTANCE_KEY,
  9. RUNTIME_SCHEMA_KEY,
  10. RUNTIME_SCHEMA_VERSION,
  11. )
  12. from hermes_cli.platforms import PLATFORMS
  13. from tools.connectors.gateway.names import CONNECTOR_NAME_PREFIX
  14. from toolsets import BUILTIN_TOOL_NAMES, BUILTIN_TOOLSET_NAMES
  15. from .shared_metrics_prepared_compaction import (
  16. PREPARED_COMPACTION_DIMENSIONS, PREPARED_COMPACTION_MARK, PREPARED_COMPACTION_METRIC,
  17. PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,
  18. )
  19. from .shared_metrics_busy import BUSY_ARRIVAL_DISPOSITIONS, BUSY_ARRIVAL_METRIC, BUSY_PEAK_FIELD, BUSY_WAIT_BUCKETS
  20. SCHEMA_KEY = "hermes.metrics.schema_version"
  21. # A random per-emit token (no payload) on rows whose producer waits to learn they were SAVED: facts
  22. # recovered from a file that is deleted only then. Events persist on the Relay thread.
  23. COMMIT_TICKET_KEY = "hermes.shared_metrics.commit_ticket"
  24. SCHEMA_VERSION = "hermes.metrics.event.v4"
  25. MODEL_CALL_SCOPE = "hermes.model_call"
  26. MODEL_CALL_PROFILE_MODEL = "unknown"
  27. TASK_SCOPE = "hermes.task_run"
  28. TOOL_CALL_SCOPE = "hermes.tool_call"
  29. CLIENT_ACTIVE_MARK = "hermes.client.active"
  30. TOOL_APPROVAL_MARK = "hermes.tool_approval"
  31. SKILL_LIFECYCLE_MARK = "hermes.skill.lifecycle"
  32. SKILL_LOAD_MARK = "hermes.skill.load"
  33. INSTALL_SNAPSHOT_MARK = "hermes.install.snapshot"
  34. SESSION_MARK = "hermes.session"
  35. SETUP_COMPLETED_MARK = "hermes.setup.completed"
  36. MODEL_TOKENS_MARK = "hermes.model_tokens"
  37. COMPRESSION_MARK = "hermes.compression"
  38. MODEL_SWITCH_MARK = "hermes.model_switch"
  39. FALLBACK_MARK = "hermes.fallback"
  40. SLASH_COMMAND_MARK = "hermes.slash_command"
  41. EXTENSION_INSTALL_MARK = "hermes.extension.install"
  42. # ---- v4 install ----
  43. STARTUP_LATENCY_MARK = "hermes.startup.latency"
  44. SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics"
  45. CLIENT_ACTIVE_METRIC = "hermes.client.active"
  46. LEGACY_MODEL_CALL_METRIC = "hermes.model_call.count"
  47. MODEL_ROUTE_METRIC = "hermes.model_route.count"
  48. TASK_STARTED_METRIC = "hermes.task_run.started"
  49. TASK_FINISHED_METRIC = "hermes.task_run.finished"
  50. TASK_DURATION_METRIC = "hermes.task_run.duration"
  51. TOOL_CALL_METRIC = "hermes.tool_call.count"
  52. TOOL_LATENCY_METRIC = "hermes.tool_call.latency"
  53. TOOL_APPROVAL_METRIC = "hermes.tool_approval.count"
  54. SKILL_LIFECYCLE_METRIC = "hermes.skill.lifecycle.count"
  55. SKILL_LOAD_METRIC = "hermes.skill.load.count"
  56. TOOL_USAGE_METRIC = "hermes.tool.usage.count"
  57. INSTALL_SNAPSHOT_METRIC = "hermes.install.snapshot"
  58. SESSION_METRIC = "hermes.session.count"
  59. MILESTONE_METRIC = "hermes.install.milestone"
  60. SETUP_COMPLETED_METRIC = "hermes.setup.completed"
  61. MODEL_TOKENS_METRIC = "hermes.model_tokens.sum"
  62. COMPRESSION_METRIC = "hermes.compression.count"
  63. MODEL_SWITCH_METRIC = "hermes.model_switch.count"
  64. FALLBACK_METRIC = "hermes.fallback.count"
  65. SLASH_COMMAND_METRIC = "hermes.slash_command.count"
  66. EXTENSION_INSTALL_METRIC = "hermes.extension.install.count"
  67. # ---- v4 loop ----
  68. MEMORY_OP_MARK = "hermes.memory.op"
  69. CURATOR_RUN_MARK = "hermes.curator.run"
  70. DELEGATION_RUN_MARK = "hermes.delegation.run"
  71. EXECUTION_BACKEND_MARK = "hermes.execution_backend"
  72. MEMORY_OP_METRIC = "hermes.memory.op.count"
  73. CURATOR_RUN_METRIC = "hermes.curator.run.count"
  74. DELEGATION_RUN_METRIC = "hermes.delegation.run.count"
  75. EXECUTION_BACKEND_METRIC = "hermes.execution_backend.count"
  76. # ---- end v4 loop ----
  77. # ---- v4 model ----
  78. MODEL_TOOL_QUALITY_MARK = "hermes.model_tool_quality"
  79. MODEL_FRICTION_MARK = "hermes.model_friction"
  80. CONTEXT_PEAK_MARK = "hermes.context_peak"
  81. MODEL_TOOL_QUALITY_METRIC = "hermes.model_tool_quality.count"
  82. MODEL_FRICTION_METRIC = "hermes.model_friction.count"
  83. CONTEXT_PEAK_METRIC = "hermes.context_peak.count"
  84. # ---- end v4 model ----
  85. # ---- v4 install ----
  86. STARTUP_LATENCY_METRIC = "hermes.startup.latency"
  87. # ---- v5 engagement ----
  88. # Not a counter: one attended turn's serving model, folded into the local daily engagement rollup.
  89. ENGAGEMENT_TURN_MARK = "hermes.engagement.turn"
  90. ENGAGEMENT_DAY_METRIC = "hermes.engagement.day.count"
  91. ENGAGEMENT_SURFACE_METRIC = "hermes.engagement.surface_day.count"
  92. MODEL_SWITCH_AFTER_MARK = "hermes.model_switch_after"
  93. MODEL_SWITCH_AFTER_METRIC = "hermes.model_switch_after.count"
  94. # ---- end v5 engagement ----
  95. MODEL_IDENTIFIER_MAX_LENGTH = 256
  96. PROVIDER_IDENTIFIER_MAX_LENGTH = 64
  97. _METRIC_IDENTIFIER_CHARACTERS = frozenset("abcdefghijklmnopqrstuvwxyz0123456789._:/@+-")
  98. _METRIC_IDENTIFIER_START_CHARACTERS = frozenset("abcdefghijklmnopqrstuvwxyz0123456789")
  99. EXECUTION_SURFACES = frozenset({
  100. "acp", "api", "batch", "cli", "desktop", "gateway", "python", "scheduled_task", "tui",
  101. "other", "unknown",
  102. })
  103. TASK_OUTCOMES = frozenset({"cancelled", "failed", "success", "timed_out", "unknown"})
  104. TASK_END_REASONS = frozenset({
  105. "approval_denied", "completed", "failed", "guardrail_blocked", "iteration_limit",
  106. "system_aborted", "timed_out", "unknown", "user_cancelled",
  107. })
  108. TASK_TERMINATIONS = frozenset({"none", "system_aborted", "timed_out", "unknown", "user_cancelled"})
  109. # ``one_shot``: a finite CLI run (``hermes -z``, ``hermes chat -q`` off a TTY, ``-Q``, ``--oneshot``) that
  110. # answers one prompt and exits — a person's shell line or their script, never the REPL.
  111. TASK_ENTRYPOINTS = frozenset({
  112. "api", "background", "batch", "delegated", "gateway_message", "interactive", "one_shot", "other",
  113. "python", "scheduled_task", "unknown",
  114. })
  115. DURATION_BUCKETS = frozenset({
  116. "1s_to_5s", "2m_to_10m", "30s_to_2m", "5s_to_30s", "gte_10m", "lt_1s",
  117. })
  118. COUNT_BUCKETS = frozenset({"0", "1", "2", "3_to_5", "6_to_10", "gte_11"})
  119. TOOL_CATEGORIES = frozenset({
  120. "browser", "code_execution", "communication", "computer_use", "connector", "delegation", "file",
  121. "home_automation", "mcp", "media", "memory", "other", "planning", "project", "scheduler",
  122. "skill", "terminal", "unknown", "web",
  123. })
  124. TOOL_OUTCOMES = frozenset({"blocked", "cancelled", "failed", "success", "timed_out", "unknown"})
  125. TOOL_APPROVAL_OUTCOMES = frozenset({"approved", "cancelled", "denied", "not_required", "timed_out", "unknown"})
  126. TOOL_APPROVAL_ATTRIBUTIONS = frozenset({"tool_call", "unattributed"})
  127. TOOL_LATENCY_BUCKETS = frozenset({
  128. "100ms_to_250ms", "10s_to_30s", "1s_to_2s", "250ms_to_500ms", "2s_to_5s", "500ms_to_1s",
  129. "5s_to_10s", "gte_30s", "lt_100ms", "unknown",
  130. })
  131. TOOL_RETRY_BUCKETS = COUNT_BUCKETS | frozenset({"unknown"})
  132. SKILL_LIFECYCLE_ACTIONS = frozenset({
  133. "archived", "created", "edited", "installed", "patched", "restored", "stale",
  134. })
  135. SKILL_PROVENANCES = frozenset({"agent_created", "external", "installed", "local", "unknown"})
  136. SKILL_REUSE_STATES = frozenset({"first_use", "reused"})
  137. SKILL_POST_PATCH_STATES = frozenset({"no_new_patch", "not_applicable", "reused_after_patch"})
  138. CLIENT_OS_FAMILIES = frozenset({"linux", "macos", "unknown", "windows"})
  139. CLIENT_ARCHITECTURES = frozenset({"arm", "arm64", "unknown", "x86", "x86_64"})
  140. CLIENT_INSTALL_METHODS = frozenset({
  141. "apt", "docker", "git", "home-manager", "homebrew", "microsoft-store", "nixos", "pip", "unknown",
  142. })
  143. CLIENT_RESOURCE_KEYS = frozenset({"architecture", "hermes_version", "install_method", "os_family"})
  144. # ---- v3 taxonomies -------------------------------------------------------------------------
  145. MODEL_CALL_ROLES = frozenset({"auxiliary", "primary"})
  146. MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"})
  147. # The classifier's own closed enum, so the exported vocabulary is exactly what recovery acts on.
  148. MODEL_ERROR_CLASSES = frozenset(reason.value for reason in FailoverReason) | {"none"}
  149. TASK_FAILURE_CLASSES = MODEL_ERROR_CLASSES | frozenset({
  150. "context_compression", "empty_response", "exception", "local_error", "other", "persistence",
  151. "repeated_errors", "restart_limit", "session_busy", "shutdown",
  152. })
  153. # Surfaces that are not messaging platforms keep their own execution_surface value.
  154. # ``relay``: a connector inbound whose platform neither the inbound nor a single-platform connector names.
  155. _CORE_GATEWAY_PLATFORMS = (frozenset(PLATFORMS) - {"api_server", "cli", "cron"}) | {"none", "plugin", "relay"}
  156. # The Tool Search bridge tools are not in a toolset; every hosted connector tool reads ``connector``.
  157. BRIDGE_TOOL_NAMES = frozenset({"tool_call", "tool_describe", "tool_search"})
  158. # Core tools a stable release shipped that core no longer has: setup_choose/start_chat with the old setup
  159. # flow; the built-in browser_* tools with agent-browser (the camofox plugin's same-named tools record as
  160. # ``plugin``); the five browser_vault_* tools folded into one browser_vault; spotify_* when Spotify left core
  161. # for its catalog plugin. v4 only adds values, and rows older releases saved with these names must still
  162. # package; recording maps through the live sets (tool_metric_name, toolset_metric_name).
  163. RETIRED_TOOL_NAMES = frozenset({
  164. "setup_choose", "start_chat",
  165. "browser_back", "browser_click", "browser_console", "browser_get_images", "browser_navigate",
  166. "browser_press", "browser_scroll", "browser_snapshot", "browser_type", "browser_vision",
  167. "browser_vault_enter_code", "browser_vault_fill", "browser_vault_list", "browser_vault_save_login",
  168. "browser_vault_unlock",
  169. "spotify_albums", "spotify_devices", "spotify_library", "spotify_playback", "spotify_playlists",
  170. "spotify_queue", "spotify_search",
  171. })
  172. RETIRED_TOOLSET_NAMES = frozenset({"setup", "spotify", "start_chat"})
  173. _LIVE_TOOL_NAMES = BUILTIN_TOOL_NAMES | BRIDGE_TOOL_NAMES | {"connector", "mcp", "plugin", "unknown"}
  174. TOOL_NAMES = _LIVE_TOOL_NAMES | RETIRED_TOOL_NAMES
  175. TOOL_ERROR_CLASSES = frozenset({
  176. "alias", "blocked", "connection_required", "connector_backend", "contract_violation", "exception",
  177. "interrupted", "invalid_arguments", "none", "not_allowed", "timeout", "tool_error", "unknown",
  178. })
  179. SIZE_BUCKETS = frozenset({
  180. "0", "1", "2", "3_to_5", "6_to_10", "11_to_25", "26_to_100", "101_to_250", "gte_251",
  181. })
  182. def _bundled_memory_providers() -> frozenset[str]:
  183. from pathlib import Path
  184. root = Path(__file__).resolve().parents[2] / "plugins" / "memory"
  185. try:
  186. return frozenset(p.name for p in root.iterdir() if (p / "__init__.py").is_file())
  187. except OSError:
  188. return frozenset()
  189. class _CatalogValues:
  190. """A closed enum backed by a public catalog (see shared_metrics_catalog), loaded on first use."""
  191. def __init__(self, *loaders: str, extra: frozenset[str]) -> None:
  192. self.loaders, self.extra = loaders, extra
  193. def values(self) -> frozenset[str]:
  194. from . import shared_metrics_catalog as catalog
  195. return self.extra.union(*(catalog._safe(getattr(catalog, name)) for name in self.loaders))
  196. def __contains__(self, value: object) -> bool:
  197. return value in self.extra or value in self.values()
  198. # A provider that moved from plugins/memory/ to the plugin catalog keeps its public name; any other
  199. # third-party provider reports as "plugin".
  200. MEMORY_PROVIDERS = _CatalogValues("plugin_catalog_names", extra=_bundled_memory_providers() | {"builtin", "plugin"})
  201. # ---- decision-data taxonomies ----------------------------------------------------------------
  202. SESSION_DURATION_BUCKETS = frozenset({"lt_1m", "1m_to_5m", "5m_to_30m", "30m_to_2h", "2h_to_8h", "gte_8h"})
  203. INSTALL_AGE_BUCKETS = frozenset({
  204. "lt_1h", "1h_to_1d", "1d_to_7d", "7d_to_30d", "30d_to_90d", "gte_90d", "unknown",
  205. })
  206. MILESTONES = frozenset({
  207. "setup_completed", "first_task_started", "first_task_success", "first_tool_success",
  208. "first_gateway_message", "first_scheduled_task", "first_delegation", "first_skill_created",
  209. "first_skill_reused", "first_mcp_tool_success", "first_long_session",
  210. })
  211. SETUP_SURFACES = frozenset({"cli", "desktop", "other"})
  212. TOKEN_TYPES = frozenset({"input", "output", "cache_read", "cache_write", "reasoning"})
  213. TTFT_BUCKETS = frozenset({
  214. "lt_500ms", "500ms_to_1s", "1s_to_2s", "2s_to_5s", "5s_to_15s", "gte_15s", "not_streamed", "unknown",
  215. })
  216. COMPRESSION_TRIGGERS = frozenset({"auto", "manual", "other", "overflow"})
  217. COMPRESSION_OUTCOMES = frozenset({"failed", "skipped", "success"})
  218. # Why an attempt was skipped or failed: the attempt log's own ``failure_class`` names
  219. # (agent/conversation_compression*.py, agent/context_compressor.py). Skips are attempts that ended
  220. # before anything could fail (another path held the lock, nothing summarizable so no model call, the
  221. # user stopped it, a newer attempt replaced it). ``exception:<Type>`` / ``rollback:<Type>`` keep only
  222. # their prefix; any other value reads ``other``, a failure with no class ``unknown``.
  223. COMPRESSION_SKIP_CLASSES = frozenset({
  224. "attempt_superseded", "empty_post_handoff_window", "explicit_interrupt", "insufficient_messages",
  225. "lock_contended", "no_compressible_window", "snapshot_stale",
  226. })
  227. COMPRESSION_FAILURE_CLASSES = COMPRESSION_SKIP_CLASSES | frozenset({
  228. "aux_model_fallback", "commit_fence_cancelled", "empty_transcript", "exception", "feasibility_skip",
  229. "no_progress", "none",
  230. "other", "pool_saturated", "rollback", "session_split_failed", "stall_deterministic_fallback",
  231. "stall_interrupted", "summary_auth_failure", "summary_empty_content_failure", "summary_generation_aborted",
  232. "summary_generation_failed", "summary_network_failure", "summary_overload_degraded",
  233. "summary_overload_failure", "summary_truncated_failure", "unknown", "would_grow",
  234. })
  235. CONTEXT_FILL_BUCKETS = frozenset({"lt_50", "50_to_75", "75_to_90", "90_to_100", "gte_100", "unknown"})
  236. EXTENSION_KINDS = frozenset({"mcp_server", "plugin", "skill"})
  237. EXTENSION_SOURCES = frozenset({"bundled", "catalog", "hub", "local", "other", "url"})
  238. EXTENSION_OUTCOMES = frozenset({"failed", "success"})
  239. # Every built-in backend in tools/terminal_tool_config.py::_BUILTIN_BACKENDS.
  240. TERMINAL_BACKENDS = frozenset({
  241. "daytona", "docker", "local", "managed_modal", "modal", "other", "singularity", "ssh", "vercel_sandbox",
  242. })
  243. AUX_TASKS = _CatalogValues("aux_task_names", extra=frozenset({"none", "other"}))
  244. SLASH_COMMANDS = _CatalogValues("slash_command_names", extra=frozenset({"plugin", "skill", "unknown"}))
  245. SKILL_NAMES = _CatalogValues("bundled_skill_names", extra=frozenset({"custom"}))
  246. EXTENSION_NAMES = _CatalogValues(
  247. "bundled_skill_names", "mcp_catalog_names", "plugin_catalog_names", extra=frozenset({"custom"})
  248. )
  249. DISPLAY_LANGUAGES = _CatalogValues("display_languages", extra=frozenset({"other"}))
  250. # ---- v4 model ----
  251. TOOL_CALL_ISSUES = frozenset({
  252. "empty_arguments", "invalid_json", "none", "repaired", "schema_mismatch", "unknown_tool",
  253. })
  254. FRICTION_SIGNALS = frozenset({"interrupt", "quick_abandon", "retry", "switch_away", "undo"})
  255. CONTEXT_WINDOW_BUCKETS = frozenset({
  256. "lt_32k", "32k_to_128k", "128k_to_256k", "256k_to_1m", "gte_1m", "unknown",
  257. })
  258. LIMIT_HIT_VALUES = frozenset({"no", "yes"})
  259. # ---- end v4 model ----
  260. # ---- v4 loop ----
  261. MEMORY_OPS = frozenset({"add", "other", "read", "remove", "replace", "search"})
  262. MEMORY_OP_OUTCOMES = frozenset({"failed", "rejected", "success"})
  263. # Who asked: the user's turn, or the unattended self-improvement review fork.
  264. MEMORY_OP_ORIGINS = frozenset({"background_review", "foreground"})
  265. # Why a memory op was refused or failed. Built-in classes name each refusal/failure return of
  266. # tools/memory_tool.py and tools/memory_tool_store.py; plugin providers report ``provider_error`` (the
  267. # tool returned an error) or ``exception`` (it raised). Successes are ``none``.
  268. MEMORY_OP_FAILURE_CLASSES = frozenset({
  269. "ambiguous", "disabled", "drift", "exception", "gate_refused", "invalid_args", "missing_content",
  270. "missing_old_text", "no_match", "none", "other", "over_budget", "provider_error", "read_failed",
  271. "retry_cap", "scan_blocked", "staged", "stale_entry", "unknown", "would_empty",
  272. })
  273. CURATOR_OUTCOMES = frozenset({"failed", "skipped", "success"})
  274. CURATOR_TRIGGERS = frozenset({"manual", "scheduled"})
  275. DELEGATION_OUTCOMES = frozenset({"cancelled", "failed", "partial", "success"})
  276. DELEGATION_DEPTHS = frozenset({"1", "2", "3", "gte_4"})
  277. DELEGATION_MODES = frozenset({"background", "foreground"})
  278. EXECUTION_KINDS = frozenset({"browser", "code", "terminal"})
  279. # Browser backend labels: tools/browser_{lightpanda,session_cdp,session_cloud}.py plus the bundled cloud
  280. # providers in plugins/browser/. ``camofox`` (now the catalog plugin) and ``extension`` (the removed
  281. # browser-extension control) stay as legacy values: removing a shipped schema value breaks older relays.
  282. BROWSER_BACKENDS = frozenset({
  283. "browser-use", "browserbase", "camofox", "cdp", "extension", "firecrawl", "lightpanda", "local", "other",
  284. })
  285. CODE_BACKENDS = frozenset({"local", "other", "remote"})
  286. EXECUTION_BACKENDS_BY_KIND = {
  287. "browser": BROWSER_BACKENDS, "code": CODE_BACKENDS, "terminal": TERMINAL_BACKENDS,
  288. }
  289. EXECUTION_BACKENDS = BROWSER_BACKENDS | CODE_BACKENDS | TERMINAL_BACKENDS
  290. EXECUTION_OUTCOMES = frozenset({"failed", "success"})
  291. # ---- end v4 loop ----
  292. # ---- v4 gateway ----
  293. PLATFORM_HEALTH_MARK = PLATFORM_HEALTH_METRIC = "hermes.platform.health"
  294. PLATFORM_DELIVERY_MARK = PLATFORM_DELIVERY_METRIC = "hermes.platform.delivery"
  295. REPLY_LATENCY_MARK = REPLY_LATENCY_METRIC = "hermes.gateway.reply_latency"
  296. CRON_RUN_MARK = CRON_RUN_METRIC = "hermes.cron.run"
  297. # Plugin platforms are named only when Nous ships them (plugins/platforms/) or the installer proved a
  298. # plugin-catalog install (shared_metrics_catalog.platform_metric_name); every other one is ``plugin``.
  299. _PLATFORM_CATALOGS = ("bundled_platform_names", "catalog_platform_names")
  300. GATEWAY_PLATFORMS = _CatalogValues(*_PLATFORM_CATALOGS, extra=_CORE_GATEWAY_PLATFORMS)
  301. def _gateway_adapter_platforms() -> frozenset[str]:
  302. """The gateway's own adapter vocabulary: ``Platform``'s declared members (relay, msgraph_webhook
  303. and api_server are adapters but not setup-wizard platforms; plugin pseudo-members are not
  304. declared). Loaded on first use: gateway.config is too heavy for this module's import."""
  305. from gateway.config import Platform
  306. return frozenset(member.value for member in Platform)
  307. class _AdapterPlatforms(_CatalogValues):
  308. def values(self) -> frozenset[str]:
  309. return super().values() | _gateway_adapter_platforms()
  310. ADAPTER_PLATFORMS = _AdapterPlatforms(*_PLATFORM_CATALOGS, extra=_CORE_GATEWAY_PLATFORMS | {"api_server"})
  311. PLATFORM_HEALTH_EVENTS = frozenset({"connect_ok", "connect_failed", "reconnect", "disconnect"})
  312. PLATFORM_ERROR_CLASSES = frozenset({"none", "auth", "network", "rate_limited", "config", "other"})
  313. DELIVERY_OUTCOMES = frozenset({"sent", "failed"})
  314. DELIVERY_FAILURE_CLASSES = frozenset({"none", "rate_limited", "too_long", "auth", "network", "forbidden", "other"})
  315. REPLY_LATENCY_BUCKETS = frozenset({"lt_2s", "2s_to_5s", "5s_to_15s", "15s_to_60s", "gte_60s"})
  316. CRON_RUN_OUTCOMES = frozenset({"success", "failed", "missed", "skipped", "cancelled"})
  317. CRON_DELIVERY_KINDS = frozenset({"local", "platform", "webhook", "none", "other"})
  318. # ---- v4 install ----
  319. STARTUP_SURFACES = frozenset({"cli", "desktop_attach", "gateway_boot", "serve_boot", "tui"})
  320. STARTUP_LATENCY_BUCKETS = frozenset({
  321. "lt_500ms", "500ms_to_1s", "1s_to_2s", "2s_to_5s", "5s_to_10s", "gte_10s",
  322. })
  323. RELEASE_CHANNELS = frozenset({"dev", "main", "stable", "unknown"})
  324. VERSION_AGE_BUCKETS = frozenset({"lt_7d", "7d_to_30d", "30d_to_90d", "gte_90d", "unknown"})
  325. BEHIND_BUCKETS = COUNT_BUCKETS | {"unknown"}
  326. RAM_BUCKETS = frozenset({
  327. "lt_8g", "8g_to_16g", "16g_to_32g", "32g_to_64g", "64g_to_128g", "gte_128g", "unknown",
  328. })
  329. GPU_CLASSES = frozenset({"amd", "apple_silicon", "intel", "none", "nvidia", "unknown"})
  330. YES_NO = frozenset({"no", "yes"})
  331. _INSTALL_V4_SNAPSHOT_DIMENSIONS = {
  332. "behind_bucket": BEHIND_BUCKETS, "gpu_class": GPU_CLASSES, "local_model_provider_used": YES_NO,
  333. "ram_bucket": RAM_BUCKETS, "release_channel": RELEASE_CHANNELS, "version_age_bucket": VERSION_AGE_BUCKETS,
  334. }
  335. # ---- v5 engagement ----
  336. ENGAGEMENT_SURFACES = frozenset({"acp", "cli", "cron", "desktop", "gateway", "tui"})
  337. ACTIVE_MINUTES_BUCKETS = frozenset({"0", "lt_5m", "5m_to_30m", "30m_to_2h", "2h_to_6h", "gte_6h"})
  338. SURFACES_USED_BUCKETS = frozenset({"0", "1", "2", "3", "gte_4"})
  339. TURNS_BEFORE_SWITCH_BUCKETS = frozenset({"1", "2_to_3", "4_to_10", "11_to_30", "gte_31"})
  340. # SIZE_BUCKETS plus a longer tail: agentic conversations run to thousands of messages.
  341. LONG_SIZE_BUCKETS = SIZE_BUCKETS | frozenset({"251_to_1000", "gte_1001"})
  342. _LONG_SIZE_THRESHOLDS = (
  343. (1, "0"), (2, "1"), (3, "2"), (6, "3_to_5"), (11, "6_to_10"), (26, "11_to_25"),
  344. (101, "26_to_100"), (251, "101_to_250"), (1001, "251_to_1000"),
  345. )
  346. def long_size_bucket(count: int) -> str:
  347. return _bucket(max(0, int(count)), _LONG_SIZE_THRESHOLDS, "gte_1001")
  348. # ---- end v5 engagement ----
  349. # ---- v4 reliability ----
  350. UPDATE_RUN_MARK = UPDATE_RUN_METRIC = "hermes.update.run"
  351. UPDATE_STAGE_MARK = UPDATE_STAGE_METRIC = "hermes.update.stage"
  352. PROCESS_EXIT_MARK = PROCESS_EXIT_METRIC = "hermes.process.exit"
  353. UPDATE_KINDS = frozenset({"cli", "desktop"})
  354. UPDATE_OUTCOMES = frozenset({"failed", "noop", "refused", "success"})
  355. # ``partial``: committed, but interrupted after the commit point or the user's changes stay parked
  356. # (only `hermes update` receipts; the Desktop RPC keeps UPDATE_OUTCOMES).
  357. UPDATE_RUN_OUTCOMES = UPDATE_OUTCOMES | {"partial"}
  358. # `hermes update` pipeline stages, in pipeline order (the receipt's stage marks use these names).
  359. UPDATE_STAGE_ORDER = ("plan", "snapshot", "apply", "deps", "build", "restart", "verify")
  360. UPDATE_STAGES = frozenset(UPDATE_STAGE_ORDER)
  361. # Desktop's packaged updaters (electron-updater, App Installer, Store) fail at their own steps.
  362. DESKTOP_UPDATE_STAGES = frozenset({"apply", "download", "restart", "verify"})
  363. UPDATE_FAILED_STAGES = UPDATE_STAGES | DESKTOP_UPDATE_STAGES | {"none", "other"}
  364. UPDATE_STAGE_OUTCOMES = frozenset({"failed", "skipped", "success"})
  365. UPDATE_DURATION_BUCKETS = frozenset({"lt_30s", "30s_to_2m", "2m_to_5m", "5m_to_15m", "gte_15m"})
  366. # package = an OS/app-store style installer applied the update (Desktop packaged builds).
  367. UPDATE_APPLY_MODES = frozenset({"external", "git", "package", "unknown", "zip"})
  368. PROCESS_KINDS = frozenset({"cli", "cron", "gateway", "other", "serve", "tui"})
  369. PROCESS_EXIT_KINDS = frozenset({"clean", "crash", "killed", "watchdog"})
  370. CRASH_CLASSES = frozenset({"import_error", "memory_error", "none", "os_error", "other", "runtime_error"})
  371. _UPDATE_DURATION_THRESHOLDS = (
  372. (30_000, "lt_30s"), (120_000, "30s_to_2m"), (300_000, "2m_to_5m"), (900_000, "5m_to_15m"),
  373. )
  374. _DAY_MS = 86_400_000
  375. _VERSION_AGE_THRESHOLDS = ((7 * _DAY_MS, "lt_7d"), (30 * _DAY_MS, "7d_to_30d"), (90 * _DAY_MS, "30d_to_90d"))
  376. # ---- v5 signals ----
  377. TOOL_UNAVAILABLE_MARK = TOOL_UNAVAILABLE_METRIC = "hermes.tool_unavailable.count"
  378. PROVIDER_SETUP_MARK = PROVIDER_SETUP_METRIC = "hermes.provider_setup.count"
  379. FEATURE_ADOPTION_METRIC = "hermes.feature_adoption.count"
  380. # Not a counter: a first-use fact the subscriber turns into at most one adoption row per feature.
  381. FEATURE_USED_MARK = "hermes.feature.used"
  382. PROVIDER_SETUP_SURFACES = frozenset({"cli_model", "cli_setup", "dashboard", "desktop", "tui"})
  383. PROVIDER_SETUP_EVENTS = frozenset({"abandoned", "completed", "failed", "started"})
  384. PROVIDER_SETUP_FAILURE_CLASSES = frozenset({"auth", "cancelled", "network", "no_models", "none", "other"})
  385. FEATURES = frozenset({
  386. "browser", "bot_mode", "cron", "curator", "delegation", "desktop", "gateway_platform", "kanban", "mcp",
  387. "memory", "plugins", "projects", "skills_created", "tui", "voice",
  388. })
  389. DAYS_SINCE_INSTALL_BUCKETS = frozenset({"same_day", "1d_to_7d", "7d_to_30d", "30d_to_90d", "gte_90d", "unknown"})
  390. FEATURE_DISABLED_MARK = FEATURE_DISABLED_METRIC = "hermes.feature_disabled.count"
  391. FEATURE_DISABLED_KINDS = frozenset({
  392. "compression", "curator", "memory", "platform", "plugin", "setting", "skill", "toolset",
  393. })
  394. FEATURE_DISABLED_SURFACES = frozenset({"cli_config", "cli_slash", "cli_tools", "dashboard", "desktop", "tui"})
  395. FEATURE_DISABLED_EVENTS = frozenset({"disabled", "re_enabled"})
  396. FEATURE_DISABLED_NAME_MAX_LENGTH = 64
  397. # ---- end v5 signals ----
  398. # ---- iuf c1 ----
  399. # Why an extension install failed, named at each failure exit of the three emitters
  400. # (hermes_cli/skills_hub.py::_install_skill, hermes_cli/plugins_cmd_install.py::recorded_install,
  401. # hermes_cli/mcp_catalog.py::recorded_catalog_install / record_mcp_install); ``none`` on success.
  402. # A raised exception with no class of its own is classified by its TYPE: network / permission /
  403. # filesystem_error, anything else ``exception:<Type>``, which keeps only the ``exception`` prefix
  404. # (as on compression rows). Never the error message.
  405. EXTENSION_COMMON_FAILURE_CLASSES = frozenset({
  406. "exception", "filesystem_error", "network", "none", "other", "permission",
  407. })
  408. EXTENSION_SKILL_FAILURE_CLASSES = EXTENSION_COMMON_FAILURE_CLASSES | frozenset({
  409. "ambiguous", # a short name matched several skills in different registries
  410. "auth_rejected", # GitHub refused every configured credential (401)
  411. "fetch_failed", # no registry served the files (unreachable, 404, removed)
  412. "invalid_bundle", # unsafe path, symlink or invalid skill name in the bundle
  413. "invalid_name", # a URL skill with no usable name (non-interactive, bad --name, cancelled prompt)
  414. "not_found", # a short name no registry knows, or a pinned registry that does not exist
  415. "rate_limited", # the GitHub API rate limit was exhausted
  416. "scan_blocked", # the security scan refused the bundle
  417. "stale_index", # listed in a registry index whose files no longer exist upstream
  418. })
  419. EXTENSION_PLUGIN_FAILURE_CLASSES = EXTENSION_COMMON_FAILURE_CLASSES | frozenset({
  420. "already_installed", # the plugin already exists / is pinned and needs --force or --ref
  421. "clone_failed", # git clone / fetch / checkout of the plugin repository failed or timed out
  422. "deps_declined", # the user declined the Python dependency prompt
  423. "deps_failed", # PM could not prepare the plugin's dependencies
  424. "git_missing", # git is not installed
  425. "incompatible", # newer manifest_version, unsupported platform / GPU / runtime version
  426. "invalid_source", # the identifier or subdirectory does not name a usable git source
  427. "manifest_invalid", # unreadable or malformed plugin.yaml / plugin.json / dependency declaration
  428. "non_interactive", # Python dependencies need consent and nobody could answer (no --yes-deps)
  429. "removed_from_catalog", # on the catalog kill list
  430. "scan_blocked", # the security scan refused the plugin
  431. })
  432. EXTENSION_MCP_FAILURE_CLASSES = EXTENSION_COMMON_FAILURE_CLASSES | frozenset({
  433. "auth_required", # the server refused the sign-in (OAuth error, HTTP 401/403)
  434. "bootstrap_failed", # a catalog entry's bootstrap command exited non-zero
  435. "clone_failed", # git clone / checkout of a catalog entry's repository failed
  436. "config_invalid", # no such catalog entry, undeclared env var, or a manifest that cannot be built
  437. "config_rejected", # Hermes refused a suspicious command/args configuration
  438. "connect_failed", # the probe could not reach or initialize the server
  439. "git_missing", # git is not installed
  440. "missing_credentials", # a required credential was not provided
  441. "server_start_failed", # the stdio server command is missing, or its native module needs another Node
  442. })
  443. EXTENSION_KIND_FAILURE_CLASSES = {
  444. "mcp_server": EXTENSION_MCP_FAILURE_CLASSES, "plugin": EXTENSION_PLUGIN_FAILURE_CLASSES,
  445. "skill": EXTENSION_SKILL_FAILURE_CLASSES,
  446. }
  447. EXTENSION_FAILURE_CLASSES = (
  448. EXTENSION_SKILL_FAILURE_CLASSES | EXTENSION_PLUGIN_FAILURE_CLASSES | EXTENSION_MCP_FAILURE_CLASSES
  449. )
  450. # The skills-hub adapter that resolved or served a skill: every ``SOURCE_ID`` the router in
  451. # tools/skills_hub_search.py::create_source_router builds. ``none`` = no adapter was involved
  452. # (bundled restores, plugin and MCP rows), ``unresolved`` = a hub lookup no adapter answered,
  453. # ``other`` = an adapter id this list does not name.
  454. EXTENSION_REGISTRY_IDS = frozenset({
  455. "browse-sh", "clawhub", "github", "hermes-index", "lobehub", "official", "skills-sh", "url", "well-known",
  456. })
  457. EXTENSION_REGISTRIES = EXTENSION_REGISTRY_IDS | frozenset({"none", "other", "unresolved"})
  458. # Why a `hermes update` run failed or was refused, derived from the FINAL receipt only
  459. # (shared_metrics_update.update_failure_class); ``none`` unless outcome is failed/refused/partial.
  460. UPDATE_FAILURE_CLASSES = frozenset({
  461. "aborted_before_apply", # fallback: exited before the checkout moved with no reason recorded
  462. "build_failed", # the build stage reported failure
  463. "deps_failed", # PM dependency preparation failed (no deps mark, or a PM error type)
  464. "exception", # the run ended on an uncaught exception (type name dropped)
  465. "fleet_stale", # post-update verification found a gateway still stale or down
  466. "fleet_unverified", # verification could not prove the fleet current (no rows, unaccounted runtime)
  467. "git_failed", # a git/installer subprocess failed before the checkout moved (stash, pull, merge)
  468. "interrupted", # KeyboardInterrupt / exit 130
  469. "lock_held", # another updater held the update lock (refused, exit 2)
  470. "managed_install", # admission refused: image/package-managed or commit-build install (refused)
  471. "none", "other",
  472. "os_error", # the run ended on an OSError (disk full, permission, file in use)
  473. "restart_failed", # the gateway restart failed, or a skipped restart left the fleet owing one
  474. "subprocess_failed", # a subprocess error (CalledProcessError, TimeoutExpired) reached the boundary
  475. "unknown", # the reporter sent no reason (Desktop packaged updaters)
  476. "disk_full", # out of disk space (ENOSPC, or no room to stage the ZIP)
  477. "local_changes_parked", # committed, but the user's stashed changes could not be re-applied (partial)
  478. "permission_denied", # the run ended on a PermissionError
  479. })
  480. # The exit that stopped a run before its apply stage mark, recorded at the exit itself as one of
  481. # these tokens (update_receipt.record_stop_reason -> the receipt's ``stop_class``).
  482. UPDATE_STOP_CLASSES = frozenset({
  483. "branch_missing", # the target branch exists neither locally nor on origin
  484. "branch_unsupported", # --branch on the Windows ZIP fallback
  485. "channel_unresolved", # the update channel could not be resolved
  486. "checkout_move_failed", # git could not move the checkout (ff refused, reset or switch failed)
  487. "commit_point_refused", # the commit point could not be armed durably; nothing moved
  488. "detached_head", # a detached checkout stayed put, or its commits could not be backed up
  489. "disk_full", # (see above)
  490. "downgrade_refused", # the target predates the gateway runtime a state.db already uses
  491. "download_failed", # the ZIP fallback download failed
  492. "fetch_failed", # git fetch failed
  493. "gateway_pause_failed", # Windows: the gateways could not be paused safely before the venv moves
  494. "git_in_progress", # a merge/rebase/cherry-pick/... was already in progress
  495. "git_index_locked", # git refused: another git process's .git/index.lock exists
  496. "git_timeout", # a network git call hit the updater's time limit
  497. "head_moved", # a ref moved during the update; HEAD is not the selected commit
  498. "local_changes_blocked", # local changes git could not stash, or a dirty tree the ZIP will not overwrite
  499. "lock_held", # (see above)
  500. "managed_install", # (see above): HERMES_MANAGED installs refuse before any receipt
  501. "merge_conflict", # local commits on a custom branch conflict with upstream
  502. "not_git_checkout", # the install is not a git checkout (non-Windows)
  503. "old_version_handoff", # an older updater's hand-off to this version could not finish
  504. "parked_branch_blocked", # the checkout is parked on another branch that is unsafe to switch
  505. "permission_denied", # (see above): git could not write a file it needs (e.g. index.lock's directory)
  506. "stash_restore_rejected", # re-applied local changes broke Hermes; the update stopped
  507. "syntax_rollback", # the pulled code failed the syntax check and was rolled back
  508. "target_syntax_error", # the target failed the syntax check before anything moved
  509. "target_unresolved", # the fetched target ref did not resolve to a commit
  510. "unexpected_branch", # the checkout ended on a branch that is not the target
  511. "venv_foreign_owner", # the venv belongs to another OS user
  512. "zip_failed", # the ZIP fallback swap failed and was rolled back
  513. })
  514. UPDATE_FAILURE_CLASSES |= UPDATE_STOP_CLASSES
  515. # ---- end iuf c1 ----
  516. # ---- iuf c2 ----
  517. # One fresh-install run of scripts/install.sh / install.ps1, from the local receipt the installer
  518. # leaves (the installer itself never sends); counted by a later Hermes start while collection is on.
  519. INSTALL_RUN_MARK = INSTALL_RUN_METRIC = "hermes.install.run"
  520. INSTALL_RUN_INSTALLERS = frozenset({"install_ps1", "install_sh", "other"})
  521. INSTALL_RUN_OUTCOMES = frozenset({"failed", "success"})
  522. # The installers' stage_names / $Stages ladder (hyphens as underscores).
  523. INSTALL_RUN_STAGES = frozenset({
  524. "complete", "config", "gateway", "prerequisites", "products", "python_deps", "repository", "setup", "venv",
  525. })
  526. INSTALL_RUN_FAILED_STAGES = INSTALL_RUN_STAGES | {"none", "other"}
  527. # The code each installer fail()/Fail call site passes; `none` on success.
  528. INSTALL_RUN_FAILURE_CLASSES = frozenset({
  529. "commit_not_on_branch", "curl_missing", "deps_install_failed", "dir_not_checkout", "download_digest_mismatch",
  530. "download_failed", "filesystem_error", "gateway_failed", "git_checkout_failed", "git_clone_failed",
  531. "git_extract_failed", "git_fetch_failed", "git_missing", "git_reset_failed", "interrupted",
  532. "libstdcxx_missing", "local_changes_blocked", "none", "other", "products_build_failed",
  533. "python_install_failed", "setup_failed", "unsupported_platform", "uv_unusable",
  534. })
  535. # ---- end iuf c2 ----
  536. def update_duration_bucket(duration_ms: Any) -> str:
  537. """Bucket an update (or update stage) wall time; non-numbers count as instant."""
  538. value = _non_negative_number(duration_ms) or 0
  539. return _bucket(value, _UPDATE_DURATION_THRESHOLDS, "gte_15m")
  540. def version_age_bucket(age_ms: Any) -> str:
  541. """Bucket the age of the version an update started from; unknown when not measurable."""
  542. value = _non_negative_number(age_ms)
  543. return "unknown" if value is None else _bucket(value, _VERSION_AGE_THRESHOLDS, "gte_90d")
  544. # ---- end v4 reliability ----
  545. # ---- v5 harness ----
  546. FILE_EDIT_MARK = "hermes.file_edit"
  547. LOOP_GUARD_MARK = "hermes.loop_guard"
  548. TOOL_RECOVERY_MARK = "hermes.tool_recovery"
  549. TERMINAL_OUTCOME_MARK = "hermes.terminal.outcome"
  550. MODEL_REPLY_ISSUE_MARK = "hermes.model_reply_issue"
  551. FILE_EDIT_METRIC = "hermes.file_edit.count"
  552. LOOP_GUARD_METRIC = "hermes.loop_guard.count"
  553. TOOL_RECOVERY_METRIC = "hermes.tool_recovery.count"
  554. TERMINAL_OUTCOME_METRIC = "hermes.terminal.outcome.count"
  555. MODEL_REPLY_ISSUE_METRIC = "hermes.model_reply_issue.count"
  556. FILE_EDIT_TOOLS = frozenset({"patch", "write_file"})
  557. FILE_EDIT_MODES = frozenset({"replace", "v4a", "whole_file"})
  558. FILE_EDIT_OUTCOMES = frozenset({"already_applied", "ambiguous", "applied", "failed", "no_match"})
  559. def _fuzzy_match_strategies() -> frozenset[str]:
  560. # The patch tool's own strategy chain, so a new strategy is a new bucket, never "none".
  561. from tools.fuzzy_match import STRATEGIES
  562. return frozenset(name for name, _fn in STRATEGIES)
  563. FILE_EDIT_STRATEGIES = _fuzzy_match_strategies() | {"none"}
  564. LOOP_GUARD_SIGNALS = frozenset({"iteration_cap", "loop_detected", "repeated_tool_call"})
  565. LOOP_GUARD_DETECTORS = frozenset({
  566. "exact_failure", "idempotent_no_progress", "identical_call_streak", "identical_cycle", "iteration_budget",
  567. "same_tool_failure", "subagent_cap", "web_search_cap",
  568. })
  569. TOOL_RECOVERY_OUTCOMES = frozenset({"error", "gave_up", "no_tool_call", "success"})
  570. TOOL_RECOVERY_NEXT_TOOLS = frozenset({"different", "none", "same"})
  571. # ---- v4 connectors ----
  572. TOOL_SEARCH_MARK = TOOL_SEARCH_METRIC = "hermes.tool_search.count"
  573. TOOL_BATCH_MARK = TOOL_BATCH_METRIC = "hermes.tool_batch.count"
  574. CONNECTOR_CHAIN_MARK = CONNECTOR_CHAIN_METRIC = "hermes.connector_chain.count"
  575. TOOL_SEARCH_SCOPES = frozenset({"both", "connector", "local"})
  576. TOOL_SEARCH_OUTCOMES = frozenset({"connector_unavailable", "error", "hits", "no_hits"})
  577. CONNECTOR_SHARES = frozenset({"all", "none", "some"})
  578. # ---- end v4 connectors ----
  579. TERMINAL_COMMAND_KINDS = frozenset({
  580. "build", "container", "file_ops", "git", "network", "node", "other", "package_manager", "python",
  581. "shell", "shell_builtin", "test_runner",
  582. })
  583. TERMINAL_OUTCOMES = frozenset({"killed", "nonzero", "ok", "timeout"})
  584. MODEL_REPLY_ISSUES = frozenset({"empty", "none", "reasoning_only", "refusal", "truncated_length"})
  585. # ---- end v5 harness ----
  586. # ---- v5 efficiency ----
  587. TASK_COST_MARK = TASK_COST_METRIC = "hermes.task_cost.count"
  588. WASTED_TOKENS_MARK = WASTED_TOKENS_METRIC = "hermes.wasted_tokens.count"
  589. TOOL_OUTPUT_TRUNCATION_MARK = TOOL_OUTPUT_TRUNCATION_METRIC = "hermes.tool_output_truncation.count"
  590. TOOL_OVERHEAD_MARK = TOOL_OVERHEAD_METRIC = "hermes.tool_overhead.count"
  591. TOOL_ENABLED_UNUSED_MARK = TOOL_ENABLED_UNUSED_METRIC = "hermes.tool_enabled_unused.count"
  592. CACHE_BREAK_MARK = CACHE_BREAK_METRIC = "hermes.cache_break.count"
  593. # Prompt + completion tokens of one user turn, summed over its primary calls.
  594. TURN_TOKEN_BUCKETS = frozenset({
  595. "lt_2k", "2k_to_10k", "10k_to_50k", "50k_to_200k", "200k_to_1m", "gte_1m", "unknown",
  596. })
  597. TASK_COST_OUTCOMES = frozenset({"completed", "failed", "interrupted"})
  598. # Tool and API calls of one agentic turn: COUNT_BUCKETS stops at gte_11, far below real loops.
  599. TURN_ACTIVITY_BUCKETS = frozenset({
  600. "0", "1", "2", "3_to_5", "6_to_10", "11_to_25", "26_to_50", "51_to_100", "gte_101",
  601. })
  602. WASTE_REASONS = frozenset({"interrupt", "retry", "undo"})
  603. # Characters of one raw tool result, before Hermes spills or truncates it.
  604. TOOL_OUTPUT_SIZE_BUCKETS = frozenset({
  605. "lt_1k", "1k_to_10k", "10k_to_50k", "50k_to_100k", "100k_to_500k", "gte_500k", "unknown",
  606. })
  607. TOOL_SCHEMA_TOKEN_BUCKETS = frozenset({
  608. "0", "lt_2k", "2k_to_5k", "5k_to_10k", "10k_to_20k", "20k_to_40k", "gte_40k",
  609. })
  610. # Toolsets Hermes ships; MCP servers and plugin toolsets collapse to custom.
  611. TOOLSET_NAMES = BUILTIN_TOOLSET_NAMES | RETIRED_TOOLSET_NAMES | {"custom"}
  612. # compression is the expected cause; the rest are Hermes invalidating a warm prefix (bugs or
  613. # user-driven), or the provider reporting a cold read Hermes did not cause.
  614. CACHE_BREAK_CAUSES = frozenset({
  615. "cache_expired", "compression", "model_switch", "provider_reported_miss", "system_prompt_rebuild",
  616. "toolset_change",
  617. })
  618. # ---- end v5 efficiency ----
  619. # ---- v5 desktop ----
  620. # Desktop love/hate: which areas get used, what gets in the way, where first run stops. Every value
  621. # is a code-defined Desktop surface id (apps/desktop/src/store/desktop-metrics.ts mirrors these
  622. # sets); anything the client names outside them collapses to `other` before it is recorded.
  623. DESKTOP_FEATURE_USE_MARK = DESKTOP_FEATURE_USE_METRIC = "hermes.desktop.feature_use"
  624. DESKTOP_FRICTION_MARK = DESKTOP_FRICTION_METRIC = "hermes.desktop.friction"
  625. DESKTOP_ONBOARDING_MARK = DESKTOP_ONBOARDING_METRIC = "hermes.desktop.onboarding"
  626. # Settings views are `SETTINGS_VIEWS` in apps/desktop/src/app/settings/index.tsx (the legacy
  627. # `connections` tab redirects to `gateway`).
  628. DESKTOP_SETTINGS_AREAS = frozenset({
  629. "settings_about", "settings_billing", "settings_config_advanced", "settings_config_appearance",
  630. "settings_config_browser", "settings_config_chat", "settings_config_memory", "settings_config_model",
  631. "settings_config_safety", "settings_config_voice", "settings_config_workspace", "settings_gateway",
  632. "settings_keybinds", "settings_keys", "settings_notifications", "settings_other", "settings_providers",
  633. "settings_sessions", "settings_vault",
  634. })
  635. DESKTOP_FEATURE_AREAS = DESKTOP_SETTINGS_AREAS | {
  636. # panes and panels
  637. "browser_pane", "file_pane", "review_pane", "terminal_pane",
  638. # overlays and pickers
  639. "command_palette", "find_in_page", "model_picker", "session_picker", "session_switcher",
  640. # first-class features
  641. "bot_mode", "projects", "session_search", "skins", "voice_conversation", "voice_dictation",
  642. # full pages (APP_ROUTES in apps/desktop/src/app/routes.ts, plus contributed plugin pages)
  643. "agents", "artifacts", "capabilities", "command_center", "cron", "extension_page", "kanban", "messaging",
  644. "profiles", "session_import", "starmap", "webhooks",
  645. "other",
  646. }
  647. # notice_dismissed: stable toast ids and dismissible strips the Desktop code defines.
  648. DESKTOP_NOTICE_IDS = frozenset({
  649. "artifacts_partial_load", "background_queue_stuck", "backend_skew", "billing_banner", "billing_block",
  650. "build_discontinued", "client_behind", "composer_queue_stuck", "credits",
  651. "free_tier_notice", "gateway_error", "gui_skew", "install_method", "mcp_health", "model_warning",
  652. "onboarding_handoff", "restored_draft", "runtime_not_ready", "session_compress", "terminal_backend", "tip",
  653. "update_available", "voice_live_unavailable", "voice_stop_hint", "other",
  654. })
  655. # error_toast: the notifyError summary rule that matched (apps/desktop/src/store/notifications.ts).
  656. DESKTOP_ERROR_TOAST_CATEGORIES = frozenset({
  657. "api_key_missing", "api_key_rejected", "disk_full", "gateway_auth_failed", "method_not_allowed",
  658. "microphone_permission", "pool_slot_timeout", "restart_required", "rpc_out_of_sync", "storage_failure",
  659. "timeout", "unclassified", "other",
  660. })
  661. DESKTOP_SLOW_FRAME_BUCKETS = frozenset({"100ms_to_250ms", "250ms_to_1s", "1s_to_5s", "gte_5s"})
  662. DESKTOP_FRICTION_DETAILS: dict[str, frozenset[str]] = {
  663. "backend_disconnect": frozenset({"backend_exit", "network", "timeout", "other"}),
  664. "error_toast": DESKTOP_ERROR_TOAST_CATEGORIES,
  665. "notice_dismissed": DESKTOP_NOTICE_IDS,
  666. # Electron render-process-gone reasons: crashed -> crash; oom; killed; anything else -> other.
  667. "renderer_crash": frozenset({"crash", "killed", "oom", "other"}),
  668. "slow_frame": DESKTOP_SLOW_FRAME_BUCKETS,
  669. }
  670. DESKTOP_FRICTION_KINDS = frozenset(DESKTOP_FRICTION_DETAILS)
  671. DESKTOP_FRICTION_DETAIL_VALUES = frozenset().union(*DESKTOP_FRICTION_DETAILS.values())
  672. # The Desktop first-run flows: the classic provider overlay (store/onboarding.ts), the guided flow
  673. # (store/onboarding-gate.ts phases), free-tier sign-in, then the consent answer and the first message.
  674. DESKTOP_ONBOARDING_STEPS = frozenset({
  675. "choose_later", "consent", "first_message", "free_tier_ready", "guide", "guide_skip", "model_pick",
  676. "provider_api_key", "provider_local", "provider_oauth", "provider_setup", "sign_in",
  677. })
  678. DESKTOP_ONBOARDING_EVENTS = frozenset({"abandoned", "completed", "reached"})
  679. # Bot Mode (a bot's canonical chat or a bot side-chat in front) vs regular Sessions mode, per day.
  680. DESKTOP_MODE_USE_MARK = DESKTOP_MODE_USE_METRIC = "hermes.desktop.mode_use"
  681. DESKTOP_MODES = frozenset({"bots", "sessions"})
  682. DESKTOP_ACTIVE_MINUTES_BUCKETS = frozenset({"0", "lt_5m", "5m_to_30m", "30m_to_2h", "2h_to_6h", "gte_6h"})
  683. _DESKTOP_ACTIVE_THRESHOLDS = ((300_000, "lt_5m"), (1_800_000, "5m_to_30m"), (7_200_000, "30m_to_2h"),
  684. (21_600_000, "2h_to_6h"))
  685. def desktop_active_minutes_bucket(active_ms: Any) -> str:
  686. """Bucket a day's active time in one Desktop mode; nothing measurable reads 0."""
  687. value = _non_negative_number(active_ms) or 0
  688. return "0" if value <= 0 else _bucket(value, _DESKTOP_ACTIVE_THRESHOLDS, "gte_6h")
  689. # ---- end v5 desktop ----
  690. # ---- v5 desktop actions ----
  691. # Button presses per day. Action ids are the Desktop keybinding registry (KEYBIND_ACTIONS in
  692. # apps/desktop/src/lib/keybinds/actions.ts) plus the typed button table DESKTOP_BUTTON_ACTIONS in
  693. # apps/desktop/src/store/desktop-metrics.ts; contributed/plugin actions collapse to `other`.
  694. DESKTOP_ACTION_USE_MARK = DESKTOP_ACTION_USE_METRIC = "hermes.desktop.action_use"
  695. DESKTOP_KEYBIND_ACTION_IDS = frozenset({
  696. "appearance.toggleMode", "composer.cancel", "composer.dictate", "composer.focus", "composer.help",
  697. "composer.history", "composer.mention", "composer.modelPicker", "composer.newline", "composer.queue",
  698. "composer.reasoningDown", "composer.reasoningUp", "composer.send", "composer.sendQueued", "composer.slash",
  699. "composer.steer", "composer.voice", "conversation.scrollPageDown", "conversation.scrollPageUp",
  700. "hud.snapToPointer", "keybinds.openPanel", "nav.agents", "nav.artifacts", "nav.capabilities",
  701. "nav.commandCenter", "nav.commandPalette", "nav.cron", "nav.messaging", "nav.profiles", "nav.settings",
  702. "profile.create", "profile.default", "profile.next", "profile.prev", "profile.toggleAll",
  703. "session.archive", "session.focusSearch", "session.new", "session.newTab", "session.newWindow",
  704. "session.next", "session.prev", "session.togglePin", "view.closeTab", "view.closeTerminal",
  705. "view.cycleSidebarGrouping", "view.findInPage", "view.findNext", "view.findPrevious", "view.flipPanes",
  706. "view.newTerminal", "view.nextTerminal", "view.prevTerminal", "view.reopenTab", "view.selectionToComposer",
  707. "view.showBrowser", "view.showFiles", "view.showTerminal", "view.terminalCopy", "view.terminalPaste",
  708. "view.toggleHud", "view.toggleProfileRail", "view.toggleReview", "view.toggleRightSidebar",
  709. "view.toggleSidebar", "view.toggleSimpleMode", "view.toggleStatusbar", "view.toggleTabStrip",
  710. "workspace.newWorktree", "workspace.openFolder",
  711. })
  712. DESKTOP_BUTTON_ACTION_IDS = frozenset({"composer.attach", "message.copy", "message.retry"})
  713. DESKTOP_ACTION_IDS = DESKTOP_KEYBIND_ACTION_IDS | DESKTOP_BUTTON_ACTION_IDS | {"other"}
  714. DESKTOP_ACTION_VIAS = frozenset({"click", "menu", "palette", "shortcut"})
  715. # Dislike signals, each naming its target from a closed set (setting keys are DEFAULT_CONFIG leaf
  716. # paths, checked against the live schema by shared_metrics_desktop and shape-checked here).
  717. DESKTOP_DISLIKE_MARK = DESKTOP_DISLIKE_METRIC = "hermes.desktop.dislike"
  718. DESKTOP_FLOW_IDS = frozenset({
  719. "command_palette", "free_tier_sign_in", "keybind_capture", "model_picker", "project_create", "provider_oauth",
  720. "session_picker", "session_switcher", "other",
  721. })
  722. DESKTOP_FEATURE_TOGGLES = frozenset({
  723. "backdrop", "bot_activity_toasts", "composer_popout_gestures", "intro_splash", "native_notifications",
  724. "notification_kind", "reactions", "thread_timeline", "tips", "tours", "vibe_hearts", "other",
  725. })
  726. DESKTOP_UNDO_TARGETS = frozenset({"closed_tab", "restored_draft", "other"})
  727. DESKTOP_DISLIKE_TARGETS: dict[str, frozenset[str]] = {
  728. "cancelled": DESKTOP_FLOW_IDS,
  729. "feature_disabled": DESKTOP_FEATURE_TOGGLES,
  730. "quick_close": DESKTOP_FEATURE_AREAS,
  731. "rage_click": DESKTOP_ACTION_IDS,
  732. "setting_off_default": frozenset({"setting"}),
  733. "undo": DESKTOP_UNDO_TARGETS,
  734. }
  735. DESKTOP_DISLIKE_SIGNALS = frozenset(DESKTOP_DISLIKE_TARGETS)
  736. DESKTOP_DISLIKE_TARGET_VALUES = frozenset().union(*DESKTOP_DISLIKE_TARGETS.values())
  737. DESKTOP_DISLIKE_DIRECTIONS = frozenset({"away_from_default", "none", "to_default"})
  738. DESKTOP_SETTING_KEY_MAX_LENGTH = 96
  739. # ---- end v5 desktop actions ----
  740. _ARCHITECTURE_ALIASES = {
  741. "amd64": "x86_64", "x64": "x86_64", "x86_64": "x86_64",
  742. "aarch64": "arm64", "arm64": "arm64",
  743. "i386": "x86", "i486": "x86", "i586": "x86", "i686": "x86", "x86": "x86",
  744. }
  745. _OS_FAMILIES = {"darwin": "macos", "linux": "linux", "macos": "macos", "windows": "windows"}
  746. def _norm(value: Any) -> str:
  747. return str(value or "").strip().lower()
  748. def _allowlisted(normalized: str, allowed: frozenset[str]) -> str:
  749. return normalized if normalized in allowed else "unknown"
  750. def client_os_family(value: Any) -> str:
  751. """Map a platform system name to the shared-metrics OS taxonomy."""
  752. return _OS_FAMILIES.get(_norm(value), "unknown")
  753. def client_architecture(value: Any) -> str:
  754. """Map a machine architecture to the shared-metrics taxonomy."""
  755. normalized = _norm(value).replace("-", "_")
  756. if normalized in _ARCHITECTURE_ALIASES:
  757. return _ARCHITECTURE_ALIASES[normalized]
  758. return "arm" if normalized.startswith("armv") else "unknown"
  759. def client_install_method(value: Any) -> str:
  760. """Return an allowlisted Hermes installation method."""
  761. normalized = _norm(value)
  762. return _allowlisted("nixos" if normalized == "nix" else normalized, CLIENT_INSTALL_METHODS)
  763. def client_resource(
  764. hermes_version: Any, *, os_name: Any, architecture: Any, install_method: Any
  765. ) -> dict[str, str]:
  766. """Build the bounded client resource attached to aggregate packages."""
  767. version = str(hermes_version or "").strip()
  768. return {
  769. "architecture": client_architecture(architecture),
  770. "hermes_version": version if 0 < len(version) <= 64 else "unknown",
  771. "install_method": client_install_method(install_method),
  772. "os_family": client_os_family(os_name),
  773. }
  774. def client_resource_is_valid(resource: Any) -> bool:
  775. """Return whether a package resource exactly matches the bounded contract."""
  776. if not isinstance(resource, dict) or set(resource) != CLIENT_RESOURCE_KEYS:
  777. return False
  778. version = resource.get("hermes_version")
  779. return (
  780. isinstance(version, str)
  781. and 0 < len(version) <= 64
  782. and resource.get("os_family") in CLIENT_OS_FAMILIES
  783. and resource.get("architecture") in CLIENT_ARCHITECTURES
  784. and resource.get("install_method") in CLIENT_INSTALL_METHODS
  785. )
  786. _LEGACY_PROVIDER_FAMILIES = frozenset({"aggregator", "custom", "direct", "local", "unknown"})
  787. _LEGACY_MODEL_LOCALITIES = frozenset({"local", "remote", "unknown"})
  788. _LEGACY_MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"})
  789. _LEGACY_MODEL_FAMILIES = frozenset({
  790. "claude", "deepseek", "gemini", "gemma", "glm", "gpt", "grok", "kimi", "llama", "minimax",
  791. "mimo", "mistral", "nemotron", "nova", "o1", "o3", "o4", "qwen", "step", "trinity",
  792. "unknown",
  793. })
  794. _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {
  795. CLIENT_ACTIVE_METRIC: {},
  796. # Retained only so pre-v2 pending rows remain packageable.
  797. LEGACY_MODEL_CALL_METRIC: {
  798. "call_role": frozenset({"primary"}), "locality": _LEGACY_MODEL_LOCALITIES,
  799. "model_family": _LEGACY_MODEL_FAMILIES, "outcome": _LEGACY_MODEL_OUTCOMES,
  800. "provider_family": _LEGACY_PROVIDER_FAMILIES,
  801. },
  802. TASK_STARTED_METRIC: {
  803. "entrypoint": TASK_ENTRYPOINTS, "execution_surface": EXECUTION_SURFACES,
  804. "platform": GATEWAY_PLATFORMS,
  805. },
  806. TASK_FINISHED_METRIC: {
  807. "duration_bucket": DURATION_BUCKETS, "end_reason": TASK_END_REASONS,
  808. "entrypoint": TASK_ENTRYPOINTS, "execution_surface": EXECUTION_SURFACES,
  809. "failure_class": TASK_FAILURE_CLASSES,
  810. "model_call_count_bucket": COUNT_BUCKETS, "outcome": TASK_OUTCOMES,
  811. "platform": GATEWAY_PLATFORMS,
  812. "retry_count_bucket": COUNT_BUCKETS, "termination": TASK_TERMINATIONS,
  813. "tool_call_count_bucket": COUNT_BUCKETS,
  814. },
  815. MODEL_ROUTE_METRIC: {
  816. "call_role": MODEL_CALL_ROLES, "error_class": MODEL_ERROR_CLASSES, "outcome": MODEL_OUTCOMES,
  817. "ttft_bucket": TTFT_BUCKETS,
  818. },
  819. TOOL_CALL_METRIC: {
  820. "approval_outcome": TOOL_APPROVAL_OUTCOMES, "latency_bucket": TOOL_LATENCY_BUCKETS,
  821. "outcome": TOOL_OUTCOMES, "retry_count_bucket": TOOL_RETRY_BUCKETS,
  822. "tool_category": TOOL_CATEGORIES,
  823. },
  824. TOOL_LATENCY_METRIC: {
  825. "latency_bucket": TOOL_LATENCY_BUCKETS, "retry_count_bucket": TOOL_RETRY_BUCKETS,
  826. "tool_category": TOOL_CATEGORIES,
  827. },
  828. TASK_DURATION_METRIC: {
  829. "duration_bucket": DURATION_BUCKETS, "execution_surface": EXECUTION_SURFACES,
  830. "outcome": TASK_OUTCOMES, "retry_count_bucket": COUNT_BUCKETS,
  831. },
  832. TOOL_USAGE_METRIC: {
  833. "error_class": TOOL_ERROR_CLASSES, "outcome": TOOL_OUTCOMES, "tool_name": TOOL_NAMES,
  834. },
  835. TOOL_APPROVAL_METRIC: {
  836. "attribution": TOOL_APPROVAL_ATTRIBUTIONS,
  837. "outcome": TOOL_APPROVAL_OUTCOMES - {"not_required"},
  838. },
  839. SKILL_LIFECYCLE_METRIC: {"action": SKILL_LIFECYCLE_ACTIONS, "provenance": SKILL_PROVENANCES},
  840. SKILL_LOAD_METRIC: {
  841. "post_patch_state": SKILL_POST_PATCH_STATES, "provenance": SKILL_PROVENANCES,
  842. "reuse_state": SKILL_REUSE_STATES, "skill_name": SKILL_NAMES, "use_count_bucket": COUNT_BUCKETS,
  843. },
  844. INSTALL_SNAPSHOT_METRIC: {
  845. "cron_job_count_bucket": SIZE_BUCKETS, "display_language": DISPLAY_LANGUAGES,
  846. "install_age_bucket": INSTALL_AGE_BUCKETS, "mcp_server_count_bucket": SIZE_BUCKETS,
  847. "memory_provider": MEMORY_PROVIDERS, "messaging_platform_count_bucket": SIZE_BUCKETS,
  848. "plugin_count_bucket": SIZE_BUCKETS, "profile_count_bucket": SIZE_BUCKETS,
  849. "skill_count_bucket": SIZE_BUCKETS, "terminal_backend": TERMINAL_BACKENDS,
  850. **_INSTALL_V4_SNAPSHOT_DIMENSIONS, **PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,
  851. },
  852. SESSION_METRIC: {
  853. "active_duration_bucket": SESSION_DURATION_BUCKETS, "entrypoint": TASK_ENTRYPOINTS,
  854. "execution_surface": EXECUTION_SURFACES, "failed_turn_count_bucket": COUNT_BUCKETS,
  855. "last_outcome": TASK_OUTCOMES, "platform": GATEWAY_PLATFORMS, "turn_count_bucket": SIZE_BUCKETS,
  856. },
  857. MILESTONE_METRIC: {"install_age_bucket": INSTALL_AGE_BUCKETS, "milestone": MILESTONES},
  858. SETUP_COMPLETED_METRIC: {"surface": SETUP_SURFACES},
  859. MODEL_TOKENS_METRIC: {"aux_task": AUX_TASKS, "call_role": MODEL_CALL_ROLES, "token_type": TOKEN_TYPES},
  860. COMPRESSION_METRIC: {
  861. "context_fill_bucket": CONTEXT_FILL_BUCKETS, "failure_class": COMPRESSION_FAILURE_CLASSES,
  862. "outcome": COMPRESSION_OUTCOMES, "trigger": COMPRESSION_TRIGGERS,
  863. },
  864. MODEL_SWITCH_METRIC: {"execution_surface": EXECUTION_SURFACES},
  865. FALLBACK_METRIC: {"error_class": MODEL_ERROR_CLASSES},
  866. SLASH_COMMAND_METRIC: {"command": SLASH_COMMANDS, "execution_surface": EXECUTION_SURFACES},
  867. EXTENSION_INSTALL_METRIC: {
  868. "kind": EXTENSION_KINDS, "name": EXTENSION_NAMES, "outcome": EXTENSION_OUTCOMES,
  869. "source": EXTENSION_SOURCES,
  870. # ---- iuf c1 ----
  871. "failure_class": EXTENSION_FAILURE_CLASSES, "registry": EXTENSION_REGISTRIES,
  872. # ---- end iuf c1 ----
  873. },
  874. # ---- v4 loop ----
  875. MEMORY_OP_METRIC: {
  876. "failure_class": MEMORY_OP_FAILURE_CLASSES, "op": MEMORY_OPS, "origin": MEMORY_OP_ORIGINS,
  877. "outcome": MEMORY_OP_OUTCOMES, "provider": MEMORY_PROVIDERS,
  878. },
  879. CURATOR_RUN_METRIC: {
  880. "archived_bucket": COUNT_BUCKETS, "created_bucket": COUNT_BUCKETS, "merged_bucket": COUNT_BUCKETS,
  881. "outcome": CURATOR_OUTCOMES, "patched_bucket": COUNT_BUCKETS, "trigger": CURATOR_TRIGGERS,
  882. },
  883. DELEGATION_RUN_METRIC: {
  884. "depth": DELEGATION_DEPTHS, "mode": DELEGATION_MODES, "outcome": DELEGATION_OUTCOMES,
  885. "subagent_count_bucket": COUNT_BUCKETS,
  886. },
  887. EXECUTION_BACKEND_METRIC: {
  888. "backend": EXECUTION_BACKENDS, "error_class": TOOL_ERROR_CLASSES, "kind": EXECUTION_KINDS,
  889. "outcome": EXECUTION_OUTCOMES,
  890. },
  891. # ---- end v4 loop ----
  892. # ---- v4 gateway ----
  893. PLATFORM_HEALTH_METRIC: {
  894. "error_class": PLATFORM_ERROR_CLASSES, "event": PLATFORM_HEALTH_EVENTS, "platform": ADAPTER_PLATFORMS,
  895. },
  896. PLATFORM_DELIVERY_METRIC: {
  897. "failure_class": DELIVERY_FAILURE_CLASSES, "outcome": DELIVERY_OUTCOMES, "platform": ADAPTER_PLATFORMS,
  898. },
  899. REPLY_LATENCY_METRIC: {"first_response_bucket": REPLY_LATENCY_BUCKETS, "platform": ADAPTER_PLATFORMS},
  900. CRON_RUN_METRIC: {
  901. "delivery_kind": CRON_DELIVERY_KINDS, "duration_bucket": DURATION_BUCKETS, "outcome": CRON_RUN_OUTCOMES,
  902. },
  903. # ---- v4 model ----
  904. MODEL_TOOL_QUALITY_METRIC: {"call_role": MODEL_CALL_ROLES, "issue": TOOL_CALL_ISSUES},
  905. MODEL_FRICTION_METRIC: {"signal": FRICTION_SIGNALS},
  906. CONTEXT_PEAK_METRIC: {
  907. "limit_hit": LIMIT_HIT_VALUES, "peak_fill_bucket": CONTEXT_FILL_BUCKETS,
  908. "window_bucket": CONTEXT_WINDOW_BUCKETS,
  909. },
  910. # ---- end v4 model ----
  911. # ---- v4 install ----
  912. STARTUP_LATENCY_METRIC: {"latency_bucket": STARTUP_LATENCY_BUCKETS, "surface": STARTUP_SURFACES},
  913. # ---- v4 reliability ----
  914. UPDATE_RUN_METRIC: {
  915. "apply_mode": UPDATE_APPLY_MODES, "duration_bucket": UPDATE_DURATION_BUCKETS,
  916. "failed_stage": UPDATE_FAILED_STAGES, "from_version_age_bucket": VERSION_AGE_BUCKETS,
  917. "kind": UPDATE_KINDS, "outcome": UPDATE_RUN_OUTCOMES,
  918. # ---- iuf c1 ----
  919. "failure_class": UPDATE_FAILURE_CLASSES,
  920. # ---- end iuf c1 ----
  921. },
  922. UPDATE_STAGE_METRIC: {
  923. "duration_bucket": UPDATE_DURATION_BUCKETS, "outcome": UPDATE_STAGE_OUTCOMES, "stage": UPDATE_STAGES,
  924. },
  925. PROCESS_EXIT_METRIC: {
  926. "crash_class": CRASH_CLASSES, "exit_kind": PROCESS_EXIT_KINDS, "process_kind": PROCESS_KINDS,
  927. },
  928. # ---- end v4 reliability ----
  929. # ---- v5 harness ----
  930. FILE_EDIT_METRIC: {
  931. "match_strategy": FILE_EDIT_STRATEGIES, "mode": FILE_EDIT_MODES, "outcome": FILE_EDIT_OUTCOMES,
  932. "tool": FILE_EDIT_TOOLS,
  933. },
  934. LOOP_GUARD_METRIC: {"detector": LOOP_GUARD_DETECTORS, "signal": LOOP_GUARD_SIGNALS},
  935. TOOL_RECOVERY_METRIC: {
  936. "next_outcome": TOOL_RECOVERY_OUTCOMES, "next_tool": TOOL_RECOVERY_NEXT_TOOLS, "tool": TOOL_NAMES,
  937. },
  938. TERMINAL_OUTCOME_METRIC: {
  939. "backend": TERMINAL_BACKENDS, "command_kind": TERMINAL_COMMAND_KINDS, "outcome": TERMINAL_OUTCOMES,
  940. },
  941. MODEL_REPLY_ISSUE_METRIC: {"issue": MODEL_REPLY_ISSUES},
  942. # ---- end v5 harness ----
  943. # ---- v5 efficiency ----
  944. TASK_COST_METRIC: {
  945. "api_calls_bucket": TURN_ACTIVITY_BUCKETS, "outcome": TASK_COST_OUTCOMES,
  946. "tokens_bucket": TURN_TOKEN_BUCKETS, "tool_calls_bucket": TURN_ACTIVITY_BUCKETS,
  947. },
  948. WASTED_TOKENS_METRIC: {"reason": WASTE_REASONS, "tokens_bucket": TURN_TOKEN_BUCKETS},
  949. TOOL_OUTPUT_TRUNCATION_METRIC: {
  950. "original_size_bucket": TOOL_OUTPUT_SIZE_BUCKETS, "tool": TOOL_NAMES, "truncated": YES_NO,
  951. },
  952. TOOL_OVERHEAD_METRIC: {
  953. "enabled_tool_count_bucket": SIZE_BUCKETS, "execution_surface": EXECUTION_SURFACES,
  954. "tool_schema_tokens_bucket": TOOL_SCHEMA_TOKEN_BUCKETS,
  955. },
  956. TOOL_ENABLED_UNUSED_METRIC: {"toolset": TOOLSET_NAMES, "used": YES_NO},
  957. CACHE_BREAK_METRIC: {"cause": CACHE_BREAK_CAUSES},
  958. # ---- end v5 efficiency ----
  959. # ---- v5 engagement ----
  960. ENGAGEMENT_DAY_METRIC: {
  961. "active_minutes_bucket": ACTIVE_MINUTES_BUCKETS, "active_profile_count_bucket": SIZE_BUCKETS,
  962. "surfaces_used_count": SURFACES_USED_BUCKETS, BUSY_PEAK_FIELD: COUNT_BUCKETS,
  963. },
  964. ENGAGEMENT_SURFACE_METRIC: {"active_minutes_bucket": ACTIVE_MINUTES_BUCKETS, "surface": ENGAGEMENT_SURFACES},
  965. MODEL_SWITCH_AFTER_METRIC: {"turns_before_switch_bucket": TURNS_BEFORE_SWITCH_BUCKETS},
  966. # ---- end v5 engagement ----
  967. # ---- v5 desktop ----
  968. DESKTOP_FEATURE_USE_METRIC: {"area": DESKTOP_FEATURE_AREAS},
  969. DESKTOP_FRICTION_METRIC: {"detail": DESKTOP_FRICTION_DETAIL_VALUES, "kind": DESKTOP_FRICTION_KINDS},
  970. DESKTOP_ONBOARDING_METRIC: {"event": DESKTOP_ONBOARDING_EVENTS, "step": DESKTOP_ONBOARDING_STEPS},
  971. DESKTOP_MODE_USE_METRIC: {
  972. "active_minutes_bucket": DESKTOP_ACTIVE_MINUTES_BUCKETS, "bot_count_bucket": SIZE_BUCKETS,
  973. "messages_sent_bucket": SIZE_BUCKETS, "mode": DESKTOP_MODES,
  974. },
  975. DESKTOP_ACTION_USE_METRIC: {"action": DESKTOP_ACTION_IDS, "count_bucket": SIZE_BUCKETS, "via": DESKTOP_ACTION_VIAS},
  976. DESKTOP_DISLIKE_METRIC: {
  977. "direction": DESKTOP_DISLIKE_DIRECTIONS, "signal": DESKTOP_DISLIKE_SIGNALS, "target": DESKTOP_DISLIKE_TARGET_VALUES,
  978. },
  979. # ---- end v5 desktop ----
  980. # ---- v5 signals ----
  981. TOOL_UNAVAILABLE_METRIC: {"tool_name": BUILTIN_TOOL_NAMES | RETIRED_TOOL_NAMES},
  982. PROVIDER_SETUP_METRIC: {
  983. "event": PROVIDER_SETUP_EVENTS, "failure_class": PROVIDER_SETUP_FAILURE_CLASSES,
  984. "surface": PROVIDER_SETUP_SURFACES,
  985. },
  986. FEATURE_ADOPTION_METRIC: {"days_since_install_bucket": DAYS_SINCE_INSTALL_BUCKETS, "feature": FEATURES},
  987. FEATURE_DISABLED_METRIC: {
  988. "event": FEATURE_DISABLED_EVENTS, "kind": FEATURE_DISABLED_KINDS, "surface": FEATURE_DISABLED_SURFACES,
  989. },
  990. # ---- end v5 signals ----
  991. # ---- iuf c2 ----
  992. INSTALL_RUN_METRIC: {
  993. "duration_bucket": UPDATE_DURATION_BUCKETS, "failed_stage": INSTALL_RUN_FAILED_STAGES,
  994. "failure_class": INSTALL_RUN_FAILURE_CLASSES, "installer": INSTALL_RUN_INSTALLERS,
  995. "outcome": INSTALL_RUN_OUTCOMES,
  996. },
  997. # ---- end iuf c2 ----
  998. # ---- v4 connectors ----
  999. TOOL_SEARCH_METRIC: {"hits_bucket": COUNT_BUCKETS, "outcome": TOOL_SEARCH_OUTCOMES, "scope": TOOL_SEARCH_SCOPES},
  1000. TOOL_BATCH_METRIC: {"connector_share": CONNECTOR_SHARES, "size_bucket": COUNT_BUCKETS},
  1001. CONNECTOR_CHAIN_METRIC: {
  1002. "described_first": YES_NO, "outcome": TOOL_OUTCOMES, "searches_before_bucket": COUNT_BUCKETS,
  1003. },
  1004. # ---- end v4 connectors ----
  1005. PREPARED_COMPACTION_METRIC: {**PREPARED_COMPACTION_DIMENSIONS, "failure_class": COMPRESSION_FAILURE_CLASSES},
  1006. BUSY_ARRIVAL_METRIC: {"disposition": BUSY_ARRIVAL_DISPOSITIONS, "platform": ADAPTER_PLATFORMS, "wait_bucket": BUSY_WAIT_BUCKETS},
  1007. }
  1008. _MODEL_ROUTE_MAX_LENGTHS = {
  1009. "model": MODEL_IDENTIFIER_MAX_LENGTH, "provider": PROVIDER_IDENTIFIER_MAX_LENGTH,
  1010. }
  1011. # metric -> {field: max length} for provider/model identifiers, validated by shape (no catalog).
  1012. _IDENTIFIER_FIELDS: dict[str, dict[str, int]] = {
  1013. MODEL_ROUTE_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1014. MODEL_TOKENS_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1015. SETUP_COMPLETED_METRIC: {"provider": PROVIDER_IDENTIFIER_MAX_LENGTH},
  1016. MODEL_SWITCH_METRIC: dict.fromkeys(("from_provider", "to_provider"), PROVIDER_IDENTIFIER_MAX_LENGTH),
  1017. FALLBACK_METRIC: dict.fromkeys(("from_provider", "to_provider"), PROVIDER_IDENTIFIER_MAX_LENGTH),
  1018. INSTALL_SNAPSHOT_METRIC: {"main_provider": PROVIDER_IDENTIFIER_MAX_LENGTH},
  1019. # ---- v4 model ----
  1020. MODEL_TOOL_QUALITY_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1021. MODEL_FRICTION_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1022. CONTEXT_PEAK_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1023. # ---- end v4 model ----
  1024. # ---- v5 harness ----
  1025. LOOP_GUARD_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1026. TOOL_RECOVERY_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1027. MODEL_REPLY_ISSUE_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1028. # ---- end v5 harness ----
  1029. # ---- v5 efficiency ----
  1030. TASK_COST_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1031. WASTED_TOKENS_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1032. CACHE_BREAK_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1033. # ---- end v5 efficiency ----
  1034. # ---- v5 engagement ----
  1035. ENGAGEMENT_DAY_METRIC: {
  1036. "primary_model": MODEL_IDENTIFIER_MAX_LENGTH, "primary_provider": PROVIDER_IDENTIFIER_MAX_LENGTH,
  1037. },
  1038. MODEL_SWITCH_AFTER_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1039. # ---- end v5 engagement ----
  1040. # ---- v5 desktop ----
  1041. DESKTOP_DISLIKE_METRIC: {"setting": DESKTOP_SETTING_KEY_MAX_LENGTH},
  1042. # ---- end v5 desktop ----
  1043. # ---- v5 signals ----
  1044. TOOL_UNAVAILABLE_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1045. PROVIDER_SETUP_METRIC: {"provider": PROVIDER_IDENTIFIER_MAX_LENGTH},
  1046. FEATURE_DISABLED_METRIC: {"name": FEATURE_DISABLED_NAME_MAX_LENGTH},
  1047. # ---- end v5 signals ----
  1048. TOOL_BATCH_METRIC: _MODEL_ROUTE_MAX_LENGTHS,
  1049. }
  1050. # ---- v5 engagement ----
  1051. # Conversation volume on the session row: new fields, so rows recorded before them still package.
  1052. SESSION_VOLUME_DIMENSIONS = dict.fromkeys(
  1053. ("message_count_bucket", "model_call_count_bucket", "tool_call_count_bucket"), LONG_SIZE_BUCKETS,
  1054. )
  1055. _COUNTER_DIMENSION_VALUES[SESSION_METRIC] = {**_COUNTER_DIMENSION_VALUES[SESSION_METRIC], **SESSION_VOLUME_DIMENSIONS}
  1056. # ---- end v5 engagement ----
  1057. # metric -> closed dimension field set
  1058. _METRIC_FIELDS: dict[str, frozenset[str]] = {
  1059. name: frozenset(contract) | frozenset(_IDENTIFIER_FIELDS.get(name, ()))
  1060. for name, contract in _COUNTER_DIMENSION_VALUES.items()
  1061. }
  1062. # Fields v2 carried on the per-task and per-tool-call rows. Together they made nearly every task and
  1063. # tool call its own row; v3 moves duration/retries/latency to the small split counters above (call
  1064. # counts per task are already on hermes.task_cost.count). Their values stay in the contract so rows
  1065. # recorded before an upgrade still validate and package.
  1066. _TASK_FINISHED_V2_ONLY = frozenset(
  1067. {"duration_bucket", "model_call_count_bucket", "retry_count_bucket", "tool_call_count_bucket"}
  1068. )
  1069. _TOOL_CALL_V2_ONLY = frozenset({"latency_bucket", "retry_count_bucket"})
  1070. _METRIC_FIELDS[TASK_FINISHED_METRIC] -= _TASK_FINISHED_V2_ONLY
  1071. _METRIC_FIELDS[TOOL_CALL_METRIC] -= _TOOL_CALL_V2_ONLY
  1072. # Older field sets still accepted at packaging so counters recorded before an upgrade drain.
  1073. _LEGACY_METRIC_FIELDS: dict[str, tuple[frozenset[str], ...]] = {
  1074. MODEL_ROUTE_METRIC: (
  1075. frozenset(_MODEL_ROUTE_MAX_LENGTHS), _METRIC_FIELDS[MODEL_ROUTE_METRIC] - {"ttft_bucket"},
  1076. ),
  1077. TASK_STARTED_METRIC: (_METRIC_FIELDS[TASK_STARTED_METRIC] - {"platform"},),
  1078. TASK_FINISHED_METRIC: (frozenset(_COUNTER_DIMENSION_VALUES[TASK_FINISHED_METRIC]) - {"failure_class", "platform"},),
  1079. TOOL_CALL_METRIC: (frozenset(_COUNTER_DIMENSION_VALUES[TOOL_CALL_METRIC]),),
  1080. # v3 rows, recorded before the failure_class split, drain as they were counted.
  1081. COMPRESSION_METRIC: (_METRIC_FIELDS[COMPRESSION_METRIC] - {"failure_class"},),
  1082. MEMORY_OP_METRIC: (_METRIC_FIELDS[MEMORY_OP_METRIC] - {"failure_class"},),
  1083. SKILL_LOAD_METRIC: (_METRIC_FIELDS[SKILL_LOAD_METRIC] - {"skill_name"},),
  1084. INSTALL_SNAPSHOT_METRIC: (_METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - {
  1085. "display_language", "install_age_bucket", "main_provider", "messaging_platform_count_bucket",
  1086. "terminal_backend", *_INSTALL_V4_SNAPSHOT_DIMENSIONS, *PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,
  1087. },
  1088. # ---- v4 install ----
  1089. _METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - {*_INSTALL_V4_SNAPSHOT_DIMENSIONS, *PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS},
  1090. _METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - set(PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS),
  1091. ),
  1092. }
  1093. # ---- v5 engagement ----
  1094. _LEGACY_METRIC_FIELDS[SESSION_METRIC] = (_METRIC_FIELDS[SESSION_METRIC] - set(SESSION_VOLUME_DIMENSIONS),)
  1095. # ---- end v5 engagement ----
  1096. # ---- iuf c1 ----
  1097. # Rows recorded before the failure_class / registry split drain as they were counted.
  1098. _LEGACY_METRIC_FIELDS[EXTENSION_INSTALL_METRIC] = (
  1099. _METRIC_FIELDS[EXTENSION_INSTALL_METRIC] - {"failure_class", "registry"},
  1100. )
  1101. _LEGACY_METRIC_FIELDS[UPDATE_RUN_METRIC] = (_METRIC_FIELDS[UPDATE_RUN_METRIC] - {"failure_class"},)
  1102. # ---- end iuf c1 ----
  1103. _LEGACY_METRIC_FIELDS[ENGAGEMENT_DAY_METRIC] = (_METRIC_FIELDS[ENGAGEMENT_DAY_METRIC] - {BUSY_PEAK_FIELD},) # busy-wait
  1104. COUNTER_METRICS = frozenset(_METRIC_FIELDS) - {LEGACY_MODEL_CALL_METRIC}
  1105. # Counters whose value is a summed quantity rather than an event count.
  1106. SUM_METRICS = frozenset({MODEL_TOKENS_METRIC})
  1107. _SKILL_MARK_METRICS = {
  1108. SKILL_LIFECYCLE_MARK: SKILL_LIFECYCLE_METRIC, SKILL_LOAD_MARK: SKILL_LOAD_METRIC,
  1109. }
  1110. # Marks projected one-to-one onto a counter of the same contract.
  1111. _DECISION_MARK_METRICS = {
  1112. SESSION_MARK: SESSION_METRIC, SETUP_COMPLETED_MARK: SETUP_COMPLETED_METRIC,
  1113. COMPRESSION_MARK: COMPRESSION_METRIC, MODEL_SWITCH_MARK: MODEL_SWITCH_METRIC,
  1114. FALLBACK_MARK: FALLBACK_METRIC, SLASH_COMMAND_MARK: SLASH_COMMAND_METRIC,
  1115. EXTENSION_INSTALL_MARK: EXTENSION_INSTALL_METRIC,
  1116. # ---- v4 loop ----
  1117. MEMORY_OP_MARK: MEMORY_OP_METRIC, CURATOR_RUN_MARK: CURATOR_RUN_METRIC,
  1118. DELEGATION_RUN_MARK: DELEGATION_RUN_METRIC, EXECUTION_BACKEND_MARK: EXECUTION_BACKEND_METRIC,
  1119. # ---- end v4 loop ----
  1120. # ---- v4 gateway ----
  1121. PLATFORM_HEALTH_MARK: PLATFORM_HEALTH_METRIC, PLATFORM_DELIVERY_MARK: PLATFORM_DELIVERY_METRIC,
  1122. REPLY_LATENCY_MARK: REPLY_LATENCY_METRIC, CRON_RUN_MARK: CRON_RUN_METRIC,
  1123. # ---- v4 model ----
  1124. MODEL_TOOL_QUALITY_MARK: MODEL_TOOL_QUALITY_METRIC, MODEL_FRICTION_MARK: MODEL_FRICTION_METRIC,
  1125. CONTEXT_PEAK_MARK: CONTEXT_PEAK_METRIC,
  1126. # ---- end v4 model ----
  1127. # ---- v4 install ----
  1128. STARTUP_LATENCY_MARK: STARTUP_LATENCY_METRIC,
  1129. # ---- v4 reliability ----
  1130. UPDATE_RUN_MARK: UPDATE_RUN_METRIC, UPDATE_STAGE_MARK: UPDATE_STAGE_METRIC,
  1131. PROCESS_EXIT_MARK: PROCESS_EXIT_METRIC,
  1132. # ---- end v4 reliability ----
  1133. # ---- v5 harness ----
  1134. FILE_EDIT_MARK: FILE_EDIT_METRIC, LOOP_GUARD_MARK: LOOP_GUARD_METRIC,
  1135. TOOL_RECOVERY_MARK: TOOL_RECOVERY_METRIC, TERMINAL_OUTCOME_MARK: TERMINAL_OUTCOME_METRIC,
  1136. MODEL_REPLY_ISSUE_MARK: MODEL_REPLY_ISSUE_METRIC,
  1137. # ---- end v5 harness ----
  1138. # ---- v5 efficiency ----
  1139. TASK_COST_MARK: TASK_COST_METRIC, WASTED_TOKENS_MARK: WASTED_TOKENS_METRIC,
  1140. TOOL_OUTPUT_TRUNCATION_MARK: TOOL_OUTPUT_TRUNCATION_METRIC, TOOL_OVERHEAD_MARK: TOOL_OVERHEAD_METRIC,
  1141. TOOL_ENABLED_UNUSED_MARK: TOOL_ENABLED_UNUSED_METRIC, CACHE_BREAK_MARK: CACHE_BREAK_METRIC,
  1142. # ---- end v5 efficiency ----
  1143. # ---- v5 engagement ----
  1144. MODEL_SWITCH_AFTER_MARK: MODEL_SWITCH_AFTER_METRIC,
  1145. # ---- end v5 engagement ----
  1146. # ---- v5 desktop ----
  1147. DESKTOP_FEATURE_USE_MARK: DESKTOP_FEATURE_USE_METRIC, DESKTOP_FRICTION_MARK: DESKTOP_FRICTION_METRIC,
  1148. DESKTOP_ONBOARDING_MARK: DESKTOP_ONBOARDING_METRIC, DESKTOP_MODE_USE_MARK: DESKTOP_MODE_USE_METRIC,
  1149. DESKTOP_ACTION_USE_MARK: DESKTOP_ACTION_USE_METRIC, DESKTOP_DISLIKE_MARK: DESKTOP_DISLIKE_METRIC,
  1150. # ---- end v5 desktop ----
  1151. # ---- v5 signals ----
  1152. TOOL_UNAVAILABLE_MARK: TOOL_UNAVAILABLE_METRIC, PROVIDER_SETUP_MARK: PROVIDER_SETUP_METRIC,
  1153. FEATURE_DISABLED_MARK: FEATURE_DISABLED_METRIC,
  1154. # ---- end v5 signals ----
  1155. # ---- iuf c2 ----
  1156. INSTALL_RUN_MARK: INSTALL_RUN_METRIC,
  1157. # ---- end iuf c2 ----
  1158. TOOL_SEARCH_MARK: TOOL_SEARCH_METRIC, TOOL_BATCH_MARK: TOOL_BATCH_METRIC,
  1159. CONNECTOR_CHAIN_MARK: CONNECTOR_CHAIN_METRIC, PREPARED_COMPACTION_MARK: PREPARED_COMPACTION_METRIC,
  1160. BUSY_ARRIVAL_METRIC: BUSY_ARRIVAL_METRIC, # busy-wait: the gateway marks under the metric's own name
  1161. }
  1162. def counter_dimensions_are_valid(metric_name: str, dimensions: dict[str, Any]) -> bool:
  1163. """Return whether dimensions match one closed shared-metric contract (current or legacy)."""
  1164. fields = frozenset(dimensions)
  1165. if fields != _METRIC_FIELDS.get(metric_name) and fields not in _LEGACY_METRIC_FIELDS.get(metric_name, ()):
  1166. return False
  1167. identifiers = _IDENTIFIER_FIELDS.get(metric_name, {})
  1168. contract = _COUNTER_DIMENSION_VALUES[metric_name]
  1169. return all(
  1170. isinstance(value := dimensions[field], str) and (
  1171. value == _metric_identifier(value, max_length=identifiers[field]) if field in identifiers
  1172. else value in contract[field]
  1173. )
  1174. for field in fields
  1175. ) and _extension_install_fields_agree(metric_name, dimensions)
  1176. # ---- iuf c1 ----
  1177. def _extension_install_fields_agree(metric_name: str, dimensions: dict[str, Any]) -> bool:
  1178. """Cross-field rule the per-field enums cannot state: an install row's ``failure_class`` is in
  1179. its kind's set and is ``none`` exactly on success; only skill rows name a registry."""
  1180. if metric_name != EXTENSION_INSTALL_METRIC or "failure_class" not in dimensions:
  1181. return True # other metrics, and the pre-split shape without failure_class/registry
  1182. kind, failure_class = dimensions["kind"], dimensions["failure_class"]
  1183. return (
  1184. failure_class in EXTENSION_KIND_FAILURE_CLASSES[kind]
  1185. and (failure_class == "none") == (dimensions["outcome"] == "success")
  1186. and (kind == "skill" or dimensions["registry"] == "none")
  1187. )
  1188. # ---- end iuf c1 ----
  1189. def _relay_metadata(
  1190. event: Any, schema_key: str, schema_version: str, *extra_keys: str
  1191. ) -> dict | None:
  1192. """Return the event metadata when it carries only the allowlisted Relay keys."""
  1193. metadata = getattr(event, "metadata", None)
  1194. if not isinstance(metadata, dict) or metadata.get(schema_key) != schema_version:
  1195. return None
  1196. allowed = {schema_key, RUNTIME_INSTANCE_KEY, COMMIT_TICKET_KEY, "otel.status_code", *extra_keys}
  1197. if set(metadata) - allowed or metadata.get("otel.status_code", "OK") not in {"OK", "ERROR"}:
  1198. return None
  1199. return metadata
  1200. def _event_text(event: Any, attr: str) -> str:
  1201. return str(getattr(event, attr, "") or "")
  1202. def _event_shape_matches(event: Any, **expected: Any) -> bool:
  1203. """Match the coarse Relay event shape (``kind`` plus any of name/category/scope_category/
  1204. category_profile).
  1205. A ``str`` expectation compares against the stringified attribute; ``None`` requires the
  1206. attribute itself to be ``None``; anything else (the ``category_profile`` dict) compares
  1207. with plain equality. Unmentioned attributes are not checked.
  1208. """
  1209. for attr, value in expected.items():
  1210. actual = _event_text(event, attr) if isinstance(value, str) else getattr(event, attr, None)
  1211. if actual != value:
  1212. return False
  1213. return True
  1214. def _bounded_dimensions(metric_name: str, data: Any) -> dict[str, str] | None:
  1215. """Project ``data`` onto the metric's closed field set, or None when it does not fit."""
  1216. expected_fields = _METRIC_FIELDS[metric_name]
  1217. if not isinstance(data, dict) or set(data) != expected_fields:
  1218. return None
  1219. dimensions = {field: data.get(field) for field in sorted(expected_fields)}
  1220. valid = counter_dimensions_are_valid(metric_name, dimensions) and _catalog_holds(metric_name, dimensions)
  1221. return dimensions if valid else None
  1222. def _catalog_holds(metric_name: str, dimensions: dict[str, str]) -> bool:
  1223. """Re-run the producer's catalog pass on a mark's provider/model fields: an identifier it would
  1224. rewrite (a user-named provider, a local model id) never reaches a counter. Record-time only, so
  1225. catalog drift can never block packaging rows already stored. ``none`` is the unset-provider value."""
  1226. from .shared_metrics_catalog import model_metric_name, provider_metric_name
  1227. for field, max_length in _IDENTIFIER_FIELDS.get(metric_name, {}).items():
  1228. value = dimensions[field]
  1229. if field.endswith("provider"):
  1230. expected = value if value == "none" else provider_metric_name(value)
  1231. elif field.endswith("model"):
  1232. expected = model_metric_name(value, dimensions[field[:-5] + "provider"], max_length=max_length)
  1233. else:
  1234. continue
  1235. if value != expected:
  1236. return False
  1237. return True
  1238. def _valid_shape(event: Any, **shape: Any) -> bool:
  1239. """Metadata allowlist check plus :func:`_event_shape_matches` in one step."""
  1240. return (
  1241. _relay_metadata(event, SCHEMA_KEY, SCHEMA_VERSION) is not None
  1242. and _event_shape_matches(event, **shape)
  1243. )
  1244. def _bounded_counter(metric_name: str | None, event: Any) -> tuple[str, dict[str, str]] | None:
  1245. if metric_name is None:
  1246. return None
  1247. dimensions = _bounded_dimensions(metric_name, getattr(event, "data", None))
  1248. return None if dimensions is None else (metric_name, dimensions)
  1249. def _scoped_dimensions(event: Any, metric_name: str, **shape: Any) -> dict[str, str] | None:
  1250. """Bounded ``event.data`` for a scope *end* event of the given shape, else None."""
  1251. if not _valid_shape(event, kind="scope", scope_category="end", **shape):
  1252. return None
  1253. return _bounded_dimensions(metric_name, getattr(event, "data", None))
  1254. _MARK_SHAPE = dict(kind="mark", category=None, scope_category=None, category_profile=None)
  1255. def _mark_counter(event: Any, metrics_by_mark: dict[str, str]) -> tuple[str, dict[str, str]] | None:
  1256. """Return the bounded counter for a safe Relay mark whose name is in *metrics_by_mark*."""
  1257. if not _valid_shape(event, **_MARK_SHAPE):
  1258. return None
  1259. return _bounded_counter(metrics_by_mark.get(_event_text(event, "name")), event)
  1260. def client_active_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1261. """Return the active-install counter for one empty allowlisted mark."""
  1262. return _mark_counter(event, {CLIENT_ACTIVE_MARK: CLIENT_ACTIVE_METRIC})
  1263. def model_call_dimensions(event: Any) -> dict[str, str] | None:
  1264. """Return package dimensions for one valid logical model-call end event."""
  1265. auxiliary = _auxiliary_model_call_dimensions(event)
  1266. if auxiliary is not None:
  1267. return auxiliary
  1268. # The synthetic scope can span provider fallback. The accepted terminal
  1269. # route is carried in the validated payload rather than this start profile.
  1270. return _scoped_dimensions(
  1271. event, MODEL_ROUTE_METRIC, category="llm", name=MODEL_CALL_SCOPE,
  1272. category_profile={"model_name": MODEL_CALL_PROFILE_MODEL},
  1273. )
  1274. def _auxiliary_model_call_dimensions(event: Any) -> dict[str, str] | None:
  1275. """Project a terminal auxiliary route from its Hermes logical scope."""
  1276. metadata = _relay_metadata(
  1277. event, RUNTIME_SCHEMA_KEY, RUNTIME_SCHEMA_VERSION, "hermes.call_role"
  1278. )
  1279. call_role = (metadata or {}).get("hermes.call_role")
  1280. data = getattr(event, "data", None)
  1281. if (
  1282. not isinstance(call_role, str)
  1283. or not call_role.startswith("auxiliary:")
  1284. or not _event_shape_matches(
  1285. event, kind="scope", category="function", name=LOGICAL_LLM_SCOPE,
  1286. scope_category="end", category_profile=None,
  1287. )
  1288. or not isinstance(data, dict)
  1289. or set(data) - {"response_model", "error_class"} != {"model", "outcome", "provider"}
  1290. or data.get("outcome") not in _LEGACY_MODEL_OUTCOMES
  1291. ):
  1292. return None
  1293. outcome = data["outcome"]
  1294. # Same reading as a primary call: a cancelled call reports ``none``, a failure without a
  1295. # classified reason ``unknown``, a success the last error it recovered from.
  1296. error_class = "none" if outcome == "cancelled" else data.get("error_class") or "none"
  1297. if outcome == "failed" and error_class == "none":
  1298. error_class = "unknown"
  1299. dimensions = model_route_fields(data, call_role="auxiliary", outcome=outcome, error_class=error_class)
  1300. return dimensions if counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions) else None
  1301. def task_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1302. """Return one validated task counter from a task scope event."""
  1303. if not _valid_shape(
  1304. event, kind="scope", category="function", name=TASK_SCOPE, category_profile=None
  1305. ):
  1306. return None
  1307. phase = _event_text(event, "scope_category")
  1308. if phase == "start":
  1309. return _bounded_counter(TASK_STARTED_METRIC, event)
  1310. return _task_end_counter(event, TASK_FINISHED_METRIC) if phase == "end" else None
  1311. def task_duration_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1312. """Return the duration/retry counter carried by the same task end event."""
  1313. if not _valid_shape(
  1314. event, kind="scope", scope_category="end", category="function", name=TASK_SCOPE,
  1315. category_profile=None,
  1316. ):
  1317. return None
  1318. return _task_end_counter(event, TASK_DURATION_METRIC)
  1319. # The task end payload (task_terminal_fields) also carries per-task call counts for other Relay
  1320. # subscribers; hermes.task_cost.count already buckets them, so no counter projects them here.
  1321. _TASK_END_FIELDS = (
  1322. _METRIC_FIELDS[TASK_FINISHED_METRIC] | _METRIC_FIELDS[TASK_DURATION_METRIC]
  1323. | {"model_call_count_bucket", "tool_call_count_bucket"}
  1324. )
  1325. def _task_end_counter(event: Any, metric_name: str) -> tuple[str, dict[str, str]] | None:
  1326. """Project the task end event onto one counter; both projections must validate first."""
  1327. data = getattr(event, "data", None)
  1328. if not isinstance(data, dict) or set(data) != _TASK_END_FIELDS:
  1329. return None
  1330. projections = {
  1331. name: _bounded_dimensions(name, {f: data[f] for f in _METRIC_FIELDS[name]})
  1332. for name in (TASK_FINISHED_METRIC, TASK_DURATION_METRIC)
  1333. }
  1334. return (metric_name, projections[metric_name]) if all(projections.values()) else None
  1335. def tool_call_dimensions(event: Any) -> dict[str, str] | None:
  1336. """Return package dimensions for one allowlisted tool lifecycle end event."""
  1337. return _tool_end_projection(event, TOOL_CALL_METRIC)
  1338. def tool_usage_dimensions(event: Any) -> dict[str, str] | None:
  1339. """Return the per-tool usage dimensions carried by the same tool lifecycle end event."""
  1340. return _tool_end_projection(event, TOOL_USAGE_METRIC)
  1341. def tool_latency_dimensions(event: Any) -> dict[str, str] | None:
  1342. """Return the per-category latency/retry dimensions carried by the same tool end event."""
  1343. return _tool_end_projection(event, TOOL_LATENCY_METRIC)
  1344. _TOOL_END_METRICS = (TOOL_CALL_METRIC, TOOL_USAGE_METRIC, TOOL_LATENCY_METRIC)
  1345. _TOOL_END_FIELDS = frozenset().union(*(_METRIC_FIELDS[name] for name in _TOOL_END_METRICS))
  1346. def _tool_end_projection(event: Any, metric_name: str) -> dict[str, str] | None:
  1347. """Project the tool end event onto one counter; both projections must validate first."""
  1348. if not _valid_shape(
  1349. event, kind="scope", scope_category="end", category="tool", name=TOOL_CALL_SCOPE,
  1350. category_profile={},
  1351. ):
  1352. return None
  1353. data = getattr(event, "data", None)
  1354. if not isinstance(data, dict) or set(data) != _TOOL_END_FIELDS:
  1355. return None
  1356. projections = {
  1357. name: _bounded_dimensions(name, {f: data[f] for f in _METRIC_FIELDS[name]})
  1358. for name in _TOOL_END_METRICS
  1359. }
  1360. return projections[metric_name] if all(projections.values()) else None
  1361. def install_snapshot_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1362. """Return the daily install-configuration snapshot from a safe mark."""
  1363. return _mark_counter(event, {INSTALL_SNAPSHOT_MARK: INSTALL_SNAPSHOT_METRIC})
  1364. def tool_approval_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1365. """Return one validated approval counter from a safe Relay mark event."""
  1366. return _mark_counter(event, {TOOL_APPROVAL_MARK: TOOL_APPROVAL_METRIC})
  1367. def skill_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1368. """Return one validated skill lifecycle or load counter from a safe mark."""
  1369. return _mark_counter(event, _SKILL_MARK_METRICS)
  1370. def decision_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  1371. """Return the counter for one session/setup/compression/switch/fallback/command/install mark."""
  1372. return _mark_counter(event, _DECISION_MARK_METRICS)
  1373. _TOKEN_MARK_DIMENSIONS = frozenset({"aux_task", "call_role", "model", "provider"})
  1374. def model_token_counters(event: Any) -> list[tuple[str, dict[str, str], int]]:
  1375. """Expand one token-usage mark into ``(metric, dimensions, amount)`` sums, one per token type."""
  1376. if not _valid_shape(event, **_MARK_SHAPE) or _event_text(event, "name") != MODEL_TOKENS_MARK:
  1377. return []
  1378. data = getattr(event, "data", None)
  1379. if not isinstance(data, dict) or set(data) != _TOKEN_MARK_DIMENSIONS | TOKEN_TYPES:
  1380. return []
  1381. base = {field: data[field] for field in _TOKEN_MARK_DIMENSIONS}
  1382. counters = []
  1383. for token_type in sorted(TOKEN_TYPES):
  1384. amount = data[token_type]
  1385. if isinstance(amount, bool) or not isinstance(amount, int) or amount < 0:
  1386. return []
  1387. dimensions = {**base, "token_type": token_type}
  1388. if not counter_dimensions_are_valid(MODEL_TOKENS_METRIC, dimensions) or not _catalog_holds(
  1389. MODEL_TOKENS_METRIC, dimensions,
  1390. ):
  1391. return []
  1392. if amount:
  1393. counters.append((MODEL_TOKENS_METRIC, dimensions, amount))
  1394. return counters
  1395. def skill_lifecycle_fields(kwargs: dict[str, Any]) -> dict[str, str] | None:
  1396. """Build bounded fields for one successful non-load skill transition."""
  1397. action = _norm(kwargs.get("action"))
  1398. if action not in SKILL_LIFECYCLE_ACTIONS:
  1399. return None
  1400. return {"action": action, "provenance": skill_provenance(kwargs.get("provenance"))}
  1401. def skill_load_fields(kwargs: dict[str, Any]) -> dict[str, str] | None:
  1402. """Build bounded skill-use fields without exporting local skill identity."""
  1403. use_count, reused = kwargs.get("use_count"), kwargs.get("reused")
  1404. reuse_after_patch = kwargs.get("reuse_after_patch")
  1405. if (
  1406. isinstance(use_count, bool) or not isinstance(use_count, int) or use_count < 1
  1407. or not isinstance(reused, bool) or not isinstance(reuse_after_patch, bool)
  1408. or (reuse_after_patch and not reused)
  1409. ):
  1410. return None
  1411. return {
  1412. "post_patch_state": (
  1413. "not_applicable" if not reused
  1414. else "reused_after_patch" if reuse_after_patch
  1415. else "no_new_patch"
  1416. ),
  1417. "provenance": skill_provenance(kwargs.get("provenance")),
  1418. "reuse_state": "reused" if reused else "first_use",
  1419. "skill_name": _skill_metric_name(kwargs.get("skill_name")),
  1420. "use_count_bucket": count_bucket(use_count),
  1421. }
  1422. def _skill_metric_name(value: Any) -> str:
  1423. from .shared_metrics_catalog import skill_metric_name
  1424. return skill_metric_name(value)
  1425. def skill_provenance(value: Any) -> str:
  1426. """Normalize producer provenance to the closed shared-metrics taxonomy."""
  1427. return _allowlisted(_norm(value), SKILL_PROVENANCES)
  1428. _SURFACE_ALIASES = {
  1429. "api_server": "api",
  1430. # The relay connector's own platform: an inbound it could not stamp is still a gateway message.
  1431. "relay": "gateway",
  1432. **dict.fromkeys(("cron", "scheduler", "scheduled"), "scheduled_task"),
  1433. }
  1434. _KNOWN_GATEWAY_PLATFORMS = frozenset({"discord", "email", "slack", "telegram", "teams", "whatsapp"})
  1435. def execution_surface(kwargs: dict[str, Any]) -> str:
  1436. """Normalize the safe session surface carried by the parent Relay scope."""
  1437. value = _norm(kwargs.get("execution_surface") or kwargs.get("platform") or "unknown")
  1438. if value in EXECUTION_SURFACES:
  1439. return value
  1440. if value in _SURFACE_ALIASES:
  1441. return _SURFACE_ALIASES[value]
  1442. try:
  1443. from hermes_cli.platforms import get_all_platforms
  1444. if value in get_all_platforms():
  1445. return "gateway"
  1446. except Exception:
  1447. pass
  1448. return "gateway" if value in _KNOWN_GATEWAY_PLATFORMS else "other"
  1449. def task_start_fields(kwargs: dict[str, Any]) -> dict[str, str]:
  1450. """Build the bounded fields recorded on a task scope start event."""
  1451. surface = execution_surface(kwargs)
  1452. return {
  1453. "entrypoint": task_entrypoint(kwargs, surface), "execution_surface": surface,
  1454. "platform": gateway_platform(kwargs, surface),
  1455. }
  1456. def gateway_platform(kwargs: dict[str, Any], surface: str | None = None) -> str:
  1457. """The messaging platform for a gateway task: core, bundled or proven catalog plugin platforms
  1458. by name, every other plugin platform anonymous."""
  1459. if (surface or execution_surface(kwargs)) != "gateway":
  1460. return "none"
  1461. name = adapter_platform(kwargs.get("platform"))
  1462. return name if name in GATEWAY_PLATFORMS else "plugin"
  1463. def adapter_platform(value: Any) -> str:
  1464. """Public name of a gateway adapter's platform (Platform enum or its value)."""
  1465. from .shared_metrics_catalog import platform_metric_name
  1466. return platform_metric_name(value, _CORE_GATEWAY_PLATFORMS | _gateway_adapter_platforms())
  1467. _SURFACE_ENTRYPOINTS = {
  1468. # An ACP session is a human in an editor (VS Code / Zed / JetBrains), same
  1469. # dispatch shape as the other interactive surfaces.
  1470. **dict.fromkeys(("acp", "cli", "desktop", "tui"), "interactive"),
  1471. **{s: s for s in ("api", "batch", "python", "scheduled_task", "unknown")},
  1472. "gateway": "gateway_message",
  1473. }
  1474. def task_entrypoint(kwargs: dict[str, Any], surface: str | None = None) -> str:
  1475. """Normalize the task dispatch owner without exporting source strings."""
  1476. declared = _norm(kwargs.get("entrypoint"))
  1477. if declared in TASK_ENTRYPOINTS:
  1478. return declared
  1479. if kwargs.get("parent_task_id") or kwargs.get("parent_session_id"):
  1480. return "delegated"
  1481. return _SURFACE_ENTRYPOINTS.get(surface or execution_surface(kwargs), "other")
  1482. def task_terminal_fields(
  1483. kwargs: dict[str, Any], *, duration_ms: int, model_call_count: int, tool_call_count: int,
  1484. retry_count: int,
  1485. ) -> dict[str, str]:
  1486. """Build the bounded terminal payload for one task scope."""
  1487. outcome, end_reason, termination = task_terminal_state(kwargs)
  1488. return {
  1489. **task_start_fields(kwargs),
  1490. "duration_bucket": duration_bucket(duration_ms),
  1491. "end_reason": end_reason,
  1492. "failure_class": task_failure_class(kwargs, outcome),
  1493. "model_call_count_bucket": count_bucket(model_call_count),
  1494. "outcome": outcome,
  1495. "retry_count_bucket": count_bucket(retry_count),
  1496. "termination": termination,
  1497. "tool_call_count_bucket": count_bucket(tool_call_count),
  1498. }
  1499. _LOCAL_FAILURE_CLASSES = {"interpreter_shutdown": "shutdown", "session_busy": "session_busy"}
  1500. # turn_exit_reason prefix -> class, for failures that never reached the provider classifier.
  1501. _EXIT_REASON_FAILURE_CLASSES = (
  1502. ("empty_response", "empty_response"), ("all_retries_exhausted", "empty_response"),
  1503. ("context_compression", "context_compression"), ("compaction_", "context_compression"),
  1504. ("ollama_runtime_context", "context_compression"),
  1505. ("local_processing_error", "local_error"), ("repeated_outer_errors", "repeated_errors"),
  1506. ("error_near_max_iterations", "repeated_errors"), ("session_persistence", "persistence"),
  1507. ("redirect_restart_limit", "restart_limit"), ("rebuilt_restart_limit", "restart_limit"),
  1508. )
  1509. def task_failure_class(kwargs: dict[str, Any], outcome: str | None = None) -> str:
  1510. """Why a failed task failed, as a closed class; ``none`` for any non-failed outcome."""
  1511. if (outcome or task_terminal_state(kwargs)[0]) != "failed":
  1512. return "none"
  1513. declared = _norm(kwargs.get("failure_class"))
  1514. if declared in TASK_FAILURE_CLASSES - {"none"}:
  1515. return declared
  1516. failure_reason = _norm(kwargs.get("failure_reason"))
  1517. if failure_reason in MODEL_ERROR_CLASSES - {"none"}:
  1518. return failure_reason
  1519. if failure_reason in _LOCAL_FAILURE_CLASSES:
  1520. return _LOCAL_FAILURE_CLASSES[failure_reason]
  1521. reason = _norm(kwargs.get("turn_exit_reason"))
  1522. return next(
  1523. (label for prefix, label in _EXIT_REASON_FAILURE_CLASSES if reason.startswith(prefix)),
  1524. "other",
  1525. )
  1526. def task_terminal_state(kwargs: dict[str, Any]) -> tuple[str, str, str]:
  1527. """Map Hermes terminal state to bounded (outcome, end_reason, termination)."""
  1528. reason = _norm(kwargs.get("turn_exit_reason"))
  1529. if kwargs.get("interrupted") or "interrupt" in reason or "cancel" in reason:
  1530. return "cancelled", "user_cancelled", "user_cancelled"
  1531. if "timeout" in reason or "timed_out" in reason:
  1532. return "timed_out", "timed_out", "timed_out"
  1533. if "max_iterations" in reason or "budget_exhausted" in reason:
  1534. return "failed", "iteration_limit", "system_aborted"
  1535. if "approval" in reason and ("denied" in reason or "rejected" in reason):
  1536. return "failed", "approval_denied", "none"
  1537. if "guardrail" in reason:
  1538. return "failed", "guardrail_blocked", "system_aborted"
  1539. if reason == "system_aborted":
  1540. return "failed", "system_aborted", "system_aborted"
  1541. if kwargs.get("completed") is True:
  1542. return "success", "completed", "none"
  1543. if kwargs.get("failed") is True or (reason and reason != "unknown"):
  1544. return "failed", "failed", "none"
  1545. return "unknown", "unknown", "unknown"
  1546. # (exclusive upper bound, label) — ascending; the trailing label catches the rest.
  1547. _DURATION_THRESHOLDS = (
  1548. (1_000, "lt_1s"), (5_000, "1s_to_5s"), (30_000, "5s_to_30s"),
  1549. (120_000, "30s_to_2m"), (600_000, "2m_to_10m"),
  1550. )
  1551. _COUNT_THRESHOLDS = ((1, "0"), (2, "1"), (3, "2"), (6, "3_to_5"), (11, "6_to_10"))
  1552. _LATENCY_THRESHOLDS = (
  1553. (100, "lt_100ms"), (250, "100ms_to_250ms"), (500, "250ms_to_500ms"), (1_000, "500ms_to_1s"),
  1554. (2_000, "1s_to_2s"), (5_000, "2s_to_5s"), (10_000, "5s_to_10s"), (30_000, "10s_to_30s"),
  1555. )
  1556. def _bucket(value: float, thresholds: tuple[tuple[float, str], ...], last: str) -> str:
  1557. return next((label for upper, label in thresholds if value < upper), last)
  1558. def duration_bucket(duration_ms: int) -> str:
  1559. """Bucket a non-negative task duration into a fixed low-cardinality range."""
  1560. return _bucket(max(0, int(duration_ms)), _DURATION_THRESHOLDS, "gte_10m")
  1561. def count_bucket(count: int) -> str:
  1562. """Bucket a non-negative per-task count into a fixed range."""
  1563. return _bucket(max(0, int(count)), _COUNT_THRESHOLDS, "gte_11")
  1564. _TOOL_CATEGORY_EXACT = {
  1565. **{category: category for category in TOOL_CATEGORIES},
  1566. "clarify": "planning", "kanban": "planning", "todo": "planning", "session_search": "memory",
  1567. "cronjob": "scheduler", "skills": "skill", "x_search": "web",
  1568. }
  1569. _TOOL_CATEGORY_PREFIXES = (
  1570. ("mcp", "mcp"),
  1571. ("browser", "browser"),
  1572. (("image", "tts", "video", "vision"), "media"),
  1573. ("homeassistant", "home_automation"), # the homeassistant catalog plugin's toolset
  1574. (("discord", "email", "feishu", "hermes-yuanbao", "slack", "sms"), "communication"),
  1575. )
  1576. def tool_category(kwargs: dict[str, Any]) -> str:
  1577. """Map Hermes registry toolset metadata to a low-cardinality category."""
  1578. if _norm(kwargs.get("tool_name")).startswith(CONNECTOR_NAME_PREFIX):
  1579. return "connector"
  1580. toolset = _norm(kwargs.get("toolset"))
  1581. if not toolset:
  1582. return "unknown"
  1583. if toolset in _TOOL_CATEGORY_EXACT:
  1584. return _TOOL_CATEGORY_EXACT[toolset]
  1585. for prefixes, category in _TOOL_CATEGORY_PREFIXES:
  1586. if toolset.startswith(prefixes):
  1587. return category
  1588. return "other"
  1589. _TOOL_STATUS_OUTCOMES = {
  1590. **{s: s for s in ("blocked", "cancelled", "failed", "success", "timed_out")},
  1591. "error": "failed", "ok": "success", "timeout": "timed_out",
  1592. }
  1593. def tool_outcome(kwargs: dict[str, Any]) -> str:
  1594. """Normalize the terminal Hermes tool status without inspecting its result."""
  1595. return _TOOL_STATUS_OUTCOMES.get(_norm(kwargs.get("status")), "unknown")
  1596. _APPROVAL_CHOICES = {
  1597. **dict.fromkeys(
  1598. ("always", "approve", "approved", "once", "session", "smart_approve"), "approved"
  1599. ),
  1600. **dict.fromkeys(("deny", "denied", "smart_deny"), "denied"),
  1601. **dict.fromkeys(("timed_out", "timeout"), "timed_out"),
  1602. "cancelled": "cancelled", # prompt withdrawn / undeliverable / unanswered — not a user decision
  1603. }
  1604. def tool_approval_outcome(kwargs: dict[str, Any]) -> str:
  1605. """Normalize a terminal approval choice to a bounded outcome."""
  1606. return _APPROVAL_CHOICES.get(_norm(kwargs.get("choice")), "unknown")
  1607. def tool_terminal_fields(
  1608. kwargs: dict[str, Any], *, category: str | None = None, approval_outcome: str = "not_required",
  1609. fallback_duration_ms: int | None = None, tool_name: str | None = None,
  1610. ) -> dict[str, str]:
  1611. """Build one bounded tool-call terminal payload (feeds the category and per-tool counters)."""
  1612. outcome = tool_outcome(kwargs)
  1613. return {
  1614. "approval_outcome": _allowlisted(approval_outcome, TOOL_APPROVAL_OUTCOMES),
  1615. "error_class": tool_error_class(kwargs, outcome),
  1616. "latency_bucket": tool_latency_bucket(
  1617. kwargs.get("duration_ms"), fallback_duration_ms=fallback_duration_ms
  1618. ),
  1619. "outcome": outcome,
  1620. "retry_count_bucket": tool_retry_bucket(kwargs.get("retry_count")),
  1621. "tool_category": category if category in TOOL_CATEGORIES else tool_category(kwargs),
  1622. "tool_name": tool_name if tool_name is not None and tool_name in _LIVE_TOOL_NAMES
  1623. else tool_metric_name(kwargs),
  1624. }
  1625. def tool_metric_name(kwargs: dict[str, Any]) -> str:
  1626. """A built-in or bridge tool's own name; connector, MCP and plugin tools collapse to their kind."""
  1627. name = _norm(kwargs.get("tool_name"))
  1628. if name in BUILTIN_TOOL_NAMES or name in BRIDGE_TOOL_NAMES:
  1629. return name
  1630. if not name:
  1631. return "unknown"
  1632. if name.startswith(CONNECTOR_NAME_PREFIX):
  1633. return "connector"
  1634. return "mcp" if name.startswith("mcp_") or tool_category(kwargs) == "mcp" else "plugin"
  1635. _TOOL_STATUS_ERROR_CLASSES = {
  1636. "blocked": "blocked", "cancelled": "interrupted", "success": "none", "timed_out": "timeout",
  1637. }
  1638. _TOOL_ERROR_TYPES = {
  1639. **dict.fromkeys(("keyboard_interrupt", "tool_interrupted", "user_interrupt"), "interrupted"),
  1640. "invalid_tool_arguments": "invalid_arguments", "thread_missing_result": "exception",
  1641. "tool_error": "tool_error", "tool_result_contract": "contract_violation",
  1642. "tool_timeout": "timeout",
  1643. }
  1644. def tool_error_class(kwargs: dict[str, Any], outcome: str | None = None) -> str:
  1645. """Closed failure class from Hermes's own error_type; exception class names become
  1646. ``exception`` so plugin-defined identifiers never leave the machine. A connector entry's
  1647. gateway error code names its class instead of ``tool_error``."""
  1648. outcome = outcome or tool_outcome(kwargs)
  1649. if outcome in _TOOL_STATUS_ERROR_CLASSES:
  1650. return _TOOL_STATUS_ERROR_CLASSES[outcome]
  1651. if outcome != "failed":
  1652. return "unknown"
  1653. error_type = _norm(kwargs.get("error_type"))
  1654. if error_type == "tool_error" and (connector_class := connector_error_class(kwargs)):
  1655. return connector_class
  1656. return _TOOL_ERROR_TYPES.get(error_type, "exception" if error_type else "unknown")
  1657. # The tool gateway's execute codes (``tools.connectors.gateway.wire.EXECUTE_ERROR_CODES``) by class;
  1658. # a code not listed here keeps ``tool_error`` unless its family is backend.
  1659. _CONNECTOR_CODE_CLASSES = {
  1660. **dict.fromkeys(("CONNECTION_REQUIRED", "CONNECTION_EXPIRED"), "connection_required"),
  1661. "TOOL_NOT_ALLOWED": "not_allowed",
  1662. **dict.fromkeys(("ALIAS_REQUIRED", "ALIAS_UNKNOWN"), "alias"),
  1663. "INVALID_ARGUMENTS": "invalid_arguments",
  1664. # Agent-side file input refusals (``tools.connectors.gateway.wire.FILE_ERROR_FAMILIES``).
  1665. "FILE_NOT_ALLOWED": "not_allowed",
  1666. **dict.fromkeys(("FILE_NOT_FOUND", "FILE_TOO_LARGE"), "invalid_arguments"),
  1667. }
  1668. _CONNECTOR_FAMILY_CLASSES = {"backend": "connector_backend"}
  1669. def connector_error_class(kwargs: dict[str, Any]) -> str | None:
  1670. """The class of a failed connector entry from its result's ``error.code``; None when the call is
  1671. not a connector call or its result names no code."""
  1672. if not _norm(kwargs.get("tool_name")).startswith(CONNECTOR_NAME_PREFIX):
  1673. return None
  1674. error = _result_error(kwargs.get("result"))
  1675. code = str(error.get("code") or "").strip().upper()
  1676. if not code:
  1677. return None
  1678. # Late import: the wire module pulls pydantic, which the contract does not need at import.
  1679. from tools.connectors.gateway.wire import execute_error_family
  1680. family = execute_error_family(code, error.get("family") if isinstance(error.get("family"), str) else None)
  1681. return _CONNECTOR_FAMILY_CLASSES.get(family or "") or _CONNECTOR_CODE_CLASSES.get(code)
  1682. def _result_error(result: Any) -> dict[str, Any]:
  1683. if isinstance(result, str):
  1684. try:
  1685. import json
  1686. result = json.loads(result)
  1687. except ValueError:
  1688. return {}
  1689. error = result.get("error") if isinstance(result, dict) else None
  1690. return error if isinstance(error, dict) else {}
  1691. def tool_latency_bucket(value: Any, *, fallback_duration_ms: int | None = None) -> str:
  1692. """Bucket a tool duration reported in milliseconds."""
  1693. duration_ms = _non_negative_number(value)
  1694. if duration_ms is None:
  1695. duration_ms = _non_negative_number(fallback_duration_ms)
  1696. if duration_ms is None:
  1697. return "unknown"
  1698. return _bucket(duration_ms, _LATENCY_THRESHOLDS, "gte_30s")
  1699. def tool_retry_bucket(value: Any) -> str:
  1700. """Bucket only explicit tool retries; missing relationships stay unknown."""
  1701. if isinstance(value, bool) or not isinstance(value, int) or value < 0:
  1702. return "unknown"
  1703. return count_bucket(value)
  1704. def _non_negative_number(value: Any) -> float | None:
  1705. if isinstance(value, bool) or not isinstance(value, (int, float)):
  1706. return None
  1707. try:
  1708. number = float(value)
  1709. except (OverflowError, TypeError, ValueError):
  1710. return None
  1711. return number if isfinite(number) and number >= 0 else None
  1712. def model_call_fields(kwargs: dict[str, Any]) -> dict[str, str]:
  1713. """Return the terminal model identity and provider route known to Hermes."""
  1714. from .shared_metrics_catalog import model_metric_name, provider_metric_name
  1715. provider = provider_metric_name(kwargs.get("provider"))
  1716. model = model_metric_name(kwargs.get("response_model"), provider, max_length=MODEL_IDENTIFIER_MAX_LENGTH)
  1717. if model == "unknown":
  1718. model = model_metric_name(kwargs.get("model"), provider, max_length=MODEL_IDENTIFIER_MAX_LENGTH)
  1719. return {"model": model, "provider": provider}
  1720. def model_route_fields(
  1721. kwargs: dict[str, Any], *, call_role: str, outcome: str, error_class: str,
  1722. ttft_bucket: str = "unknown",
  1723. ) -> dict[str, str]:
  1724. """The terminal route plus how the logical call ended. ``error_class`` is the last
  1725. classified attempt error, so ``success`` + ``rate_limit`` reads as "recovered from a 429"."""
  1726. return {
  1727. **model_call_fields(kwargs),
  1728. "call_role": _allowlisted(call_role, MODEL_CALL_ROLES),
  1729. "error_class": error_class if error_class in MODEL_ERROR_CLASSES else "unknown",
  1730. "outcome": outcome if outcome in MODEL_OUTCOMES else "failed",
  1731. "ttft_bucket": ttft_bucket if ttft_bucket in TTFT_BUCKETS else "unknown",
  1732. }
  1733. def model_error_class(kwargs: dict[str, Any]) -> str:
  1734. """The classifier's FailoverReason for one failed provider attempt."""
  1735. reason = _norm(kwargs.get("reason"))
  1736. return reason if reason in MODEL_ERROR_CLASSES - {"none"} else "unknown"
  1737. # (exclusive upper bound, label) for install-scale counts (skills routinely exceed 100).
  1738. _SIZE_THRESHOLDS = (
  1739. (1, "0"), (2, "1"), (3, "2"), (6, "3_to_5"), (11, "6_to_10"), (26, "11_to_25"),
  1740. (101, "26_to_100"), (251, "101_to_250"),
  1741. )
  1742. def size_bucket(count: int) -> str:
  1743. """Bucket a non-negative install-scale count."""
  1744. return _bucket(max(0, int(count)), _SIZE_THRESHOLDS, "gte_251")
  1745. def install_snapshot_fields(
  1746. *, memory_provider: Any, mcp_servers: int, plugins: int, skills: int, cron_jobs: int,
  1747. profiles: int, messaging_platforms: int, install_age_bucket: str, main_provider: Any,
  1748. terminal_backend: Any, display_language: Any,
  1749. ) -> dict[str, str]:
  1750. """Bounded daily configuration snapshot: counts, closed enums and public names only."""
  1751. from .shared_metrics_catalog import display_language_metric_name, provider_metric_name
  1752. provider = _norm(memory_provider)
  1753. backend = _norm(terminal_backend) or "local"
  1754. return {
  1755. "cron_job_count_bucket": size_bucket(cron_jobs),
  1756. "display_language": display_language_metric_name(display_language),
  1757. "install_age_bucket": install_age_bucket if install_age_bucket in INSTALL_AGE_BUCKETS else "unknown",
  1758. "main_provider": provider_metric_name(main_provider) if main_provider else "none",
  1759. "mcp_server_count_bucket": size_bucket(mcp_servers),
  1760. "memory_provider": "builtin" if not provider
  1761. else provider if provider in MEMORY_PROVIDERS else "plugin",
  1762. "messaging_platform_count_bucket": size_bucket(messaging_platforms),
  1763. "plugin_count_bucket": size_bucket(plugins),
  1764. "profile_count_bucket": size_bucket(profiles),
  1765. "skill_count_bucket": size_bucket(skills),
  1766. "terminal_backend": backend if backend in TERMINAL_BACKENDS else "other",
  1767. }
  1768. def _metric_identifier(value: Any, *, max_length: int) -> str:
  1769. """Normalize one structurally safe identifier without a product catalog."""
  1770. if not isinstance(value, str):
  1771. return "unknown"
  1772. identifier = value.strip().lower()
  1773. if (
  1774. not identifier
  1775. or len(identifier) > max_length
  1776. or identifier[0] not in _METRIC_IDENTIFIER_START_CHARACTERS
  1777. or not _METRIC_IDENTIFIER_CHARACTERS.issuperset(identifier)
  1778. ):
  1779. return "unknown"
  1780. return identifier