shared_metrics_subscriber.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190
  1. """Relay subscriber for the persisted Hermes shared-metrics slice."""
  2. from __future__ import annotations
  3. import logging
  4. import platform
  5. import threading
  6. from typing import Any
  7. from agent.relay_runtime import RUNTIME_INSTANCE_KEY
  8. from hermes_constants import get_hermes_home
  9. from . import shared_metrics_engagement as engagement
  10. from . import shared_metrics_signals as signals
  11. from .shared_metrics import SharedMetricsStore
  12. from .shared_metrics_fields import milestones_for
  13. from .shared_metrics_install import detect_client_install_method
  14. from .shared_metrics_contract import (
  15. CLIENT_ACTIVE_METRIC,
  16. COMMIT_TICKET_KEY,
  17. FEATURE_DISABLED_METRIC,
  18. FEATURE_USED_MARK,
  19. INSTALL_SNAPSHOT_METRIC,
  20. MODEL_ROUTE_METRIC,
  21. TOOL_CALL_METRIC,
  22. TOOL_LATENCY_METRIC,
  23. TOOL_USAGE_METRIC,
  24. client_active_counter,
  25. client_resource,
  26. decision_counter,
  27. install_snapshot_counter,
  28. model_call_dimensions,
  29. model_token_counters,
  30. skill_counter,
  31. task_counter,
  32. task_duration_counter,
  33. tool_approval_counter,
  34. tool_call_dimensions,
  35. tool_latency_dimensions,
  36. tool_usage_dimensions,
  37. )
  38. logger = logging.getLogger(__name__)
  39. # Contract projections; each yields (metric_name, dimensions) or None. One tool end event feeds
  40. # the category, per-tool and latency counters (one task end event the task and duration counters),
  41. # so every match is recorded.
  42. _COUNTERS = (
  43. client_active_counter,
  44. install_snapshot_counter,
  45. lambda event: _named(MODEL_ROUTE_METRIC, model_call_dimensions(event)),
  46. lambda event: _named(TOOL_CALL_METRIC, tool_call_dimensions(event)),
  47. lambda event: _named(TOOL_USAGE_METRIC, tool_usage_dimensions(event)),
  48. lambda event: _named(TOOL_LATENCY_METRIC, tool_latency_dimensions(event)),
  49. task_counter,
  50. task_duration_counter,
  51. tool_approval_counter,
  52. skill_counter,
  53. decision_counter,
  54. signals.feature_used_counter,
  55. )
  56. def _named(metric_name: str, dimensions: dict | None) -> tuple[str, dict] | None:
  57. return None if dimensions is None else (metric_name, dimensions)
  58. class SharedMetricsSubscriber:
  59. """Persist validated Hermes counters from Relay lifecycle events."""
  60. def __init__(
  61. self,
  62. store: SharedMetricsStore,
  63. hermes_version: str,
  64. *,
  65. runtime_id: str | None = None,
  66. ) -> None:
  67. self.store = store
  68. self._client_resource = client_resource(
  69. hermes_version,
  70. os_name=platform.system(),
  71. architecture=platform.machine(),
  72. install_method=detect_client_install_method(),
  73. )
  74. self._runtime_id = runtime_id
  75. self._active = True
  76. self._lock = threading.RLock()
  77. self._milestones_done: set[str] = set(store.recorded_milestones())
  78. self._features_done: set[str] = set(store.recorded_features())
  79. self._saved_tickets: dict[str, int] = {}
  80. # Events arrive on the Relay thread, which carries no profile binding.
  81. self._hermes_home = get_hermes_home()
  82. def deactivate(self) -> None:
  83. """Stop accepting events before telemetry is disabled or torn down."""
  84. with self._lock:
  85. self._active = False
  86. @staticmethod
  87. def _classify(event: Any) -> list[tuple[str, dict, int]]:
  88. """Return every ``(metric_name, dimensions, amount)`` the event satisfies."""
  89. counted = [(*m, 1) for m in (project(event) for project in _COUNTERS) if m is not None]
  90. return counted + model_token_counters(event)
  91. def _record_milestones(self, metric_name: str, dimensions: dict) -> None:
  92. """Latch every install milestone this counter reaches (each once per install, ever)."""
  93. reached = [m for m in milestones_for(metric_name, dimensions) if m not in self._milestones_done]
  94. if not reached:
  95. return
  96. from .shared_metrics_snapshot import install_age_bucket
  97. age = install_age_bucket(self._hermes_home)
  98. for milestone in reached:
  99. self.store.record_milestone(milestone, age, self._client_resource)
  100. self._milestones_done.add(milestone)
  101. def _record_engagement(self, event: Any, classified: list[tuple[str, dict, int]]) -> None:
  102. """Fold a turn start/end or an attended turn's model into the local daily engagement rollup."""
  103. surface, route = engagement.interaction_surface(classified), engagement.turn_route(event)
  104. if surface is None and route is None:
  105. return
  106. with self._lock:
  107. if not self._active:
  108. return
  109. try:
  110. engagement.record(self.store, self._client_resource, surface=surface, route=route)
  111. except Exception:
  112. logger.warning("Unable to update the Hermes engagement rollup", exc_info=True)
  113. def _persist(self, metric_name: str, dimensions: dict, amount: int) -> None:
  114. store, resource = self.store, self._client_resource
  115. special = {
  116. FEATURE_USED_MARK: lambda: None, # a first-use fact, not a counter row
  117. FEATURE_DISABLED_METRIC: lambda: store.record_counter_once_per_day(metric_name, dimensions, resource),
  118. CLIENT_ACTIVE_METRIC: lambda: store.record_client_active(resource),
  119. INSTALL_SNAPSHOT_METRIC: lambda: store.record_install_snapshot(dimensions, resource),
  120. }.get(metric_name)
  121. if special is not None:
  122. special()
  123. else:
  124. store.record_counter(metric_name, dimensions, resource, amount)
  125. def _record_features(self, metric_name: str, dimensions: dict) -> None:
  126. """Latch each feature's first real use (once per install, with the owning profile's age)."""
  127. reached = [f for f in signals.features_for(metric_name, dimensions) if f not in self._features_done]
  128. if not reached:
  129. return
  130. age = signals.days_since_install_bucket(self._hermes_home)
  131. for feature in reached:
  132. try:
  133. self.store.record_feature_adoption(feature, age, self._client_resource)
  134. except Exception: # the counter row is saved; a later use retries the latch
  135. logger.debug("Feature adoption not latched: %s", feature, exc_info=True)
  136. continue
  137. self._features_done.add(feature)
  138. def take_saved(self, ticket: str) -> int:
  139. """How many events carrying ``ticket`` settled without a store error (and forget the ticket)."""
  140. with self._lock:
  141. return self._saved_tickets.pop(ticket, 0)
  142. def __call__(self, event: Any) -> None:
  143. metadata = getattr(event, "metadata", None)
  144. if self._runtime_id is not None:
  145. if (
  146. not isinstance(metadata, dict)
  147. or metadata.get(RUNTIME_INSTANCE_KEY) != self._runtime_id
  148. ):
  149. return
  150. ticket = metadata.get(COMMIT_TICKET_KEY) if isinstance(metadata, dict) else None
  151. saved = True # a row the contract rejects is settled too: no retry can change that
  152. classified = self._classify(event)
  153. self._record_engagement(event, classified)
  154. for metric_name, dimensions, amount in classified:
  155. with self._lock:
  156. if not self._active:
  157. return
  158. try:
  159. self._persist(metric_name, dimensions, amount)
  160. self._record_milestones(metric_name, dimensions)
  161. self._record_features(metric_name, dimensions)
  162. except Exception:
  163. saved = False
  164. logger.warning(
  165. "Unable to persist the Hermes shared metric: %s", metric_name, exc_info=True
  166. )
  167. if ticket and saved:
  168. with self._lock:
  169. if self._active:
  170. self._saved_tickets[ticket] = self._saved_tickets.get(ticket, 0) + 1