shared_metrics_engagement.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. """Daily engagement rollup, per-conversation session tally and turns-before-switch.
  2. Engagement is accumulated locally per UTC day in the profile's metrics store and emitted once the
  3. day closes (the first interaction of a later day): one ``hermes.engagement.surface_day.count`` row
  4. per surface used, and one ``hermes.engagement.day.count`` row with the day's total active time, how
  5. many surfaces it touched and the model that served most attended turns. Active time is the sum of
  6. the gaps between consecutive interactions (a turn starting or ending), each capped at
  7. ``IDLE_CAP_MS``. Days-active-per-week and next-day/next-week return by model are derived
  8. server-side from these daily rows and the package ``install_id``: nothing here keeps a weekly
  9. window or any identifier of its own.
  10. The day row also carries ``active_profile_count_bucket``: how many distinct profiles of this host
  11. had a user turn that UTC day. Every profile's interactions are folded into ONE host accumulator,
  12. the root (default) profile's day state, as opaque local hashes of the profile home (a set, so a
  13. profile is counted once whichever process or multiplexed runtime served it). Only the root
  14. profile's day row reports the count; every other profile's row carries ``0`` so no row claims the
  15. host total twice. When the root profile has collection off, nothing is written to its store and
  16. the count is not reported.
  17. The day row also carries ``gateway_peak_busy_chats_bucket``: the most chats of this profile that had
  18. a turn running at the same moment that day, as the gateway process serving them saw it
  19. (``shared_metrics_busy``); folded in at every interaction, ``0`` for a profile no gateway served.
  20. Only closed enums, buckets, catalog-sanitized provider/model names and profile hashes are stored
  21. in the state row; the hashes never leave it.
  22. """
  23. from __future__ import annotations
  24. import hashlib
  25. import threading
  26. import time
  27. from dataclasses import dataclass
  28. from datetime import datetime, timezone, UTC
  29. from pathlib import Path
  30. from typing import Any, Callable
  31. from . import shared_metrics_contract as contract
  32. # A gap longer than this is idle time: it counts only up to the cap.
  33. IDLE_CAP_MS = 5 * 60_000
  34. STATE_KEY = "engagement_day_state"
  35. _MINUTE_MS = 60_000
  36. _ACTIVE_THRESHOLDS = (
  37. (5 * _MINUTE_MS, "lt_5m"), (30 * _MINUTE_MS, "5m_to_30m"), (120 * _MINUTE_MS, "30m_to_2h"),
  38. (360 * _MINUTE_MS, "2h_to_6h"),
  39. )
  40. _SWITCH_THRESHOLDS = ((2, "1"), (4, "2_to_3"), (11, "4_to_10"), (31, "11_to_30"))
  41. # Task entrypoints that are a person using Hermes. Unattended cron runs (counted by hermes.cron.run),
  42. # delegated children, background review forks, batch and API/python embedding are not engagement.
  43. # ``one_shot`` keeps the treatment it had while it read ``interactive``: a person typing ``hermes -z`` and a
  44. # script looping it look the same from inside the process; the label lets the rollup split them instead.
  45. _ENGAGED_ENTRYPOINTS = frozenset({"gateway_message", "interactive", "one_shot"})
  46. _INTERACTION_METRICS = frozenset({contract.TASK_STARTED_METRIC, contract.TASK_FINISHED_METRIC})
  47. _ALL = "*"
  48. _BUSY_PEAK = "busy_chats_peak"
  49. # Test / live-probe seam: wall clock in epoch seconds.
  50. _now: Callable[[], float] = time.time
  51. def active_minutes_bucket(active_ms: int) -> str:
  52. return "0" if active_ms <= 0 else contract._bucket(active_ms, _ACTIVE_THRESHOLDS, "gte_6h")
  53. def surfaces_used_bucket(count: int) -> str:
  54. return str(count) if count < 4 else "gte_4"
  55. def turns_before_switch_bucket(turns: int) -> str:
  56. return contract._bucket(turns, _SWITCH_THRESHOLDS, "gte_31")
  57. # ---- producer side (relay subscriber) ------------------------------------------------------------
  58. def interaction_surface(classified: list[tuple[str, dict, int]]) -> str | None:
  59. """The engagement surface of a task start/end counter, else None."""
  60. for metric_name, dimensions, _ in classified:
  61. if metric_name in _INTERACTION_METRICS and dimensions.get("entrypoint") in _ENGAGED_ENTRYPOINTS:
  62. surface = dimensions.get("execution_surface", "")
  63. return surface if surface in contract.ENGAGEMENT_SURFACES else None
  64. return None
  65. def engaged_turn(start_fields: dict[str, str] | None, user_turn: bool) -> bool:
  66. """A turn a person sent on an engagement surface (its model counts toward the day's primary model)."""
  67. return user_turn and (start_fields or {}).get("entrypoint") in _ENGAGED_ENTRYPOINTS
  68. def turn_route(event: Any) -> tuple[str, str] | None:
  69. """(provider, model) of one validated engagement turn mark."""
  70. if not contract._valid_shape(event, **contract._MARK_SHAPE):
  71. return None
  72. if contract._event_text(event, "name") != contract.ENGAGEMENT_TURN_MARK:
  73. return None
  74. data = getattr(event, "data", None)
  75. if not isinstance(data, dict) or set(data) != {"model", "provider"}:
  76. return None
  77. limits = contract._MODEL_ROUTE_MAX_LENGTHS
  78. route = tuple(data[k] for k in ("provider", "model"))
  79. if not all(isinstance(v, str) and v == contract._metric_identifier(v, max_length=limits[k])
  80. for k, v in zip(("provider", "model"), route)):
  81. return None
  82. return route # type: ignore[return-value]
  83. def record(store: Any, resource: dict[str, str], *, surface: str | None, route: tuple[str, str] | None) -> None:
  84. """The clock is read inside each write transaction: writers then apply in clock order, so no
  85. process closes a day another already closed. A busy store defers the interaction, never drops it."""
  86. profile_home = Path(store.database_path).parent.parent.parent
  87. root_home = _root_home(profile_home)
  88. owner = root_home == profile_home
  89. profile = _profile_hash(profile_home) if surface is not None else None
  90. from .shared_metrics_busy import day_peak
  91. store.update_rollup_state(STATE_KEY, lambda state: apply(
  92. state, now_ms=_now_ms(), resource=resource, surface=surface, route=route,
  93. profile=profile if owner else None, owner=owner, busy_peak=lambda day: day_peak(profile_home, day),
  94. ))
  95. if profile is not None and not owner and _collects(root_home):
  96. _root_store(root_home).update_rollup_state(STATE_KEY, lambda state: apply(
  97. state, now_ms=_now_ms(), resource=resource, profile=profile, owner=True,
  98. ))
  99. def _now_ms() -> int:
  100. return int(_now() * 1000)
  101. def _root_home(profile_home: Path) -> Path:
  102. from hermes_constants import get_default_hermes_root
  103. try:
  104. return get_default_hermes_root(home=profile_home).resolve()
  105. except Exception:
  106. return profile_home.resolve()
  107. def _profile_hash(profile_home: Path) -> str:
  108. return hashlib.sha256(str(profile_home.resolve()).encode()).hexdigest()[:16]
  109. def _collects(home: Path) -> bool:
  110. """The root profile's own collection consent (read-only; never touches its runtime)."""
  111. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  112. token = set_hermes_home_override(str(home))
  113. try:
  114. from hermes_cli.config import read_raw_config_readonly
  115. config: Any = read_raw_config_readonly() or {}
  116. except Exception:
  117. return False
  118. finally:
  119. reset_hermes_home_override(token)
  120. for key in ("telemetry", "shared_metrics"):
  121. config = config.get(key) if isinstance(config, dict) else None
  122. return isinstance(config, dict) and config.get("enabled") is True
  123. _root_stores: dict[Path, Any] = {}
  124. _root_stores_lock = threading.Lock()
  125. def _root_store(home: Path) -> Any:
  126. from .shared_metrics import SharedMetricsStore
  127. with _root_stores_lock:
  128. if home not in _root_stores:
  129. root = home / "telemetry" / "shared_metrics"
  130. _root_stores[home] = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox")
  131. return _root_stores[home]
  132. # ---- the rollup (pure) ---------------------------------------------------------------------------
  133. def _utc_day(now_ms: int) -> str:
  134. return datetime.fromtimestamp(now_ms / 1000, tz=UTC).date().isoformat()
  135. def _fresh(day: str, resource: dict[str, str], owner: bool) -> dict[str, Any]:
  136. return {"day": day, "resource": resource, "surfaces": {}, "models": {}, "profiles": [], "owner": owner}
  137. def _valid_state(state: Any) -> bool:
  138. return (
  139. isinstance(state, dict) and isinstance(state.get("day"), str)
  140. and isinstance(state.get("surfaces"), dict) and isinstance(state.get("models"), dict)
  141. )
  142. def apply(
  143. state: dict[str, Any] | None, *, now_ms: int, resource: dict[str, str],
  144. surface: str | None = None, route: tuple[str, str] | None = None, profile: str | None = None,
  145. owner: bool = False, busy_peak: Callable[[str], int] | None = None,
  146. ) -> tuple[dict[str, Any], list[tuple[str, dict, dict, str]]]:
  147. """Fold one interaction (``surface``), attended turn (``route``), a profile's user turn
  148. (``profile``, host accumulator only) and/or the day's running-chat peak (``busy_peak(day)``) into
  149. the day's state; close the stored day first when the clock has moved past it."""
  150. today, rows = _utc_day(now_ms), []
  151. if state is not None and not _valid_state(state):
  152. state = None
  153. if state is not None and state["day"] < today:
  154. if busy_peak is not None:
  155. _fold_busy_peak(state, busy_peak(state["day"])) # a peak reached just before midnight
  156. # The only close path. A stored day ahead of the clock (a step back across midnight) keeps
  157. # the interaction: the stored day never moves backwards, so no day is re-opened or emitted twice.
  158. rows, state = close_day(state), None
  159. state = state or _fresh(today, resource, owner)
  160. state["owner"] = owner
  161. profiles = state.setdefault("profiles", [])
  162. if profile is not None and isinstance(profiles, list) and profile not in profiles:
  163. profiles.append(profile)
  164. if surface in contract.ENGAGEMENT_SURFACES:
  165. for stream in (surface, _ALL):
  166. active_ms, last_ms = state["surfaces"].get(stream) or (0, None)
  167. if isinstance(last_ms, int):
  168. active_ms += min(max(0, now_ms - last_ms), IDLE_CAP_MS)
  169. state["surfaces"][stream] = [active_ms, max(now_ms, last_ms or 0)]
  170. if route is not None:
  171. key = " ".join(route)
  172. state["models"][key] = int(state["models"].get(key, 0)) + 1
  173. if busy_peak is not None:
  174. _fold_busy_peak(state, busy_peak(today))
  175. return state, rows
  176. def _fold_busy_peak(state: dict[str, Any], peak: int) -> None:
  177. stored = state.get(_BUSY_PEAK)
  178. state[_BUSY_PEAK] = max(stored if isinstance(stored, int) else 0, peak)
  179. def _primary(models: dict[str, Any]) -> tuple[str, str]:
  180. counted = [(int(n), key) for key, n in models.items() if isinstance(n, int) and key.count(" ") == 1]
  181. if not counted:
  182. return "none", "none"
  183. provider, model = max(counted)[1].split(" ")
  184. return provider, model
  185. def close_day(state: dict[str, Any]) -> list[tuple[str, dict, dict, str]]:
  186. """The rows one closed day reports: none for a day with no engaged surface, except the root
  187. profile's host row (0 minutes, 0 surfaces) carrying the day's active-profile count."""
  188. day, resource = state["day"], state.get("resource") or {}
  189. streams = {k: v for k, v in state["surfaces"].items() if isinstance(v, list) and len(v) == 2}
  190. surfaces = {k: v for k, v in streams.items() if k in contract.ENGAGEMENT_SURFACES}
  191. profiles = state.get("profiles") if state.get("owner") is True else None
  192. active_profiles = len(profiles) if isinstance(profiles, list) else 0
  193. if not surfaces and not active_profiles:
  194. return []
  195. rows = [
  196. (contract.ENGAGEMENT_SURFACE_METRIC,
  197. {"active_minutes_bucket": active_minutes_bucket(int(v[0])), "surface": name}, resource, day)
  198. for name, v in sorted(surfaces.items())
  199. ]
  200. provider, model = _primary(state["models"])
  201. peak = state.get(_BUSY_PEAK)
  202. rows.append((contract.ENGAGEMENT_DAY_METRIC, {
  203. "active_minutes_bucket": active_minutes_bucket(int((streams.get(_ALL) or [0])[0])),
  204. "active_profile_count_bucket": contract.size_bucket(active_profiles),
  205. contract.BUSY_PEAK_FIELD: contract.count_bucket(peak if isinstance(peak, int) else 0),
  206. "primary_model": model, "primary_provider": provider,
  207. "surfaces_used_count": surfaces_used_bucket(len(surfaces)),
  208. }, resource, day))
  209. return rows
  210. # ---- per-conversation state kept by the relay runtime -------------------------------------------
  211. @dataclass
  212. class RouteRun:
  213. """Consecutive turns the conversation's current model served since the last /model switch."""
  214. route: dict[str, str] | None = None
  215. turns: int = 0
  216. def observe(self, route: dict[str, str]) -> None:
  217. if route == self.route:
  218. self.turns += 1
  219. else:
  220. self.route, self.turns = route, 1
  221. def take(self) -> tuple[dict[str, str], int] | None:
  222. """The model being left and its turn count; resets (the next model starts from zero)."""
  223. taken = (self.route, self.turns) if self.route is not None and self.turns else None
  224. self.route, self.turns = None, 0
  225. return taken
  226. def switch_after_fields(route: dict[str, str], turns: int) -> dict[str, str]:
  227. return {**route, "turns_before_switch_bucket": turns_before_switch_bucket(turns)}
  228. @dataclass
  229. class SessionTally:
  230. """A session summary that merges the segments compression rotation splits one conversation into."""
  231. start_fields: dict[str, str] | None = None
  232. turns: int = 0
  233. failed_turns: int = 0
  234. last_outcome: str = "unknown"
  235. first_turn_ns: int = 0
  236. last_turn_ns: int = 0
  237. model_calls: int = 0
  238. tool_calls: int = 0
  239. replies: int = 0
  240. def absorb(self, other: SessionTally) -> None:
  241. if not other.turns:
  242. return
  243. if not self.turns or other.first_turn_ns < self.first_turn_ns:
  244. self.start_fields, self.first_turn_ns = other.start_fields, other.first_turn_ns
  245. if other.last_turn_ns >= self.last_turn_ns:
  246. self.last_outcome, self.last_turn_ns = other.last_outcome, other.last_turn_ns
  247. self.turns += other.turns
  248. self.failed_turns += other.failed_turns
  249. self.model_calls += other.model_calls
  250. self.tool_calls += other.tool_calls
  251. self.replies += other.replies
  252. def fields(self) -> dict[str, str] | None:
  253. if not self.turns or self.start_fields is None:
  254. return None
  255. from .shared_metrics_fields import session_fields
  256. return {
  257. **session_fields(
  258. self.start_fields, turns=self.turns, failed_turns=self.failed_turns, last_outcome=self.last_outcome,
  259. active_ms=max(0, self.last_turn_ns - self.first_turn_ns) // 1_000_000,
  260. ),
  261. # Messages the conversation added: each user turn, each primary-model reply, each tool result.
  262. "message_count_bucket": contract.long_size_bucket(self.turns + self.replies + self.tool_calls),
  263. "model_call_count_bucket": contract.long_size_bucket(self.model_calls),
  264. "tool_call_count_bucket": contract.long_size_bucket(self.tool_calls),
  265. }