shared_metrics_signals.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. """v5 signals: which disabled built-in tools models reach for, and when an install first adopts a feature.
  2. ``hermes.tool_unavailable.count`` is counted where the agent validates the calls a model emitted: a
  3. name Hermes ships (``toolsets.BUILTIN_TOOL_NAMES``) that this session did not enable. Any other name
  4. stays the v4 ``unknown_tool`` quality issue only, so a plugin, MCP or hallucinated name never leaves.
  5. ``hermes.feature_adoption.count`` rides on counters the subscriber already records: the first counter
  6. row that proves a feature was really used latches it once per install, with the owning profile's age.
  7. Features no counter can see report through :func:`record_feature_used`.
  8. """
  9. from __future__ import annotations
  10. import logging
  11. import sqlite3
  12. import time
  13. from pathlib import Path
  14. from typing import Any, Callable
  15. from . import shared_metrics_contract as contract
  16. logger = logging.getLogger(__name__)
  17. _DAY_S = 86_400
  18. _DAYS_SINCE_INSTALL_THRESHOLDS = (
  19. (_DAY_S, "same_day"), (7 * _DAY_S, "1d_to_7d"), (30 * _DAY_S, "7d_to_30d"), (90 * _DAY_S, "30d_to_90d"),
  20. )
  21. # ---- tool unavailable ------------------------------------------------------------------------
  22. def tool_unavailable_fields(agent: Any, name: Any, issue: str, route: dict[str, str]) -> dict[str, str] | None:
  23. """Fields for a call to a shipped built-in the session has disabled; None for anything else.
  24. Background reviews, delegated children and cron jobs run with toolsets Hermes (or the parent
  25. model) narrowed on purpose, so their misses say nothing about what users should get by default.
  26. A built-in deferred behind ``tool_search`` is enabled (reachable through ``tool_call``), not missing.
  27. """
  28. if issue != "unknown_tool" or not isinstance(name, str) or name not in contract.BUILTIN_TOOL_NAMES:
  29. return None
  30. if getattr(agent, "_delegate_depth", 0) or getattr(agent, "platform", None) == "cron":
  31. return None
  32. from agent.tool_executor import _tool_search_scoped_names
  33. from tools.skill_provenance import is_background_review
  34. if is_background_review() or name in _tool_search_scoped_names(agent):
  35. return None
  36. return {**route, "tool_name": name}
  37. # ---- feature adoption ------------------------------------------------------------------------
  38. def _success(d: dict[str, str]) -> bool:
  39. return d.get("outcome") == "success"
  40. def _tool(*names: str, prefix: str = "") -> Callable[[dict[str, str]], bool]:
  41. def used(d: dict[str, str]) -> bool:
  42. tool = d.get("tool_name") or ""
  43. return _success(d) and (tool in names or bool(prefix) and tool.startswith(prefix))
  44. return used
  45. def user_created_skill(d: dict[str, str]) -> bool:
  46. """A skill created at the user's request (``agent_created`` is Hermes' own background review)."""
  47. return d.get("action") == "created" and d.get("provenance") != "agent_created"
  48. _TASK_SURFACE = {"desktop": "desktop", "tui": "tui"}
  49. # metric -> [(feature, predicate over the recorded dimensions)]. Unattended Hermes-owned work only
  50. # counts where the user set it up (a cron job, a curator run the user started).
  51. _FEATURE_RULES: dict[str, tuple[tuple[str, Callable[[dict[str, str]], bool]], ...]] = {
  52. contract.MEMORY_OP_METRIC: (("memory", lambda d: _success(d) and d.get("origin") == "foreground"),),
  53. contract.SKILL_LIFECYCLE_METRIC: (("skills_created", user_created_skill),),
  54. contract.DELEGATION_RUN_METRIC: (("delegation", lambda d: True),),
  55. contract.CRON_RUN_METRIC: (("cron", lambda d: d.get("outcome") in {"success", "failed"}),),
  56. contract.CURATOR_RUN_METRIC: (("curator", lambda d: _success(d) and d.get("trigger") == "manual"),),
  57. contract.TASK_STARTED_METRIC: (
  58. ("gateway_platform", lambda d: d.get("platform") not in {None, "none"}),
  59. ("desktop", lambda d: d.get("execution_surface") == "desktop"),
  60. ("tui", lambda d: d.get("execution_surface") == "tui"),
  61. ),
  62. contract.TOOL_USAGE_METRIC: (
  63. ("mcp", _tool("mcp")),
  64. ("plugins", _tool("plugin")),
  65. ("browser", _tool(prefix="browser_")),
  66. ("voice", _tool("text_to_speech")),
  67. ("kanban", _tool(prefix="kanban_")),
  68. ("projects", _tool("desktop_project")),
  69. ),
  70. contract.SLASH_COMMAND_METRIC: (
  71. ("voice", lambda d: d.get("command") == "voice"),
  72. ("kanban", lambda d: d.get("command") == "kanban"),
  73. ),
  74. contract.FEATURE_USED_MARK: tuple((feature, lambda d, f=feature: d.get("feature") == f) for feature in sorted(contract.FEATURES)),
  75. }
  76. def features_for(metric_name: str, dimensions: dict[str, str]) -> tuple[str, ...]:
  77. """Features a recorded counter proves were used (each is latched once per install by the store)."""
  78. return tuple(name for name, used in _FEATURE_RULES.get(metric_name, ()) if used(dimensions))
  79. def days_since_install_bucket(home: Path) -> str:
  80. """The owning profile's age (its first session), bucketed; ``unknown`` when state.db is unreadable."""
  81. from .shared_metrics_snapshot import _first_session_started_at
  82. try:
  83. first = _first_session_started_at(home)
  84. except (sqlite3.Error, OSError, TypeError, ValueError): # unreadable, or a non-numeric started_at
  85. return "unknown"
  86. age = contract._non_negative_number(time.time() - first) if first is not None else None
  87. return "unknown" if age is None else contract._bucket(age, _DAYS_SINCE_INSTALL_THRESHOLDS, "gte_90d")
  88. def feature_used_counter(event: Any) -> tuple[str, dict[str, str]] | None:
  89. """A first-use fact for a feature no counter sees (never stored as a counter row itself)."""
  90. if not contract._valid_shape(event, **contract._MARK_SHAPE):
  91. return None
  92. if contract._event_text(event, "name") != contract.FEATURE_USED_MARK:
  93. return None
  94. data = getattr(event, "data", None)
  95. if not isinstance(data, dict) or set(data) != {"feature"} or data["feature"] not in contract.FEATURES:
  96. return None
  97. return contract.FEATURE_USED_MARK, {"feature": data["feature"]}
  98. def record_feature_used(feature: str, *, hermes_home: Any = None) -> None:
  99. """Report one real use of ``feature`` (only the first per install becomes a row). Never raises.
  100. ``hermes_home`` binds the owning profile for callers outside it."""
  101. token = None
  102. try:
  103. if hermes_home:
  104. from hermes_constants import set_hermes_home_override
  105. token = set_hermes_home_override(str(hermes_home))
  106. from .relay_shared_metrics import enabled, record_process_mark
  107. if feature in contract.FEATURES and enabled():
  108. record_process_mark(contract.FEATURE_USED_MARK, {"feature": feature})
  109. except Exception:
  110. logger.debug("Shared-metrics feature use not recorded", exc_info=True)
  111. finally:
  112. if token is not None:
  113. from hermes_constants import reset_hermes_home_override
  114. reset_hermes_home_override(token)