shared_metrics_loop.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388
  1. """Learning-loop, delegation and execution-backend shared metrics.
  2. Memory operations, curator runs, delegate_task fan-outs and terminal/browser/execute_code backend
  3. calls. Producers call the ``record_*`` functions with RAW values; every builder here maps them onto
  4. the closed enums and buckets in ``shared_metrics_contract``. Recording is a no-op unless shared
  5. metrics are enabled for the owning profile and never raises into the caller.
  6. This module imports only the stdlib at load time so tool modules can import it on their hot path.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import contextvars
  11. import json
  12. import logging
  13. import threading
  14. from dataclasses import dataclass, field
  15. from typing import Any, Callable, Iterable, Iterator
  16. logger = logging.getLogger(__name__)
  17. @contextlib.contextmanager
  18. def _bound_home(hermes_home: Any) -> Iterator[None]:
  19. """Record in the profile captured by the producer (curator/delegation threads carry no binding)."""
  20. if not hermes_home:
  21. yield
  22. return
  23. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  24. token = set_hermes_home_override(hermes_home)
  25. try:
  26. yield
  27. finally:
  28. reset_hermes_home_override(token)
  29. def _emit(mark_attr: str, build: Callable[..., dict[str, str] | None], *, hermes_home: Any = None, **raw: Any) -> None:
  30. try:
  31. from . import shared_metrics_contract as contract
  32. from .shared_metrics_events import _emit as emit_mark
  33. with _bound_home(hermes_home):
  34. emit_mark(getattr(contract, mark_attr), build, **raw)
  35. except Exception:
  36. logger.debug("Shared-metrics %s not recorded", mark_attr, exc_info=True)
  37. def _norm(value: Any) -> str:
  38. return str(value or "").strip().lower()
  39. def _parsed(result: Any) -> dict[str, Any]:
  40. if isinstance(result, dict):
  41. return result
  42. try:
  43. data = json.loads(result) if isinstance(result, (str, bytes)) else None
  44. except (TypeError, ValueError):
  45. return {}
  46. return data if isinstance(data, dict) else {}
  47. def _current_home() -> str | None:
  48. try:
  49. from hermes_constants import get_hermes_home
  50. return str(get_hermes_home())
  51. except Exception:
  52. return None
  53. # ---- memory ----------------------------------------------------------------------------------
  54. _MEMORY_OP_WORDS = {
  55. **dict.fromkeys(
  56. ("add", "conclude", "create", "ingest", "remember", "retain", "save", "store", "upload"), "add"
  57. ),
  58. **dict.fromkeys(("edit", "replace", "update"), "replace"),
  59. **dict.fromkeys(("delete", "forget", "remove"), "remove"),
  60. **dict.fromkeys(("probe", "query", "reason", "reasoning", "recall", "related", "search"), "search"),
  61. **dict.fromkeys(("context", "get", "list", "profile", "read"), "read"),
  62. }
  63. # A batch can carry many operations; one call never exports more rows than this.
  64. _MAX_BATCH_OPS = 25
  65. def memory_op(value: Any) -> str:
  66. return _MEMORY_OP_WORDS.get(_norm(value), "other")
  67. def memory_provider_op(tool_name: Any, args: Any) -> str:
  68. """The operation a provider tool performs: its ``action`` argument, else a verb in its name."""
  69. action = args.get("action") if isinstance(args, dict) else None
  70. if isinstance(action, str) and _norm(action) in _MEMORY_OP_WORDS:
  71. return _MEMORY_OP_WORDS[_norm(action)]
  72. for word in reversed(_norm(tool_name).replace("-", "_").split("_")):
  73. if word in _MEMORY_OP_WORDS:
  74. return _MEMORY_OP_WORDS[word]
  75. return "other"
  76. def memory_provider_name(value: Any) -> str:
  77. from .shared_metrics_contract import MEMORY_PROVIDERS
  78. name = _norm(value) or "builtin"
  79. return name if name in MEMORY_PROVIDERS else "plugin"
  80. def _memory_origin() -> str:
  81. from tools.skill_provenance import is_background_review
  82. return "background_review" if is_background_review() else "foreground"
  83. def memory_op_fields(*, op: Any, provider: Any, outcome: Any, origin: Any, failure_class: Any) -> dict[str, str]:
  84. from .shared_metrics_contract import MEMORY_OP_FAILURE_CLASSES, MEMORY_OP_ORIGINS, MEMORY_OP_OUTCOMES, MEMORY_OPS
  85. outcome_value, origin_value, class_value = _norm(outcome), _norm(origin), _norm(failure_class)
  86. outcome_value = outcome_value if outcome_value in MEMORY_OP_OUTCOMES else "failed"
  87. return {
  88. "failure_class": "none" if outcome_value == "success" else (
  89. class_value if class_value in MEMORY_OP_FAILURE_CLASSES - {"none"} else "unknown"
  90. ),
  91. "op": op if op in MEMORY_OPS else "other",
  92. "origin": origin_value if origin_value in MEMORY_OP_ORIGINS else "foreground",
  93. "outcome": outcome_value,
  94. "provider": memory_provider_name(provider),
  95. }
  96. def _record_memory_ops(ops: Iterable[str], *, provider: Any, outcome: str, failure_class: str) -> None:
  97. try:
  98. origin = _memory_origin()
  99. except Exception:
  100. origin = "foreground"
  101. for op in list(ops)[:_MAX_BATCH_OPS]:
  102. _emit(
  103. "MEMORY_OP_MARK", memory_op_fields, op=op, provider=provider, outcome=outcome, origin=origin,
  104. failure_class=failure_class,
  105. )
  106. def builtin_memory_ops(action: Any, operations: Any) -> list[str]:
  107. if isinstance(operations, list) and operations:
  108. return [memory_op(op.get("action") if isinstance(op, dict) else None) for op in operations]
  109. return [memory_op(action)]
  110. def record_builtin_memory_call(action: Any, operations: Any, *, outcome: str, failure_class: str = "unknown") -> None:
  111. """One row per operation of a built-in ``memory`` tool call (a batch applies all or none).
  112. ``failure_class`` is the store's closed refusal/failure name, ``none`` on success."""
  113. _record_memory_ops(
  114. builtin_memory_ops(action, operations), provider="builtin", outcome=outcome, failure_class=failure_class,
  115. )
  116. def _provider_result_failed(result: Any) -> bool:
  117. if not isinstance(result, str):
  118. return False
  119. head = result.lstrip()[:200]
  120. return head.startswith('{"error"') or '"success": false' in head
  121. def record_provider_memory_call(provider: Any, tool_name: Any, args: Any, result: Any = None, *, raised: bool = False) -> None:
  122. """One row per memory-provider tool call (plugin providers expose their own tools)."""
  123. failed = "exception" if raised else "provider_error" if _provider_result_failed(result) else None
  124. _record_memory_ops(
  125. [memory_provider_op(tool_name, args)], provider=provider, outcome="failed" if failed else "success",
  126. failure_class=failed or "none",
  127. )
  128. # ---- curator ---------------------------------------------------------------------------------
  129. _CURATOR_COUNT_KEYS = ("archived", "created", "merged", "patched")
  130. def curator_run_fields(*, trigger: Any, outcome: Any, counts: Any) -> dict[str, str]:
  131. from .shared_metrics_contract import CURATOR_OUTCOMES, count_bucket
  132. resolved = counts() if callable(counts) else counts
  133. resolved = resolved if isinstance(resolved, dict) else {}
  134. outcome_value = _norm(outcome)
  135. fields = {
  136. f"{key}_bucket": count_bucket(value if isinstance(value, int) and not isinstance(value, bool) else 0)
  137. for key in _CURATOR_COUNT_KEYS for value in (resolved.get(key),)
  138. }
  139. return {
  140. **fields,
  141. "outcome": outcome_value if outcome_value in CURATOR_OUTCOMES else "failed",
  142. "trigger": "scheduled" if _norm(trigger) == "scheduled" else "manual",
  143. }
  144. def record_curator_run(
  145. *, trigger: str, outcome: str, counts: dict[str, int] | Callable[[], dict[str, int]] | None = None,
  146. hermes_home: Any = None,
  147. ) -> None:
  148. """One row per curator pass. ``counts`` may be a callable so the diff is only computed when on."""
  149. _emit(
  150. "CURATOR_RUN_MARK", curator_run_fields, trigger=trigger, outcome=outcome, counts=counts or {},
  151. hermes_home=hermes_home,
  152. )
  153. # ---- delegation ------------------------------------------------------------------------------
  154. _SUCCESS_STATUSES = frozenset({"completed"})
  155. _CANCELLED_STATUSES = frozenset({"interrupted", "cancelled"})
  156. # Runs whose units never report back (a crashed async runner) must not grow this forever.
  157. _MAX_OPEN_DELEGATIONS = 256
  158. @dataclass
  159. class _DelegationRun:
  160. expected: int
  161. depth: int
  162. hermes_home: str | None
  163. statuses: list[Any] = field(default_factory=list)
  164. background: bool = False
  165. _DELEGATIONS: dict[int, _DelegationRun] = {}
  166. _DELEGATIONS_LOCK = threading.Lock()
  167. def delegation_depth(depth: Any) -> str:
  168. value = depth if isinstance(depth, int) and not isinstance(depth, bool) else 1
  169. return str(max(1, value)) if value < 4 else "gte_4"
  170. def delegation_outcome(statuses: Iterable[Any]) -> str:
  171. values = [_norm(s) for s in statuses]
  172. succeeded = sum(s in _SUCCESS_STATUSES for s in values)
  173. if values and succeeded == len(values):
  174. return "success"
  175. if succeeded:
  176. return "partial"
  177. return "cancelled" if any(s in _CANCELLED_STATUSES for s in values) else "failed"
  178. def delegation_run_fields(*, statuses: Any, depth: Any, background: Any, subagents: Any) -> dict[str, str]:
  179. from .shared_metrics_contract import count_bucket
  180. return {
  181. "depth": delegation_depth(depth),
  182. "mode": "background" if background is True else "foreground",
  183. "outcome": delegation_outcome(statuses or ()),
  184. "subagent_count_bucket": count_bucket(subagents if isinstance(subagents, int) else 0),
  185. }
  186. def begin_delegation_run(call_key: object, *, subagents: int, depth: int) -> None:
  187. """Open one delegate_task call; ``call_key`` is the object every unit of the call shares."""
  188. if subagents <= 0:
  189. return
  190. run = _DelegationRun(expected=subagents, depth=depth, hermes_home=_current_home())
  191. with _DELEGATIONS_LOCK:
  192. while len(_DELEGATIONS) >= _MAX_OPEN_DELEGATIONS:
  193. _DELEGATIONS.pop(next(iter(_DELEGATIONS)))
  194. _DELEGATIONS[id(call_key)] = run
  195. def finish_delegation_unit(call_key: object, results: Iterable[Any], *, background: bool) -> None:
  196. """Fold one joined unit into its call; the call's single row is emitted by its last unit."""
  197. statuses = [r.get("status") if isinstance(r, dict) else None for r in results]
  198. with _DELEGATIONS_LOCK:
  199. run = _DELEGATIONS.get(id(call_key))
  200. if run is None:
  201. return
  202. run.statuses.extend(statuses)
  203. run.background = run.background or background
  204. if len(run.statuses) < run.expected:
  205. return
  206. del _DELEGATIONS[id(call_key)]
  207. _emit(
  208. "DELEGATION_RUN_MARK", delegation_run_fields, statuses=run.statuses, depth=run.depth,
  209. background=run.background, subagents=run.expected, hermes_home=run.hermes_home,
  210. )
  211. # ---- execution backends ----------------------------------------------------------------------
  212. _STATUS_ERROR_CLASSES = {
  213. "timeout": "timeout", "timed_out": "timeout", "interrupted": "interrupted", "cancelled": "interrupted",
  214. "blocked": "blocked",
  215. }
  216. def _terminal_failure(data: dict[str, Any]) -> str | None:
  217. # 124 is the tool's own foreground-deadline code; it carries partial output and no ``error``.
  218. if data.get("exit_code") == 124:
  219. return "timeout"
  220. return "tool_error" if data.get("error") else None
  221. def _code_failure(data: dict[str, Any]) -> str | None:
  222. status = _norm(data.get("status"))
  223. if status in _STATUS_ERROR_CLASSES:
  224. return _STATUS_ERROR_CLASSES[status]
  225. # A traceback in the user's own code still ran on the backend; only a backend-level error
  226. # (no kernel reply, sandbox launch failure) carries ``error`` without any output.
  227. if status == "error" and data.get("error") and not data.get("output"):
  228. return "tool_error"
  229. return None
  230. def _browser_failure(data: dict[str, Any]) -> str | None:
  231. return "tool_error" if data.get("success") is False or data.get("error") else None
  232. _FAILURE_CLASSIFIERS = {"browser": _browser_failure, "code": _code_failure, "terminal": _terminal_failure}
  233. def execution_backend_fields(*, kind: Any, backend: Any, result: Any, error_class: Any) -> dict[str, str] | None:
  234. from .shared_metrics_contract import EXECUTION_BACKENDS_BY_KIND, TOOL_ERROR_CLASSES
  235. kind_value = _norm(kind)
  236. if kind_value not in EXECUTION_BACKENDS_BY_KIND:
  237. return None
  238. backend_value = _norm(backend() if callable(backend) else backend)
  239. failure = error_class if error_class is not None else _FAILURE_CLASSIFIERS[kind_value](_parsed(result))
  240. return {
  241. "backend": backend_value if backend_value in EXECUTION_BACKENDS_BY_KIND[kind_value] else "other",
  242. "error_class": "none" if failure is None else failure if failure in TOOL_ERROR_CLASSES else "unknown",
  243. "kind": kind_value,
  244. "outcome": "success" if failure is None else "failed",
  245. }
  246. # Set while Hermes itself drives a backend (TUI/Desktop path completion listings): that is not the
  247. # user's workload, so it must not show up as backend usage.
  248. _UNMETERED = contextvars.ContextVar("shared_metrics_unmetered_backend", default=False)
  249. @contextlib.contextmanager
  250. def unmetered_backend_calls() -> Iterator[None]:
  251. """Run Hermes-owned terminal/browser/code calls without counting them as execution-backend use."""
  252. token = _UNMETERED.set(True)
  253. try:
  254. yield
  255. finally:
  256. _UNMETERED.reset(token)
  257. def record_execution_backend(kind: str, backend: Any, result: Any = None, *, error_class: str | None = None) -> Any:
  258. """Count one tool call that reached its backend; returns ``result`` unchanged. ``backend`` may be a
  259. callable so resolving it costs nothing while collection is off. Calls the background review /
  260. curator forks make are Hermes' own work, not the user's (terminal.outcome skips them too)."""
  261. from tools.skill_provenance import is_background_review
  262. if _UNMETERED.get() or is_background_review():
  263. return result
  264. _emit(
  265. "EXECUTION_BACKEND_MARK", execution_backend_fields, kind=kind, backend=backend, result=result,
  266. error_class=error_class,
  267. )
  268. return result
  269. def record_terminal_backend(plan: Any, result: Any, *, error_class: str | None = None) -> Any:
  270. """Terminal calls that got past planning; ``plan`` is None when planning itself failed."""
  271. if plan is None:
  272. return result
  273. return record_execution_backend("terminal", getattr(plan, "env_type", None), result, error_class=error_class)
  274. def record_browser_call(call: Callable[[], Any], resolve_backend: Callable[[], str]) -> Any:
  275. """Run one browser_exec call and count it against the backend that served it (``resolve_backend``
  276. runs only while collection is on, after the call)."""
  277. try:
  278. result = call()
  279. except Exception:
  280. _safe_record_browser(resolve_backend, None, "exception")
  281. raise
  282. _safe_record_browser(resolve_backend, result, None)
  283. return result
  284. def _safe_record_browser(backend: Callable[[], str], result: Any, error_class: str | None) -> None:
  285. def name() -> str:
  286. try:
  287. return backend()
  288. except Exception:
  289. return "other"
  290. record_execution_backend("browser", name, result, error_class=error_class)