| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388 |
- """Learning-loop, delegation and execution-backend shared metrics.
- Memory operations, curator runs, delegate_task fan-outs and terminal/browser/execute_code backend
- calls. Producers call the ``record_*`` functions with RAW values; every builder here maps them onto
- the closed enums and buckets in ``shared_metrics_contract``. Recording is a no-op unless shared
- metrics are enabled for the owning profile and never raises into the caller.
- This module imports only the stdlib at load time so tool modules can import it on their hot path.
- """
- from __future__ import annotations
- import contextlib
- import contextvars
- import json
- import logging
- import threading
- from dataclasses import dataclass, field
- from typing import Any, Callable, Iterable, Iterator
- logger = logging.getLogger(__name__)
- @contextlib.contextmanager
- def _bound_home(hermes_home: Any) -> Iterator[None]:
- """Record in the profile captured by the producer (curator/delegation threads carry no binding)."""
- if not hermes_home:
- yield
- return
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- token = set_hermes_home_override(hermes_home)
- try:
- yield
- finally:
- reset_hermes_home_override(token)
- def _emit(mark_attr: str, build: Callable[..., dict[str, str] | None], *, hermes_home: Any = None, **raw: Any) -> None:
- try:
- from . import shared_metrics_contract as contract
- from .shared_metrics_events import _emit as emit_mark
- with _bound_home(hermes_home):
- emit_mark(getattr(contract, mark_attr), build, **raw)
- except Exception:
- logger.debug("Shared-metrics %s not recorded", mark_attr, exc_info=True)
- def _norm(value: Any) -> str:
- return str(value or "").strip().lower()
- def _parsed(result: Any) -> dict[str, Any]:
- if isinstance(result, dict):
- return result
- try:
- data = json.loads(result) if isinstance(result, (str, bytes)) else None
- except (TypeError, ValueError):
- return {}
- return data if isinstance(data, dict) else {}
- def _current_home() -> str | None:
- try:
- from hermes_constants import get_hermes_home
- return str(get_hermes_home())
- except Exception:
- return None
- # ---- memory ----------------------------------------------------------------------------------
- _MEMORY_OP_WORDS = {
- **dict.fromkeys(
- ("add", "conclude", "create", "ingest", "remember", "retain", "save", "store", "upload"), "add"
- ),
- **dict.fromkeys(("edit", "replace", "update"), "replace"),
- **dict.fromkeys(("delete", "forget", "remove"), "remove"),
- **dict.fromkeys(("probe", "query", "reason", "reasoning", "recall", "related", "search"), "search"),
- **dict.fromkeys(("context", "get", "list", "profile", "read"), "read"),
- }
- # A batch can carry many operations; one call never exports more rows than this.
- _MAX_BATCH_OPS = 25
- def memory_op(value: Any) -> str:
- return _MEMORY_OP_WORDS.get(_norm(value), "other")
- def memory_provider_op(tool_name: Any, args: Any) -> str:
- """The operation a provider tool performs: its ``action`` argument, else a verb in its name."""
- action = args.get("action") if isinstance(args, dict) else None
- if isinstance(action, str) and _norm(action) in _MEMORY_OP_WORDS:
- return _MEMORY_OP_WORDS[_norm(action)]
- for word in reversed(_norm(tool_name).replace("-", "_").split("_")):
- if word in _MEMORY_OP_WORDS:
- return _MEMORY_OP_WORDS[word]
- return "other"
- def memory_provider_name(value: Any) -> str:
- from .shared_metrics_contract import MEMORY_PROVIDERS
- name = _norm(value) or "builtin"
- return name if name in MEMORY_PROVIDERS else "plugin"
- def _memory_origin() -> str:
- from tools.skill_provenance import is_background_review
- return "background_review" if is_background_review() else "foreground"
- def memory_op_fields(*, op: Any, provider: Any, outcome: Any, origin: Any, failure_class: Any) -> dict[str, str]:
- from .shared_metrics_contract import MEMORY_OP_FAILURE_CLASSES, MEMORY_OP_ORIGINS, MEMORY_OP_OUTCOMES, MEMORY_OPS
- outcome_value, origin_value, class_value = _norm(outcome), _norm(origin), _norm(failure_class)
- outcome_value = outcome_value if outcome_value in MEMORY_OP_OUTCOMES else "failed"
- return {
- "failure_class": "none" if outcome_value == "success" else (
- class_value if class_value in MEMORY_OP_FAILURE_CLASSES - {"none"} else "unknown"
- ),
- "op": op if op in MEMORY_OPS else "other",
- "origin": origin_value if origin_value in MEMORY_OP_ORIGINS else "foreground",
- "outcome": outcome_value,
- "provider": memory_provider_name(provider),
- }
- def _record_memory_ops(ops: Iterable[str], *, provider: Any, outcome: str, failure_class: str) -> None:
- try:
- origin = _memory_origin()
- except Exception:
- origin = "foreground"
- for op in list(ops)[:_MAX_BATCH_OPS]:
- _emit(
- "MEMORY_OP_MARK", memory_op_fields, op=op, provider=provider, outcome=outcome, origin=origin,
- failure_class=failure_class,
- )
- def builtin_memory_ops(action: Any, operations: Any) -> list[str]:
- if isinstance(operations, list) and operations:
- return [memory_op(op.get("action") if isinstance(op, dict) else None) for op in operations]
- return [memory_op(action)]
- def record_builtin_memory_call(action: Any, operations: Any, *, outcome: str, failure_class: str = "unknown") -> None:
- """One row per operation of a built-in ``memory`` tool call (a batch applies all or none).
- ``failure_class`` is the store's closed refusal/failure name, ``none`` on success."""
- _record_memory_ops(
- builtin_memory_ops(action, operations), provider="builtin", outcome=outcome, failure_class=failure_class,
- )
- def _provider_result_failed(result: Any) -> bool:
- if not isinstance(result, str):
- return False
- head = result.lstrip()[:200]
- return head.startswith('{"error"') or '"success": false' in head
- def record_provider_memory_call(provider: Any, tool_name: Any, args: Any, result: Any = None, *, raised: bool = False) -> None:
- """One row per memory-provider tool call (plugin providers expose their own tools)."""
- failed = "exception" if raised else "provider_error" if _provider_result_failed(result) else None
- _record_memory_ops(
- [memory_provider_op(tool_name, args)], provider=provider, outcome="failed" if failed else "success",
- failure_class=failed or "none",
- )
- # ---- curator ---------------------------------------------------------------------------------
- _CURATOR_COUNT_KEYS = ("archived", "created", "merged", "patched")
- def curator_run_fields(*, trigger: Any, outcome: Any, counts: Any) -> dict[str, str]:
- from .shared_metrics_contract import CURATOR_OUTCOMES, count_bucket
- resolved = counts() if callable(counts) else counts
- resolved = resolved if isinstance(resolved, dict) else {}
- outcome_value = _norm(outcome)
- fields = {
- f"{key}_bucket": count_bucket(value if isinstance(value, int) and not isinstance(value, bool) else 0)
- for key in _CURATOR_COUNT_KEYS for value in (resolved.get(key),)
- }
- return {
- **fields,
- "outcome": outcome_value if outcome_value in CURATOR_OUTCOMES else "failed",
- "trigger": "scheduled" if _norm(trigger) == "scheduled" else "manual",
- }
- def record_curator_run(
- *, trigger: str, outcome: str, counts: dict[str, int] | Callable[[], dict[str, int]] | None = None,
- hermes_home: Any = None,
- ) -> None:
- """One row per curator pass. ``counts`` may be a callable so the diff is only computed when on."""
- _emit(
- "CURATOR_RUN_MARK", curator_run_fields, trigger=trigger, outcome=outcome, counts=counts or {},
- hermes_home=hermes_home,
- )
- # ---- delegation ------------------------------------------------------------------------------
- _SUCCESS_STATUSES = frozenset({"completed"})
- _CANCELLED_STATUSES = frozenset({"interrupted", "cancelled"})
- # Runs whose units never report back (a crashed async runner) must not grow this forever.
- _MAX_OPEN_DELEGATIONS = 256
- @dataclass
- class _DelegationRun:
- expected: int
- depth: int
- hermes_home: str | None
- statuses: list[Any] = field(default_factory=list)
- background: bool = False
- _DELEGATIONS: dict[int, _DelegationRun] = {}
- _DELEGATIONS_LOCK = threading.Lock()
- def delegation_depth(depth: Any) -> str:
- value = depth if isinstance(depth, int) and not isinstance(depth, bool) else 1
- return str(max(1, value)) if value < 4 else "gte_4"
- def delegation_outcome(statuses: Iterable[Any]) -> str:
- values = [_norm(s) for s in statuses]
- succeeded = sum(s in _SUCCESS_STATUSES for s in values)
- if values and succeeded == len(values):
- return "success"
- if succeeded:
- return "partial"
- return "cancelled" if any(s in _CANCELLED_STATUSES for s in values) else "failed"
- def delegation_run_fields(*, statuses: Any, depth: Any, background: Any, subagents: Any) -> dict[str, str]:
- from .shared_metrics_contract import count_bucket
- return {
- "depth": delegation_depth(depth),
- "mode": "background" if background is True else "foreground",
- "outcome": delegation_outcome(statuses or ()),
- "subagent_count_bucket": count_bucket(subagents if isinstance(subagents, int) else 0),
- }
- def begin_delegation_run(call_key: object, *, subagents: int, depth: int) -> None:
- """Open one delegate_task call; ``call_key`` is the object every unit of the call shares."""
- if subagents <= 0:
- return
- run = _DelegationRun(expected=subagents, depth=depth, hermes_home=_current_home())
- with _DELEGATIONS_LOCK:
- while len(_DELEGATIONS) >= _MAX_OPEN_DELEGATIONS:
- _DELEGATIONS.pop(next(iter(_DELEGATIONS)))
- _DELEGATIONS[id(call_key)] = run
- def finish_delegation_unit(call_key: object, results: Iterable[Any], *, background: bool) -> None:
- """Fold one joined unit into its call; the call's single row is emitted by its last unit."""
- statuses = [r.get("status") if isinstance(r, dict) else None for r in results]
- with _DELEGATIONS_LOCK:
- run = _DELEGATIONS.get(id(call_key))
- if run is None:
- return
- run.statuses.extend(statuses)
- run.background = run.background or background
- if len(run.statuses) < run.expected:
- return
- del _DELEGATIONS[id(call_key)]
- _emit(
- "DELEGATION_RUN_MARK", delegation_run_fields, statuses=run.statuses, depth=run.depth,
- background=run.background, subagents=run.expected, hermes_home=run.hermes_home,
- )
- # ---- execution backends ----------------------------------------------------------------------
- _STATUS_ERROR_CLASSES = {
- "timeout": "timeout", "timed_out": "timeout", "interrupted": "interrupted", "cancelled": "interrupted",
- "blocked": "blocked",
- }
- def _terminal_failure(data: dict[str, Any]) -> str | None:
- # 124 is the tool's own foreground-deadline code; it carries partial output and no ``error``.
- if data.get("exit_code") == 124:
- return "timeout"
- return "tool_error" if data.get("error") else None
- def _code_failure(data: dict[str, Any]) -> str | None:
- status = _norm(data.get("status"))
- if status in _STATUS_ERROR_CLASSES:
- return _STATUS_ERROR_CLASSES[status]
- # A traceback in the user's own code still ran on the backend; only a backend-level error
- # (no kernel reply, sandbox launch failure) carries ``error`` without any output.
- if status == "error" and data.get("error") and not data.get("output"):
- return "tool_error"
- return None
- def _browser_failure(data: dict[str, Any]) -> str | None:
- return "tool_error" if data.get("success") is False or data.get("error") else None
- _FAILURE_CLASSIFIERS = {"browser": _browser_failure, "code": _code_failure, "terminal": _terminal_failure}
- def execution_backend_fields(*, kind: Any, backend: Any, result: Any, error_class: Any) -> dict[str, str] | None:
- from .shared_metrics_contract import EXECUTION_BACKENDS_BY_KIND, TOOL_ERROR_CLASSES
- kind_value = _norm(kind)
- if kind_value not in EXECUTION_BACKENDS_BY_KIND:
- return None
- backend_value = _norm(backend() if callable(backend) else backend)
- failure = error_class if error_class is not None else _FAILURE_CLASSIFIERS[kind_value](_parsed(result))
- return {
- "backend": backend_value if backend_value in EXECUTION_BACKENDS_BY_KIND[kind_value] else "other",
- "error_class": "none" if failure is None else failure if failure in TOOL_ERROR_CLASSES else "unknown",
- "kind": kind_value,
- "outcome": "success" if failure is None else "failed",
- }
- # Set while Hermes itself drives a backend (TUI/Desktop path completion listings): that is not the
- # user's workload, so it must not show up as backend usage.
- _UNMETERED = contextvars.ContextVar("shared_metrics_unmetered_backend", default=False)
- @contextlib.contextmanager
- def unmetered_backend_calls() -> Iterator[None]:
- """Run Hermes-owned terminal/browser/code calls without counting them as execution-backend use."""
- token = _UNMETERED.set(True)
- try:
- yield
- finally:
- _UNMETERED.reset(token)
- def record_execution_backend(kind: str, backend: Any, result: Any = None, *, error_class: str | None = None) -> Any:
- """Count one tool call that reached its backend; returns ``result`` unchanged. ``backend`` may be a
- callable so resolving it costs nothing while collection is off. Calls the background review /
- curator forks make are Hermes' own work, not the user's (terminal.outcome skips them too)."""
- from tools.skill_provenance import is_background_review
- if _UNMETERED.get() or is_background_review():
- return result
- _emit(
- "EXECUTION_BACKEND_MARK", execution_backend_fields, kind=kind, backend=backend, result=result,
- error_class=error_class,
- )
- return result
- def record_terminal_backend(plan: Any, result: Any, *, error_class: str | None = None) -> Any:
- """Terminal calls that got past planning; ``plan`` is None when planning itself failed."""
- if plan is None:
- return result
- return record_execution_backend("terminal", getattr(plan, "env_type", None), result, error_class=error_class)
- def record_browser_call(call: Callable[[], Any], resolve_backend: Callable[[], str]) -> Any:
- """Run one browser_exec call and count it against the backend that served it (``resolve_backend``
- runs only while collection is on, after the call)."""
- try:
- result = call()
- except Exception:
- _safe_record_browser(resolve_backend, None, "exception")
- raise
- _safe_record_browser(resolve_backend, result, None)
- return result
- def _safe_record_browser(backend: Callable[[], str], result: Any, error_class: str | None) -> None:
- def name() -> str:
- try:
- return backend()
- except Exception:
- return "other"
- record_execution_backend("browser", name, result, error_class=error_class)
|