| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675 |
- """Direct NeMo Relay integration for Hermes shared client metrics."""
- from __future__ import annotations
- import atexit
- import contextlib
- import contextvars
- import logging
- import threading
- import uuid
- from collections import OrderedDict, deque
- from dataclasses import dataclass, field
- from time import monotonic_ns
- from typing import Any, Callable, Iterator
- from agent import relay_runtime
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- from agent.portal_tags import get_conversation_context
- from hermes_cli.version_info import get_version_info
- from .shared_metrics import SharedMetricsStore
- from . import shared_metrics_contract as contract
- from . import shared_metrics_connectors as connectors_
- from . import shared_metrics_efficiency as eff
- from . import shared_metrics_engagement as engagement_
- from . import shared_metrics_fields as fields_
- from . import shared_metrics_model as model_
- from .shared_metrics_contract import MODEL_CALL_SCOPE, SUBSCRIBER_NAME, TASK_SCOPE
- from .shared_metrics_subscriber import SharedMetricsSubscriber
- logger = logging.getLogger(__name__)
- _RUNTIME_FAILED = object()
- _RUNTIMES: dict[str, _Runtime | object] = {}
- _RUNTIME_LOCK = threading.RLock()
- _ABORTED = {"failed": True, "turn_exit_reason": "system_aborted"}
- # The store latch allows one snapshot per 24h; re-reading it hourly keeps task starts cheap.
- _SNAPSHOT_RECHECK_NS = 3_600 * 1_000_000_000
- def _text(event: dict[str, Any], key: str) -> str:
- return str(event.get(key) or "")
- def _session_pair(event: dict[str, Any], key: str) -> tuple[str, str] | None:
- """(session_id, event[key]) when both are non-empty."""
- session_id, value = _text(event, "session_id"), _text(event, key)
- return (session_id, value) if session_id and value else None
- def _retry_ordinal(event: dict[str, Any]) -> int:
- """Hermes's provider-local retry ordinal; 0 when absent or malformed."""
- value = event.get("retry_count")
- return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0
- def _forget(index: dict[Any, _MetricsSession], key: Any, owner: _MetricsSession) -> None:
- """Drop ``key`` from ``index`` only while it still points at ``owner``."""
- if index.get(key) is owner:
- index.pop(key, None)
- def _task_parent_handle(session: _MetricsSession, task_id: str) -> Any:
- """The active turn's handle when it owns this exact task, else the session handle."""
- active_turn = relay_runtime.active_turn(session.session_id)
- if (
- active_turn is not None
- and active_turn.lease.session_id == session.session_id
- and active_turn.task_id == task_id
- and active_turn.handle is not None
- ):
- return active_turn.handle
- return session.relay_session.handle
- # Session sources a Hermes dispatcher stamps on the finite CLI child it spawns: a kanban worker
- # (``kanban_db_dispatch``) and an A2A forward (``--source a2a``). Nobody is at either keyboard.
- _DISPATCHED_SOURCES = frozenset({"a2a", "kanban"})
- def _with_launch_entrypoint(event: dict[str, Any]) -> dict[str, Any]:
- """Declare the entrypoint of a finite CLI run (``hermes -z``, ``chat -q`` off a TTY, ``-Q``): the
- ``HERMES_SINGLE_QUERY_SESSION`` marker those paths set (the same one ``cache_ttl: auto`` and the session
- source read). ``background`` for a dispatcher-spawned child, else ``one_shot``; the REPL, other surfaces,
- delegated children and an already-declared entrypoint are untouched."""
- if (event.get("entrypoint") or event.get("parent_task_id") or event.get("parent_session_id")
- or contract.execution_surface(event) != "cli"):
- return event
- from agent.oneshot_footprint import is_single_query_session
- if not is_single_query_session():
- return event
- from gateway.session_context import get_session_env
- source = str(get_session_env("HERMES_SESSION_SOURCE", "") or "").strip().lower()
- return {**event, "entrypoint": "background" if source in _DISPATCHED_SOURCES else "one_shot"}
- def _elapsed_ms(started_ns: int) -> int:
- return max(0, (monotonic_ns() - started_ns) // 1_000_000)
- def _scope_handle(session: _MetricsSession, task: _TaskRun | None) -> Any:
- return task.handle if task is not None else session.relay_session.handle
- def _sole(items: Any) -> Any:
- """The single distinct element of ``items`` (identity-deduplicated), else None."""
- unique = {id(item): item for item in items}
- return next(iter(unique.values())) if len(unique) == 1 else None
- def _identities_compatible(candidate: tuple[str, str, str], observed: tuple[str, str, str]) -> bool:
- """Match partial hook context without crossing known call boundaries."""
- if not observed[2] or candidate[2] != observed[2]:
- return False
- return all(
- not left or not right or left == right
- for left, right in zip(candidate[:2], observed[:2], strict=True)
- )
- def _compatible_tool_call_keys(
- session: _MetricsSession, task_id: str, identity: tuple[str, str, str]
- ) -> list[tuple[str, str, str, str]]:
- return [
- key
- for key in session.tool_calls
- if key[0] == task_id and _identities_compatible(key[1:], identity)
- ]
- @dataclass
- class _ModelCall:
- handle: Any
- task_id: str
- fields: dict[str, str]
- error_class: str = "none"
- ttft_bucket: str = "unknown"
- started_ns: int = field(default_factory=monotonic_ns)
- @dataclass
- class _ToolCall:
- handle: Any
- category: str
- tool_name: str
- started_ns: int
- approval_outcome: str = "not_required"
- @dataclass
- class _TaskRun:
- task_id: str
- handle: Any
- context: contextvars.Context
- started_ns: int
- start_fields: dict[str, str]
- model_call_ids: set[str] = field(default_factory=set)
- tool_call_ids: set[tuple[str, str, str]] = field(default_factory=set)
- turn_ids: set[str] = field(default_factory=set)
- retired_turn_ids: frozenset[str] = field(default_factory=frozenset)
- completed_tool_call_ids: set[tuple[str, str, str]] = field(default_factory=set)
- unidentified_tool_calls: int = 0
- retry_count: int = 0
- model_route: dict[str, str] | None = None
- # The route the turn was sent on (its first request): provider failover does not change it.
- selected_route: dict[str, str] | None = None
- cost: eff.TurnCost = field(default_factory=eff.TurnCost)
- @dataclass
- class _MetricsSession:
- session_id: str
- relay_session: relay_runtime.RelaySession
- lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
- closing: bool = False
- model_calls: dict[tuple[str, str], _ModelCall] = field(default_factory=dict)
- tasks: dict[str, _TaskRun] = field(default_factory=dict)
- tool_calls: dict[tuple[str, str, str, str], _ToolCall] = field(default_factory=dict)
- retired_turn_ids: deque[str] = field(default_factory=lambda: deque(maxlen=256))
- # Session-level aggregation, emitted once as hermes.session.count when the session closes.
- start_fields: dict[str, str] | None = None
- turns: int = 0
- failed_turns: int = 0
- last_outcome: str = "unknown"
- first_turn_ns: int = 0
- last_turn_ns: int = 0
- model_state: model_.ModelSessionState = field(default_factory=model_.ModelSessionState)
- # Compression rotated this id away: close once its in-flight turn ends.
- retiring: bool = False
- efficiency: eff.SessionEfficiency = field(default_factory=eff.SessionEfficiency)
- connectors: connectors_.ConnectorSessionState = field(default_factory=connectors_.ConnectorSessionState)
- route_run: engagement_.RouteRun = field(default_factory=engagement_.RouteRun)
- api_calls: int = 0
- tool_call_total: int = 0
- replies: int = 0
- @dataclass
- class _PeakLineage:
- """Open segments of one conversation (compression rotates the session id), their merged peak and
- session tally, the conversation's current-model turn run and its spent user turns."""
- open: set[str] = field(default_factory=set)
- peak: model_.ModelSessionState = field(default_factory=model_.ModelSessionState)
- tools: eff.ToolUsage = field(default_factory=eff.ToolUsage)
- tally: engagement_.SessionTally = field(default_factory=engagement_.SessionTally)
- run: engagement_.RouteRun = field(default_factory=engagement_.RouteRun)
- spent: deque | None = None
- connectors: connectors_.ConnectorSessionState | None = None
- # Backstop: conversations whose rotated-to segment a surface never closed are flushed past this many.
- _MAX_LINEAGES = 512
- def _absorb_peak(into: model_.ModelSessionState, segment: model_.ModelSessionState) -> None:
- """Merge one closed segment's context peak (fullest fill wins, a limit hit anywhere sticks)."""
- if segment.peak_route is None:
- return
- into.limit_hit = into.limit_hit or segment.limit_hit
- fuller = segment.peak_window is not None and (
- into.peak_window is None or segment.peak_tokens / segment.peak_window > into.peak_tokens / into.peak_window
- )
- if into.peak_route is None or fuller:
- into.peak_route, into.peak_tokens, into.peak_window = segment.peak_route, segment.peak_tokens, segment.peak_window
- class _Runtime:
- """Own shared-metrics state layered on the Hermes core Relay host."""
- def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None:
- resolved_host = host or relay_runtime.get_runtime()
- if resolved_host is None:
- raise RuntimeError("Hermes core Relay runtime is unavailable")
- self.host: relay_runtime.RelayRuntime = resolved_host
- self.relay = self.host.relay
- self._active = True
- self._sessions: dict[str, _MetricsSession] = {}
- self._task_sessions: dict[tuple[str, str], _MetricsSession] = {}
- self._turn_sessions: dict[tuple[str, str], _MetricsSession] = {}
- self._sessions_lock = threading.RLock()
- # Leaf lock (nothing is acquired under it): taken while a session.lock is held. Lineages are
- # keyed by the conversation's first segment id; _lineage_of maps every open segment id to it.
- self._lineages: dict[str, _PeakLineage] = {}
- self._lineage_of: dict[str, str] = {}
- self._lineage_lock = threading.Lock()
- # Leaf lock: conversations whose next cold cache read Hermes already announced.
- self._cold_expected: OrderedDict[str, None] = OrderedDict()
- self._cold_lock = threading.Lock()
- self._task_creation_lock = threading.RLock()
- self._task_sessions_lock = threading.RLock()
- # Guards the opt-in send pass: at most one in flight per process.
- self._send_lock = threading.RLock()
- self._send_thread: threading.Thread | None = None
- # Flushing is a process-wide Relay barrier. Keep it off interactive turn threads while
- # coalescing concurrent finish hooks into one worker.
- self._flush_lock = threading.RLock()
- self._flush_pending = False
- self._flush_thread: threading.Thread | None = None
- self._snapshot_checked_ns: int | None = None
- self._subscriber_name = f"{SUBSCRIBER_NAME}.{self.host.runtime_id}"
- self.subscriber = SharedMetricsSubscriber(
- SharedMetricsStore(), get_version_info().base_version, runtime_id=self.host.runtime_id
- )
- self.relay.subscribers.register(self._subscriber_name, self.subscriber)
- self.host.retain_managed_execution(self._subscriber_name)
- self._registered = True
- atexit.register(self.shutdown)
- def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None:
- session_id = _text(event, "session_id")
- if not session_id:
- return None
- with self._sessions_lock:
- if not self._active:
- return None
- relay_session = self.host.ensure_session(event)
- if relay_session is None:
- return None
- session = self._sessions.get(session_id)
- if session is None:
- session = _MetricsSession(session_id=session_id, relay_session=relay_session)
- self._sessions[session_id] = session
- self._share_spent(session)
- with session.lock:
- return None if session.closing else session
- def record_client_active(self, event: dict[str, Any]) -> None:
- """Emit one payload-free activation attempt under the session scope."""
- session = self.ensure_session(event)
- if session is not None:
- self._emit_client_active(session)
- def _emit_client_active(self, session: _MetricsSession) -> None:
- with session.lock:
- if not session.closing:
- self._mark(session, None, contract.CLIENT_ACTIVE_MARK, {})
- self._safe(self._emit_install_snapshot_if_due, session)
- def _emit_install_snapshot_if_due(self, session: _MetricsSession) -> None:
- now = monotonic_ns()
- if self._snapshot_checked_ns is not None and now - self._snapshot_checked_ns < _SNAPSHOT_RECHECK_NS:
- return
- self._snapshot_checked_ns = now
- if not self.subscriber.store.install_snapshot_due():
- return
- from .shared_metrics_snapshot import collect_install_snapshot
- fields = collect_install_snapshot(_raw_config())
- self._mark(session, None, contract.INSTALL_SNAPSHOT_MARK, fields)
- def _mark(
- self, session: _MetricsSession, task: _TaskRun | None, name: str, data: dict[str, str]
- ) -> None:
- """Emit one Relay mark under the task scope when given, else the session scope."""
- self._run_scoped(
- session, task, self.relay.scope.event, name,
- handle=_scope_handle(session, task), data=data, metadata=self._event_metadata(),
- )
- def start_task(self, event: dict[str, Any]) -> _TaskRun | None:
- """Open one Relay function scope for a Hermes task run."""
- task_key = _session_pair(event, "task_id")
- if task_key is None:
- return None
- _, task_id = task_key
- with self._task_creation_lock:
- owner = self._task_session(event)
- if owner is not None:
- with owner.lock:
- if owner.closing:
- return None
- task = owner.tasks.get(task_id)
- if task is not None and not self._admits(owner, task, event):
- return None
- return task
- session = self.ensure_session(event)
- if session is None:
- return None
- with session.lock:
- turn_id = _text(event, "turn_id")
- if (
- session.closing
- or (turn_id and turn_id in session.retired_turn_ids)
- or session.relay_session.context is None
- ):
- return None
- self._emit_client_active(session)
- task_context = session.relay_session.context.copy()
- start_fields = contract.task_start_fields(_with_launch_entrypoint(event))
- handle = task_context.run(
- self._with_scope_stack, self.relay.scope.push,
- TASK_SCOPE, self.relay.ScopeType.Function,
- handle=_task_parent_handle(session, task_id), input=start_fields,
- metadata=self._event_metadata(),
- )
- task = _TaskRun(
- task_id=task_id,
- handle=handle,
- context=task_context,
- started_ns=monotonic_ns(),
- start_fields=start_fields,
- retired_turn_ids=frozenset(session.retired_turn_ids),
- )
- session.tasks[task_id] = task
- with self._task_sessions_lock:
- self._task_sessions[task_key] = session
- self._remember_turn(session, task, event)
- return task
- def start_user_turn(self, event: dict[str, Any]) -> None:
- """pre_llm_call: a user message started this task (Hermes-owned forks never fire it)."""
- task = self.start_task(event)
- if task is not None:
- task.cost.user_turn = True
- def _run_in_task(
- self, task: _TaskRun, callback: Callable[..., Any], *args: Any, **kwargs: Any
- ) -> Any:
- return task.context.copy().run(self._with_scope_stack, callback, *args, **kwargs)
- def _with_scope_stack(self, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
- self.relay.get_scope_stack()
- return callback(*args, **kwargs)
- def start_model_call(self, event: dict[str, Any]) -> None:
- task_id = _text(event, "task_id")
- session, task = self._task_pair(event, start=True, allow_task_id_fallback=True)
- if task_id and task is None:
- return
- session = session or self.ensure_session(event)
- if session is None:
- return
- request_id = _text(event, "api_request_id")
- if not request_id:
- return
- model_call_key = (task_id, request_id)
- fields = contract.model_call_fields(event)
- with session.lock:
- if session.closing:
- return
- if task is not None and not self._admits(session, task, event, current=True):
- return
- existing = session.model_calls.get(model_call_key)
- if existing is not None:
- existing.fields = fields
- if task is not None:
- # Every repeated start for one logical request is another physical
- # attempt. Provider fallback resets Hermes's provider-local retry
- # ordinal, so ordinal deltas are not a reliable task-level counter.
- task.retry_count += 1
- return
- if task is not None:
- task.model_route = fields
- task.selected_route = task.selected_route or fields
- task.model_call_ids.add(request_id)
- if _retry_ordinal(event) > 0:
- # A real Hermes retry can advance api_request_id while carrying the
- # retry ordinal. Count that physical attempt.
- task.retry_count += 1
- handle = self._run_scoped(
- session, task, self.relay.llm.call, MODEL_CALL_SCOPE, self.relay.LLMRequest({}, {}),
- handle=_scope_handle(session, task), metadata=self._event_metadata(),
- model_name=contract.MODEL_CALL_PROFILE_MODEL,
- )
- session.model_calls[model_call_key] = _ModelCall(handle, task_id, fields)
- def update_model_call(self, event: dict[str, Any], *, finish: bool) -> None:
- """Refresh the located model call's fields from ``event``; ``finish`` closes it.
- ``api_request_error`` retains the latest attempt error without closing the logical
- call; ``post_api_request`` closes it.
- """
- session = self._any_session(event)
- if session is None:
- return
- with session.lock:
- if session.closing:
- return
- model_call_key = self._existing_model_call_key(session, event)
- model_call = session.model_calls.get(model_call_key) if model_call_key else None
- if model_call is None:
- return
- model_call.fields = contract.model_call_fields(event)
- if finish:
- task = session.tasks.get(model_call.task_id)
- session.replies += task is not None and task.cost.user_turn
- session.model_state.observe_call(model_call.fields, event.get("usage"), event.get("context_length"))
- self._observe_turn_call(session, session.tasks.get(model_call.task_id), model_call, event)
- model_call.ttft_bucket = fields_.ttft_bucket(event)
- self._finish_model_call(session, model_call_key, "success")
- tokens = fields_.model_token_fields(
- event.get("usage"), call_role="primary", **model_call.fields
- )
- if tokens is not None:
- self._guarded(
- "Hermes shared-metrics token mark failed", self._mark,
- session, session.tasks.get(model_call.task_id), contract.MODEL_TOKENS_MARK, tokens,
- )
- else:
- model_call.error_class = contract.model_error_class(event)
- session.model_state.observe_error(model_call.fields, model_call.error_class)
- def start_tool_call(self, event: dict[str, Any]) -> None:
- """Open one privacy-safe Relay tool lifecycle under its task."""
- task_id = _text(event, "task_id")
- session, task = self._task_pair(event, start=True, allow_task_id_fallback=True)
- if session is None or task is None or not _text(event, "tool_call_id"):
- return
- if connectors_.is_connector_batch(event):
- return # each entry opens its own lifecycle; the wrapper counts only if no entry ran
- identity = self._tool_call_identity(event)
- with session.lock:
- if not self._admits(session, task, event):
- return
- key = (task_id, *identity)
- if identity in task.completed_tool_call_ids or key in session.tool_calls:
- return
- task.tool_call_ids.add(identity)
- session.tool_calls[key] = self._open_tool_call(task, event)
- def record_approval(self, event: dict[str, Any]) -> None:
- """Record one bounded approval result without approval text or commands."""
- session, task = self._approval_task(event)
- if session is None or task is None:
- return
- outcome = contract.tool_approval_outcome(event)
- attribution = "unattributed"
- with session.lock:
- if session.closing or not self._event_matches_task_turn(task, event):
- return
- if _text(event, "tool_call_id"):
- identity = self._tool_call_identity(event)
- tool_call = session.tool_calls.get((task.task_id, *identity))
- if tool_call is None:
- key = _sole(_compatible_tool_call_keys(session, task.task_id, identity))
- tool_call = session.tool_calls[key] if key is not None else None
- if tool_call is not None:
- tool_call.approval_outcome = outcome
- attribution = "tool_call"
- self._mark(
- session, task, contract.TOOL_APPROVAL_MARK,
- {"attribution": attribution, "outcome": outcome},
- )
- def record_tool_call(self, event: dict[str, Any]) -> None:
- """Close and count one unique privacy-safe tool lifecycle."""
- task_id = _text(event, "task_id")
- session, task = self._task_pair(event, allow_task_id_fallback=True)
- if session is None or task is None:
- return
- with session.lock:
- if not self._admits(session, task, event):
- return
- tool_call = None
- if _text(event, "tool_call_id"):
- observed_identity = self._tool_call_identity(event)
- if self._settle_connector_batch(session, task, observed_identity, event):
- return
- if observed_identity in task.completed_tool_call_ids:
- return
- identity = observed_identity
- tool_call = session.tool_calls.pop((task_id, *identity), None)
- if tool_call is None:
- if any(
- _identities_compatible(completed, observed_identity)
- for completed in task.completed_tool_call_ids
- ):
- return
- matching_keys = _compatible_tool_call_keys(session, task_id, observed_identity)
- if len(matching_keys) > 1:
- # Partial context cannot safely choose between concurrent calls
- # that reused the provider-local ID.
- return
- if matching_keys:
- identity = matching_keys[0][1:]
- tool_call = session.tool_calls.pop(matching_keys[0])
- task.completed_tool_call_ids.update({identity, observed_identity})
- task.tool_call_ids.add(identity)
- else:
- task.unidentified_tool_calls += 1
- if tool_call is None:
- tool_call = self._open_tool_call(task, event)
- if task.cost.user_turn:
- session.efficiency.tools.used.add(eff.toolset_metric_name(event.get("toolset")))
- self._finish_tool_call(task, tool_call, event)
- chain = session.connectors.observe(event)
- if chain is not None:
- self._mark(session, task, contract.CONNECTOR_CHAIN_MARK, chain)
- def _settle_connector_batch(
- self, session: _MetricsSession, task: _TaskRun, wrapper: tuple[str, str, str], event: dict[str, Any],
- ) -> bool:
- """A connector batch wrapper's post is not a call of its own once an entry started: entries
- still open (the executor timed out or abandoned the batch) end with the wrapper's outcome.
- True when the wrapper was settled this way; a batch refused before any entry counts as itself."""
- if not connectors_.is_connector_batch(event):
- return False
- started = [key for key in session.tool_calls
- if key[0] == task.task_id and connectors_.is_batch_entry(key[1:], wrapper)]
- if not started and not connectors_.batch_entry_ran(task.completed_tool_call_ids, wrapper):
- return False
- for key in started:
- task.completed_tool_call_ids.add(key[1:])
- self._finish_tool_call(task, session.tool_calls.pop(key), event)
- task.completed_tool_call_ids.add(wrapper)
- return True
- def record_skill_lifecycle(self, event: dict[str, Any]) -> None:
- """Emit one allowlisted skill fact without its local identity."""
- if _text(event, "action").strip().lower() == "loaded":
- mark, fields = contract.SKILL_LOAD_MARK, contract.skill_load_fields(event)
- else:
- mark, fields = contract.SKILL_LIFECYCLE_MARK, contract.skill_lifecycle_fields(event)
- if fields is None:
- return
- session_id, task_id = _text(event, "session_id"), _text(event, "task_id")
- session, task = self._task_pair(event, allow_task_id_fallback=not session_id)
- if session is None:
- if not (session_id and task_id):
- # No owning task: a bare process-level mark.
- self._with_scope_stack(
- self.relay.scope.event, mark, data=fields, metadata=self._event_metadata()
- )
- return
- if task is None:
- return
- with session.lock:
- if (
- not session.closing
- and session.tasks.get(task.task_id) is task
- and self._event_matches_task_turn(task, event)
- ):
- self._mark(session, task, mark, fields)
- def finish_task(self, event: dict[str, Any]) -> None:
- """Close one task scope exactly once with bounded terminal fields."""
- session = self._any_session(event)
- if session is None:
- return
- with session.lock:
- finished = not session.closing and self._finish_task(
- session, _text(event, "task_id"), event
- )
- retired = finished and session.retiring and not session.tasks
- if retired:
- self.close_session({"session_id": session.session_id})
- elif finished:
- self._schedule_flush_and_export("Hermes shared-metrics task flush failed")
- def close_session(self, event: dict[str, Any]) -> None:
- session = self._session(event)
- if session is None:
- self._close_unseen_segment(_text(event, "session_id"))
- return
- if not self._abort_session(
- session, {**event, **_ABORTED, "completed": False, "interrupted": False}
- ):
- return
- self._emit_session_summary(session)
- self._schedule_flush_and_export(
- f"Hermes shared-metrics session {session.session_id} flush failed"
- )
- with self._sessions_lock:
- _forget(self._sessions, session.session_id, session)
- def shutdown(self) -> None:
- # Callers (the atexit hook, an os._exit flush, a multiplex host's teardown) hold the launch
- # profile's scope: this profile's own collection gate and send consent decide its exit.
- with self._owner_scope():
- if not enabled(): # revoked since its last turn: drop the data, never export it
- if self._registered:
- self.deactivate()
- return
- self._close_and_export()
- def _close_and_export(self) -> None:
- with self._sessions_lock:
- self._active = False
- session_ids = list(self._sessions)
- for session_id in session_ids:
- self._safe(self.close_session, {"session_id": session_id})
- with self._lineage_lock:
- unseen = list(self._lineage_of)
- for session_id in unseen:
- self._safe(self._close_unseen_segment, session_id)
- if not self._registered:
- return
- self._flush_and_export("Hermes shared-metrics shutdown flush failed")
- self._deregister()
- self._release()
- @contextlib.contextmanager
- def _owner_scope(self) -> Iterator[None]:
- """Bind this runtime's profile home for work that can run outside its own turns."""
- token = set_hermes_home_override(self.host.profile_key)
- try:
- yield
- finally:
- reset_hermes_home_override(token)
- def _deregister(self) -> None:
- self._safe(self.relay.subscribers.deregister, self._subscriber_name)
- self.host.release_managed_execution(self._subscriber_name)
- self._registered = False
- def deactivate(self) -> None:
- """Stop collection without exporting locally aggregated metrics."""
- with self._sessions_lock:
- self._active = False
- self.subscriber.deactivate()
- if self._registered:
- self._deregister()
- with self._sessions_lock:
- sessions = list(self._sessions.values())
- for session in sessions:
- self._abort_session(session, {"session_id": session.session_id, **_ABORTED})
- with self._sessions_lock:
- self._sessions.clear()
- with self._task_sessions_lock:
- self._task_sessions.clear()
- self._turn_sessions.clear()
- with self._lineage_lock:
- self._lineages.clear()
- self._lineage_of.clear()
- self._release()
- def _release(self) -> None:
- """Let an in-flight send finish briefly, then drop the atexit hook.
- A short-lived CLI process would otherwise exit and kill the daemon send thread
- mid-request — the common case for this feature's one cadence.
- """
- self._join_send_thread()
- with contextlib.suppress(Exception):
- atexit.unregister(self.shutdown)
- def _join_send_thread(self, timeout: float = 2.0) -> None:
- """Bounded on purpose: pending packages stay in SQLite and go out next run, so
- blocking on a slow network is the wrong trade; the daemon thread dies with the process."""
- with self._send_lock:
- thread = self._send_thread
- if thread is not None and thread.is_alive():
- try:
- thread.join(timeout)
- except Exception:
- logger.debug("Shared-metrics send thread join failed", exc_info=True)
- def _session(self, event: dict[str, Any]) -> _MetricsSession | None:
- with self._sessions_lock:
- return self._sessions.get(_text(event, "session_id"))
- def _any_session(self, event: dict[str, Any]) -> _MetricsSession | None:
- """Owner session by task/turn correlation, else by session_id."""
- return self._task_session(event, allow_task_id_fallback=True) or self._session(event)
- def _task_pair(
- self, event: dict[str, Any], *, start: bool = False, **lookup: Any
- ) -> tuple[_MetricsSession | None, _TaskRun | None]:
- """Resolve (session, task) for a task-scoped hook; ``start`` opens a missing task."""
- session = self._task_session(event, **lookup)
- task = session.tasks.get(_text(event, "task_id")) if session is not None else None
- if task is None and start:
- task = self.start_task(event)
- session = self._task_session(event) if task is not None else None
- return session, task
- def _run_scoped(
- self, session: _MetricsSession, task: _TaskRun | None, callback: Callable[..., Any],
- *args: Any, **kwargs: Any,
- ) -> Any:
- """Run under the task context when the call belongs to a task, else the session."""
- if task is not None:
- return self._run_in_task(task, callback, *args, **kwargs)
- return self.host.run_in_session(session.relay_session, callback, *args, **kwargs)
- def _schedule_flush_and_export(self, failure_message: str) -> None:
- """Schedule the process-wide Relay flush without blocking the finishing turn.
- ``flush()`` also waits for every managed tool/LLM call in flight in ANY session, so on a
- turn thread one session finishing stalls behind another session's long tool. One worker
- per runtime (= per profile); triggers landing while it runs coalesce into one more pass.
- """
- from agent.memory_provider import spawn_context_thread
- with self._flush_lock:
- self._flush_pending = True
- if self._flush_thread is not None:
- return
- # Context copy: the export reads the profile's send/consent config via contextvars.
- thread = spawn_context_thread(
- self._run_flush_worker, name="hermes-shared-metrics-flush", args=(failure_message,)
- )
- # The worker's first act is taking this lock, so it cannot clear the slot before it is set.
- thread.start()
- self._flush_thread = thread
- def _run_flush_worker(self, failure_message: str) -> None:
- while True:
- with self._flush_lock:
- if not self._flush_pending:
- # Cleared under the lock that schedulers check: a trigger can never land on a
- # worker that has already decided to exit.
- self._flush_thread = None
- return
- self._flush_pending = False
- self._flush_and_export(failure_message)
- def _flush_and_export(self, failure_message: str) -> None:
- """Flush the Relay subscriber, then export under this profile's scope (its send consent,
- whoever triggered the flush); a failed flush skips the export."""
- try:
- self.relay.subscribers.flush()
- except Exception:
- logger.warning(failure_message, exc_info=True)
- else:
- with self._owner_scope():
- self._export()
- def _abort_session(self, session: _MetricsSession, base_event: dict[str, Any]) -> bool:
- """Mark the session closing and system-abort its open tasks; False if already closing."""
- with session.lock:
- if session.closing:
- return False
- session.closing = True
- for task_id in list(session.tasks):
- self._finish_task(session, task_id, {**base_event, "task_id": task_id})
- self._end_pending_model_calls(session, base_event)
- return True
- def _task_session(
- self, event: dict[str, Any], *, allow_task_id_fallback: bool = False
- ) -> _MetricsSession | None:
- """Owner session by (session, turn), then (session, task), then unique task_id."""
- session_id, task_id = _text(event, "session_id"), _text(event, "task_id")
- if not task_id:
- return None
- with self._task_sessions_lock:
- owner = self._turn_sessions.get(_session_pair(event, "turn_id"))
- if owner is None and session_id:
- owner = self._task_sessions.get((session_id, task_id))
- if owner is not None or not allow_task_id_fallback:
- return owner
- return _sole(
- session for (_, tid), session in self._task_sessions.items() if tid == task_id
- )
- def _remember_turn(
- self, session: _MetricsSession, task: _TaskRun, event: dict[str, Any]
- ) -> None:
- turn_id = _text(event, "turn_id")
- if turn_id:
- task.turn_ids.add(turn_id)
- with self._task_sessions_lock:
- self._turn_sessions[(session.session_id, turn_id)] = session
- @staticmethod
- def _tool_call_identity(event: dict[str, Any]) -> tuple[str, str, str]:
- """Identify one provider-local tool call without exporting its IDs."""
- return _text(event, "api_request_id"), _text(event, "turn_id"), _text(event, "tool_call_id")
- @staticmethod
- def _event_matches_task_turn(task: _TaskRun, event: dict[str, Any]) -> bool:
- """Reject delayed hooks from a prior run that reused the task ID."""
- turn_id = _text(event, "turn_id")
- if not turn_id:
- return True
- return turn_id not in task.retired_turn_ids and (
- not task.turn_ids or turn_id in task.turn_ids
- )
- def _admits(
- self,
- session: _MetricsSession,
- task: _TaskRun,
- event: dict[str, Any],
- *,
- current: bool = False,
- ) -> bool:
- """Whether ``event`` may act on ``task`` (caller holds ``session.lock``).
- Rejects closing sessions and stale turns; with ``current`` also requires ``task`` to
- still be the session's live run for its ID. Admitted events have their turn remembered.
- """
- if (
- session.closing
- or not self._event_matches_task_turn(task, event)
- or (current and session.tasks.get(task.task_id) is not task)
- ):
- return False
- self._remember_turn(session, task, event)
- return True
- def _approval_task(
- self, event: dict[str, Any]
- ) -> tuple[_MetricsSession | None, _TaskRun | None]:
- """Resolve approval correlation without guessing across ambiguous turns."""
- active = relay_runtime.active_turn()
- if active is not None:
- session, task = self._task_pair(
- {**event, "session_id": active.lease.session_id, "task_id": active.task_id}
- )
- if task is not None:
- return session, task
- session, task = self._task_pair(event)
- if task is not None:
- return session, task
- turn_id = _text(event, "turn_id")
- session = None
- if turn_id:
- with self._task_sessions_lock:
- session = _sole(
- candidate
- for (owner_id, candidate_turn_id), candidate in self._turn_sessions.items()
- if candidate_turn_id == turn_id and self._sessions.get(owner_id) is candidate
- )
- if session is None:
- return None, None
- task = _sole(task for task in session.tasks.values() if turn_id in task.turn_ids)
- return (None, None) if task is None else (session, task)
- def _open_tool_call(self, task: _TaskRun, event: dict[str, Any]) -> _ToolCall:
- handle = self._run_in_task(
- task, self.relay.tools.call, contract.TOOL_CALL_SCOPE, {},
- handle=task.handle, metadata=self._event_metadata(),
- )
- return _ToolCall(
- handle, contract.tool_category(event), contract.tool_metric_name(event), monotonic_ns()
- )
- def _finish_tool_call(
- self, task: _TaskRun, tool_call: _ToolCall, event: dict[str, Any]
- ) -> None:
- fields = contract.tool_terminal_fields(
- event, category=tool_call.category, approval_outcome=tool_call.approval_outcome,
- fallback_duration_ms=_elapsed_ms(tool_call.started_ns), tool_name=tool_call.tool_name,
- )
- self._guarded(
- "Hermes shared-metrics tool call close failed",
- lambda: self._run_in_task(
- task, self.relay.tools.call_end, tool_call.handle,
- self.relay.ToolExecutionResult(fields),
- metadata=self._event_metadata(),
- ),
- )
- def _end_pending_tool_calls(
- self, session: _MetricsSession, task: _TaskRun, event: dict[str, Any]
- ) -> None:
- task_outcome, _, _ = contract.task_terminal_state(event)
- status = {"cancelled": "cancelled", "timed_out": "timeout"}.get(task_outcome, "error")
- for key in [key for key in session.tool_calls if key[0] == task.task_id]:
- self._finish_tool_call(task, session.tool_calls.pop(key), {**event, "status": status})
- def _finish_model_call(
- self, session: _MetricsSession, model_call_key: tuple[str, str], outcome: str
- ) -> None:
- model_call = session.model_calls.pop(model_call_key, None)
- if model_call is None:
- return
- error_class = model_call.error_class
- if outcome == "failed" and error_class == "none":
- error_class = "unknown"
- fields = contract.model_route_fields(
- model_call.fields, call_role="primary", outcome=outcome, error_class=error_class,
- ttft_bucket=model_call.ttft_bucket,
- )
- self._guarded(
- "Hermes shared-metrics model call close failed",
- self._run_scoped, session, session.tasks.get(model_call.task_id),
- self.relay.llm.call_end, model_call.handle, fields,
- metadata=self._event_metadata(),
- )
- def _end_pending_model_calls(self, session: _MetricsSession, event: dict[str, Any]) -> None:
- """Close calls that never saw a successful response as failed or cancelled."""
- task_id = _text(event, "task_id")
- cancelled = contract.task_terminal_state(event)[0] == "cancelled"
- pending = [k for k, c in session.model_calls.items() if not task_id or c.task_id == task_id]
- for key in pending:
- self._finish_model_call(session, key, "cancelled" if cancelled else "failed")
- @staticmethod
- def _existing_model_call_key(
- session: _MetricsSession, event: dict[str, Any]
- ) -> tuple[str, str] | None:
- """(task_id, request_id) of an open call; a task-less event may match by request alone."""
- request_id = _text(event, "api_request_id")
- if not request_id:
- return None
- key = (_text(event, "task_id"), request_id)
- if key in session.model_calls or key[0]:
- return key if key in session.model_calls else None
- candidates = [candidate for candidate in session.model_calls if candidate[1] == request_id]
- return candidates[0] if len(candidates) == 1 else None
- def _finish_task(self, session: _MetricsSession, task_id: str, event: dict[str, Any]) -> bool:
- task = session.tasks.get(task_id)
- if task is None:
- return False
- self._end_pending_tool_calls(session, task, event)
- self._end_pending_model_calls(session, {**event, "task_id": task_id})
- fields = contract.task_terminal_fields(
- {**task.start_fields, **event},
- duration_ms=_elapsed_ms(task.started_ns),
- model_call_count=len(task.model_call_ids),
- tool_call_count=len(task.tool_call_ids) + task.unidentified_tool_calls,
- retry_count=task.retry_count,
- )
- if task.cost.user_turn: # background review forks share the session id and never fire pre_llm_call
- self._count_session_turn(session, task, fields["outcome"])
- if not session.closing:
- self._observe_model_turn(session, task, fields)
- try:
- popped = self._guarded(
- "Hermes shared-metrics task close failed",
- self._run_in_task, task, relay_runtime.pop_relay_scope_if_top, self.relay, task.handle,
- output=fields, metadata=self._event_metadata(),
- )
- if popped is False:
- logger.debug("Left shared-metrics task scope %s under a concurrent turn's scope; session close drains it", task_id)
- finally:
- session.tasks.pop(task_id, None)
- session.retired_turn_ids.extend(task.turn_ids)
- with self._task_sessions_lock:
- _forget(self._task_sessions, (session.session_id, task_id), session)
- for turn_id in task.turn_ids:
- _forget(self._turn_sessions, (session.session_id, turn_id), session)
- return True
- @staticmethod
- def _count_session_turn(session: _MetricsSession, task: _TaskRun, outcome: str) -> None:
- if session.start_fields is None:
- session.start_fields = dict(task.start_fields)
- session.first_turn_ns = task.started_ns
- session.turns += 1
- session.failed_turns += outcome == "failed"
- session.api_calls += len(task.model_call_ids)
- session.tool_call_total += len(task.tool_call_ids) + task.unidentified_tool_calls
- session.last_outcome = outcome
- session.last_turn_ns = monotonic_ns()
- def _observe_model_turn(self, session: _MetricsSession, task: _TaskRun, fields: dict[str, str]) -> None:
- """A turn the user saw end (session-close aborts excluded): trailing-failure state and interrupts."""
- route = task.model_route or session.model_state.last_route
- session.model_state.observe_turn(fields["outcome"], route, session.last_turn_ns)
- if task.selected_route is not None and task.cost.user_turn:
- self._with_route_run(session.session_id, session.route_run, lambda run: run.observe(task.selected_route))
- if task.model_route is not None and engagement_.engaged_turn(task.start_fields, task.cost.user_turn):
- self._guarded(
- "Hermes shared-metrics engagement mark failed", self._mark,
- session, None, contract.ENGAGEMENT_TURN_MARK, dict(task.model_route),
- )
- if fields["end_reason"] == "user_cancelled" and route is not None and model_.attended(task.start_fields):
- self._guarded(
- "Hermes shared-metrics friction mark failed", self._mark,
- session, None, contract.MODEL_FRICTION_MARK, model_.friction_fields("interrupt", route),
- )
- if task.cost.user_turn and model_.attended(task.start_fields):
- rows = session.efficiency.finish_turn(
- task.cost, fields, route=route, model_calls=len(task.model_call_ids),
- tool_calls=len(task.tool_call_ids) + task.unidentified_tool_calls,
- )
- self._emit_rows(session, rows)
- def _emit_rows(self, session: _MetricsSession | None, rows: list[tuple[str, dict[str, str]]]) -> None:
- for mark, data in rows:
- if session is None:
- self._guarded("Hermes shared-metrics efficiency mark failed", self.record_process_mark, mark, data)
- else:
- self._guarded("Hermes shared-metrics efficiency mark failed", self._mark, session, None, mark, data)
- def _observe_turn_call(
- self, session: _MetricsSession, task: _TaskRun | None, model_call: _ModelCall, event: dict[str, Any]
- ) -> None:
- """One finished primary call of a user turn: its tokens, and a cold cache read nobody announced."""
- if task is None or not task.cost.user_turn:
- return
- route, usage = model_call.fields, event.get("usage")
- task.cost.add_call(route, usage)
- expected = self._consume_cold(self._cold_key(session.session_id))
- cause = session.efficiency.observe_cache(route, usage, model_call.started_ns, monotonic_ns(), expected=expected)
- if cause is not None:
- self._emit_rows(session, [eff.cache_break_row(cause, route)])
- @staticmethod
- def _cold_key(session_id: str) -> str:
- """The conversation (lineage root survives compression rotation), else the session."""
- return get_conversation_context() or session_id
- def _announce_cold(self, key: str) -> bool:
- """Expect the next read of ``key`` to be cold; False when a break was already announced."""
- with self._cold_lock:
- if key in self._cold_expected:
- return False
- self._cold_expected[key] = None
- while len(self._cold_expected) > 256:
- self._cold_expected.popitem(last=False)
- return True
- def _consume_cold(self, key: str) -> bool:
- with self._cold_lock:
- return self._cold_expected.pop(key, False) is None
- def record_known_cache_break(self, cause: str, route: dict[str, str], session_id: str) -> None:
- """Hermes invalidated the prefix itself: count it, and don't recount the cold read it causes.
- Several causes before one cold read are one break (the first cause names it)."""
- if self._announce_cold(self._cold_key(session_id)):
- self._emit_rows(None, [eff.cache_break_row(cause, route)])
- def record_session_tools(self, session_id: str, agent: Any, tools_for_api: list) -> None:
- session = self._session({"session_id": session_id}) if session_id else None
- if session is None:
- return
- key = (id(getattr(agent, "tools", None)), id(tools_for_api), len(tools_for_api))
- with session.lock:
- if session.closing or session.efficiency.tools_key == key:
- return
- snapshot = eff.tool_snapshot(agent, tools_for_api) # registry + estimator: outside the lock
- with session.lock:
- if session.closing:
- return
- session.efficiency.tools_key = key
- changed = session.efficiency.tools.observe(*snapshot)
- route = model_.model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
- if changed and self._announce_cold(self._cold_key(session_id)):
- self._emit_rows(session, [eff.cache_break_row("toolset_change", route)])
- def rotate_segment(self, old_id: str, new_id: str) -> None:
- """Compression rotated ``old_id`` to ``new_id``: the new id continues the same conversation.
- Only this explicit hand-off links segments (a reset, /new or /branch parent is a new
- conversation). The old segment closes into the lineage now, or once its in-flight turn ends."""
- old = self._session({"session_id": old_id})
- with self._lineage_lock:
- key = self._lineage_of.get(old_id)
- if key is None:
- if old is None:
- return # a segment this process never served: the new id is its own conversation
- key = old_id
- self._lineages[key] = _PeakLineage(
- open={old_id}, run=old.route_run, spent=old.efficiency.spent, connectors=old.connectors)
- self._lineage_of[old_id] = key
- self._lineages[key].open.add(new_id)
- self._lineage_of[new_id] = key
- overflow = list(self._lineages)[:-_MAX_LINEAGES]
- new = self._session({"session_id": new_id})
- if new is not None:
- self._share_spent(new)
- if old is not None:
- with old.lock:
- old.retiring = not old.closing
- idle = old.retiring and not old.tasks
- if idle:
- self.close_session({"session_id": old_id})
- for stale in overflow:
- self._flush_lineage(stale)
- def _lineage_spent(self, session_id: str) -> deque | None:
- with self._lineage_lock:
- lineage = self._lineages.get(self._lineage_of.get(session_id, ""))
- return lineage.spent if lineage is not None else None
- def _share_spent(self, session: _MetricsSession) -> None:
- """A rotated-to segment's /undo and /retry reach the turns the earlier segments spent, and its
- connector chain continues the earlier segments' searches and descriptions."""
- spent = self._lineage_spent(session.session_id)
- if spent is not None:
- session.efficiency.spent = spent
- with self._lineage_lock:
- lineage = self._lineages.get(self._lineage_of.get(session.session_id, ""))
- if lineage is not None and lineage.connectors is not None:
- session.connectors = lineage.connectors
- def _with_route_run(
- self, session_id: str, own: engagement_.RouteRun | None, update: Callable[[engagement_.RouteRun], Any],
- ) -> Any:
- """Apply ``update`` to the conversation's model-turn run (the lineage's, shared by its segments)."""
- with self._lineage_lock:
- lineage = self._lineages.get(self._lineage_of.get(session_id, ""))
- run = lineage.run if lineage is not None else own
- return update(run) if run is not None else None
- def _close_segment(
- self, session_id: str, state: model_.ModelSessionState | None, tally: engagement_.SessionTally | None,
- tools: eff.ToolUsage,
- ) -> tuple[model_.ModelSessionState | None, engagement_.SessionTally | None, eff.ToolUsage | None]:
- """This segment's context peak, tally and tool usage, or, for a lineage segment, the
- conversation's merged ones once its last open segment closes (None before that)."""
- with self._lineage_lock:
- key = self._lineage_of.pop(session_id, None)
- lineage = self._lineages.get(key) if key is not None else None
- if lineage is None:
- return state, tally, tools
- lineage.open.discard(session_id)
- if state is not None and tally is not None:
- _absorb_peak(lineage.peak, state)
- lineage.tally.absorb(tally)
- lineage.tools.absorb(tools)
- if lineage.open:
- return None, None, None
- del self._lineages[key]
- return lineage.peak, lineage.tally, lineage.tools
- def _close_unseen_segment(self, session_id: str) -> None:
- """A rotated-to segment closed (or the process ended) before it served a turn here."""
- if session_id:
- self._emit_summary_rows(None, *self._close_segment(session_id, None, None, eff.ToolUsage()))
- def _flush_lineage(self, key: str) -> None:
- """Backstop for a conversation a surface never closed: report what it holds, detach its segments."""
- with self._lineage_lock:
- lineage = self._lineages.pop(key, None)
- for segment in lineage.open if lineage is not None else ():
- self._lineage_of.pop(segment, None)
- if lineage is not None:
- self._emit_summary_rows(None, lineage.peak, lineage.tally, lineage.tools)
- def _emit_summary_rows(
- self, session: _MetricsSession | None, peak: model_.ModelSessionState | None,
- tally: engagement_.SessionTally | None, tools: eff.ToolUsage | None, *extra: tuple[str, Any],
- ) -> None:
- marks = [
- (contract.CONTEXT_PEAK_MARK, peak.context_peak_fields() if peak is not None else None),
- (contract.SESSION_MARK, tally.fields() if tally is not None else None),
- *(tools.rows() if tools is not None else ()),
- *extra,
- ]
- self._emit_rows(session, [(mark, data) for mark, data in marks if data is not None])
- def _emit_session_summary(self, session: _MetricsSession) -> None:
- """One hermes.session.count and context-peak row per closed top-level conversation (delegated
- children excluded; compression-rotated segments merged), its tool usage, plus a quick-abandon friction."""
- start = session.start_fields
- counted = bool(session.turns) and start is not None and start.get("entrypoint") != "delegated"
- tally = engagement_.SessionTally(
- session.start_fields, session.turns, session.failed_turns, session.last_outcome,
- session.first_turn_ns, session.last_turn_ns, session.api_calls, session.tool_call_total, session.replies,
- ) if counted else None
- tools = session.efficiency.tools if counted and model_.attended(start) else eff.ToolUsage()
- tools.surface = (start or {}).get("execution_surface", "unknown")
- merged = self._close_segment(session.session_id, session.model_state if counted else None, tally, tools)
- # A segment compression retired is not the user walking away.
- abandoned = session.model_state.quick_abandon_route(monotonic_ns()) if counted and not session.retiring else None
- friction = (
- [(contract.MODEL_FRICTION_MARK, model_.friction_fields("quick_abandon", abandoned))]
- if abandoned is not None and model_.attended(start) else []
- )
- self._emit_summary_rows(session, *merged, *friction)
- def record_switch_after(self, session_id: str) -> None:
- """A counted /model switch: how many turns the model being left served in this conversation."""
- session = self._session({"session_id": session_id})
- with session.lock if session is not None else contextlib.nullcontext():
- taken = self._with_route_run(
- session_id, session.route_run if session is not None else None, lambda run: run.take()
- )
- if taken is not None:
- self.record_process_mark(contract.MODEL_SWITCH_AFTER_MARK, engagement_.switch_after_fields(*taken))
- def record_friction(self, signal: str, session_id: str, fallback_route: dict[str, str], turns: int = 1) -> None:
- """A user friction action, attributed to the session's last primary model when known; an
- /undo or /retry also counts the tokens of the turns it threw away."""
- session = self._session({"session_id": session_id}) if session_id else None
- route, wasted = None, []
- # A session this process never saw (restart, remote host) still counts, with tokens unknown.
- with session.lock if session is not None else contextlib.nullcontext():
- if session is not None:
- route = session.model_state.last_route
- if signal in contract.WASTE_REASONS:
- efficiency = session.efficiency if session is not None else eff.SessionEfficiency()
- if session is None and (spent := self._lineage_spent(session_id)) is not None:
- efficiency.spent = spent # a rotated-to segment that has not served a turn yet
- wasted = efficiency.discard_turns(signal, turns, route or fallback_route)
- data = model_.friction_fields(signal, route or fallback_route)
- if data is not None:
- self.record_process_mark(contract.MODEL_FRICTION_MARK, data)
- self._emit_rows(None, wasted)
- def record_process_mark(self, mark: str, data: dict[str, Any]) -> None:
- """A process-level fact with no owning task (setup, installs, commands, switches)."""
- # No flush here: the next task close or the atexit shutdown drains the subscriber.
- self._with_scope_stack(self.relay.scope.event, mark, data=data, metadata=self._event_metadata())
- def record_process_marks_saved(self, marks: list[tuple[str, dict[str, Any]]]) -> int:
- """Emit ``marks`` and wait for the store; how many settled without a store error."""
- ticket = uuid.uuid4().hex
- metadata = {**self._event_metadata(), contract.COMMIT_TICKET_KEY: ticket}
- for mark, data in marks:
- self._with_scope_stack(self.relay.scope.event, mark, data=data, metadata=metadata)
- self.relay.subscribers.flush()
- return self.subscriber.take_saved(ticket)
- def record_auxiliary_tokens(self, event: dict[str, Any]) -> None:
- tokens = fields_.model_token_fields(
- event.get("usage"), call_role="auxiliary", aux_task=event.get("aux_task"),
- model=event.get("response_model") or event.get("model"), provider=event.get("provider"),
- )
- if tokens is not None:
- self._with_scope_stack(
- self.relay.scope.event, contract.MODEL_TOKENS_MARK, data=tokens,
- metadata=self._event_metadata(),
- )
- def _export(self) -> None:
- exported = self._safe(self.subscriber.store.create_and_export_package_if_due)
- # Sending must never delay the caller: _export runs on finish_task, the user's
- # interactive path. The thread is about latency, not correctness.
- if exported is not None:
- self._safe(self._send_exported_packages)
- def _send_exported_packages(self) -> None:
- try:
- resolved = _resolved_send_config()
- except Exception:
- logger.debug("Unable to read shared-metrics send policy", exc_info=True)
- return
- # Observe the consent EDGE before deciding whether to send: the dominant revocation
- # case is "sending turned off while no pass is running", invisible to the send loop.
- # Failures never break the export hook but log at warning (privacy-relevant).
- self._guarded(
- "Unable to record a shared-metrics consent transition",
- _reconcile_store_consent, self.subscriber.store, resolved.send,
- )
- if not resolved.send:
- return
- with self._send_lock:
- # One in-flight pass per process; the next hook fire picks up what is pending.
- if self._send_thread is None or not self._send_thread.is_alive():
- from agent.memory_provider import spawn_context_thread
- # Context copy: the pass re-checks consent (still_consented) in THIS profile's scope.
- self._send_thread = spawn_context_thread(
- self._run_send_pass, name="hermes-shared-metrics-send", args=(resolved.endpoint,)
- )
- self._send_thread.start()
- def _run_send_pass(self, endpoint: str) -> None:
- from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender
- def still_consented() -> bool:
- """Re-read consent so revoking `send` stops an in-flight pass."""
- resolved = _resolved_send_config()
- return resolved.send and resolved.endpoint == endpoint
- sender = SharedMetricsSender(self.subscriber.store, endpoint, consent_check=still_consented)
- self._guarded("Shared-metrics send pass failed", sender.send_pending)
- def _event_metadata(self) -> dict[str, str]:
- return {
- contract.SCHEMA_KEY: contract.SCHEMA_VERSION,
- relay_runtime.RUNTIME_INSTANCE_KEY: self.host.runtime_id,
- }
- @staticmethod
- def _guarded(message: str, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
- """Run ``callback``; log-and-swallow any exception, returning None."""
- try:
- return callback(*args, **kwargs)
- except Exception:
- logger.warning(message, exc_info=True)
- return None
- @classmethod
- def _safe(cls, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
- return cls._guarded("Hermes shared metrics operation failed", callback, *args, **kwargs)
- def _raw_config() -> dict[str, Any]:
- """Read-only config snapshot (lazy import: tests patch ``hermes_cli.config``).
- Collection consent is profile-owned: managed overlays cannot opt a profile in or out.
- The read-only path matters because this gate runs 2-3x per agent turn and the mutable
- read_raw_config() paid a full config deepcopy on every call.
- """
- from hermes_cli.config import read_raw_config_readonly
- return read_raw_config_readonly() or {}
- def _resolved_send_config():
- """Resolve the opt-in send policy from the read-only config snapshot."""
- from hermes_cli.observability.shared_metrics_send_config import resolve_send_config
- return resolve_send_config(_raw_config())
- def _reconcile_store_consent(store: SharedMetricsStore, send_enabled: bool) -> None:
- from hermes_cli.observability.shared_metrics_sender import reconcile_send_consent
- from hermes_cli.sqlite_util import write_txn
- with store._connection() as connection:
- with write_txn(connection):
- reconcile_send_consent(connection, send_enabled)
- def enabled() -> bool:
- """Return the shared-metrics policy for the active Hermes profile."""
- profile_key = relay_runtime.current_profile_key()
- try:
- config: Any = _raw_config()
- except Exception:
- logger.debug("Unable to read Hermes shared-metrics policy", exc_info=True)
- config = None
- for key in ("telemetry", "shared_metrics"):
- config = config.get(key) if isinstance(config, dict) else None
- if isinstance(config, dict) and config.get("enabled") is True:
- return True
- if profile_key not in _RUNTIMES: # the opted-out hot path: nothing to tear down, no lock
- return False
- with _RUNTIME_LOCK:
- runtime = _RUNTIMES.pop(profile_key, None)
- if isinstance(runtime, _Runtime):
- runtime.deactivate()
- return False
- def handles_hook(hook_name: str) -> bool:
- return hook_name in HANDLED_HOOKS and enabled()
- _consent_reconcile_done = False
- def _reconcile_send_consent_once() -> None:
- """Reconcile consent windows with config, once per process.
- Runs BEFORE and INDEPENDENT of the collection gate, so a user with ``enabled: false``
- still gets send-consent windows reconciled. Skipped only when there is no store on disk
- AND consent is off: nothing to protect, and creating ``~/.hermes/telemetry`` for every
- fully-disabled user would be the wrong behaviour change.
- """
- global _consent_reconcile_done
- if _consent_reconcile_done:
- return
- _consent_reconcile_done = True
- try:
- # Lazy: tests patch ``shared_metrics.SharedMetricsStore`` at its origin.
- from hermes_cli.observability.shared_metrics import SharedMetricsStore
- from hermes_constants import get_hermes_home
- resolved = _resolved_send_config()
- # Probe WITHOUT constructing a store: the constructor creates the directory and
- # schema as a side effect, which would make the skip below dead code.
- default_path = get_hermes_home() / "telemetry" / "shared_metrics" / "metrics.sqlite3"
- if not resolved.send and not default_path.exists():
- return
- _reconcile_store_consent(SharedMetricsStore(), resolved.send)
- except Exception:
- logger.warning("Unable to reconcile shared-metrics send consent", exc_info=True)
- # Hooks whose only effect is a process mark (no task, no scope): gated like record_process_mark.
- _PROCESS_MARK_HOOKS = frozenset({"post_auxiliary_call"})
- def _process_marks_admitted() -> bool:
- """Whether the inherited turn lets a process-level mark through.
- A process mark has no owning task and opens no scope, so a turn that has CLOSED does not gate
- it: background work spawned under the turn (a turn-end compaction pass, title generation)
- inherits that turn through its copied context and still reports after it ends. A turn Relay
- skipped (a concurrent turn on the session) keeps gating, open or closed."""
- turn = relay_runtime.current_turn()
- return turn is None or turn.relay_enabled
- def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
- """Project one Hermes lifecycle event into the core Relay integration."""
- _reconcile_send_consent_once()
- if not handles_hook(hook_name):
- return
- if not (
- _process_marks_admitted() if hook_name in _PROCESS_MARK_HOOKS
- else relay_runtime.relay_instrumentation_enabled()
- ):
- return
- runtime = _get_runtime()
- if runtime is None:
- return
- try:
- _HOOK_HANDLERS[hook_name](runtime, kwargs)
- except Exception:
- logger.warning("Hermes shared metrics hook failed: %s", hook_name, exc_info=True)
- def _with_runtime_toolset(event: dict[str, Any]) -> dict[str, Any]:
- """Attach the toolset already declared by Hermes's runtime registry."""
- tool_name = _text(event, "tool_name")
- if event.get("toolset") or not tool_name:
- return event
- try:
- from model_tools import get_toolset_for_tool
- toolset = get_toolset_for_tool(tool_name)
- except Exception:
- toolset = None
- return {**event, "toolset": toolset or "other"}
- def _close_child_session(runtime: _Runtime, kwargs: dict[str, Any]) -> None:
- child_session_id = _text(kwargs, "child_session_id")
- if child_session_id:
- runtime.close_session({"session_id": child_session_id})
- _HOOK_HANDLERS: dict[str, Callable[[_Runtime, dict[str, Any]], Any]] = {
- "on_session_start": lambda rt, kw: rt.record_client_active(kw),
- "pre_llm_call": lambda rt, kw: rt.start_user_turn(kw),
- "pre_api_request": lambda rt, kw: rt.start_model_call(kw),
- "pre_tool_call": lambda rt, kw: rt.start_tool_call(_with_runtime_toolset(kw)),
- "post_tool_call": lambda rt, kw: rt.record_tool_call(_with_runtime_toolset(kw)),
- "post_approval_response": lambda rt, kw: rt.record_approval(kw),
- "on_skill_lifecycle": lambda rt, kw: rt.record_skill_lifecycle(kw),
- "post_api_request": lambda rt, kw: rt.update_model_call(kw, finish=True),
- "post_auxiliary_call": lambda rt, kw: rt.record_auxiliary_tokens(kw),
- "api_request_error": lambda rt, kw: rt.update_model_call(kw, finish=False),
- "on_session_end": lambda rt, kw: rt.finish_task(kw),
- "subagent_stop": _close_child_session,
- "on_session_finalize": lambda rt, kw: rt.close_session(kw),
- "on_session_reset": lambda rt, kw: rt.close_session(kw),
- }
- HANDLED_HOOKS = frozenset(_HOOK_HANDLERS)
- def _prepare_core_session(host: relay_runtime.RelayRuntime, context: dict[str, Any]) -> None:
- """Prepare the profile subscriber before the coordinator opens a scope."""
- del context
- if host.profile_key == relay_runtime.current_profile_key() and enabled():
- _get_runtime(retry_failed=True, host=host)
- def start_task_run(
- *, session_id: str, task_id: str, platform: str, parent_session_id: str = ""
- ) -> None:
- """Start task metrics at the outer Hermes execution boundary."""
- _run_task_hook(
- "start_task", retry_failed=True, session_id=session_id, task_id=task_id,
- platform=platform, parent_session_id=parent_session_id,
- )
- def shutdown_runtimes() -> None:
- """Run each live runtime's exit flush now, for a process that leaves through ``os._exit`` (the
- ``hermes -z`` exit) and so never reaches the atexit hook that closes sessions and writes
- ``task_run.finished`` / ``session.count``. Each runtime binds its own profile while it does."""
- with _RUNTIME_LOCK:
- runtimes = [runtime for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]
- for runtime in runtimes:
- runtime._safe(runtime.shutdown)
- def close_session_run(session_id: str) -> None:
- """Emit the summary of a session a surface retired without a finalize hook (gateway
- auto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""
- if not session_id or not enabled():
- return
- runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
- if isinstance(runtime, _Runtime):
- runtime._safe(runtime.close_session, {"session_id": session_id})
- def rotate_segment(old_session_id: str, new_session_id: str) -> None:
- """Compression rotated a conversation's session id (the one seam that continues a conversation
- under a new id). Never creates a runtime: a runtime that does not exist served no segment."""
- if not old_session_id or not new_session_id or old_session_id == new_session_id or not enabled():
- return
- runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
- if isinstance(runtime, _Runtime):
- runtime._safe(runtime.rotate_segment, old_session_id, new_session_id)
- def finish_task_run(
- *, session_id: str, task_id: str, platform: str,
- result: dict[str, Any] | None = None, error: BaseException | None = None,
- ) -> None:
- """Finish task metrics for every return or exception path."""
- _run_task_hook(
- "finish_task", session_id=session_id, task_id=task_id, platform=platform,
- **_terminal_flags(result, error),
- )
- def record_process_mark(mark: str, data: dict[str, Any]) -> None:
- """Emit one process-level decision mark when shared metrics are on (never raises)."""
- if not enabled() or not _process_marks_admitted():
- return
- runtime = _get_runtime(retry_failed=True)
- if runtime is not None:
- runtime._safe(runtime.record_process_mark, mark, data)
- def record_process_marks_saved(marks: list[tuple[str, dict[str, Any]]]) -> int:
- """``record_process_mark`` for facts recovered from a file the caller deletes only once they are
- saved (a busy store would otherwise lose them for good). How many rows are settled: saved, rejected
- by the contract, or nothing to record because collection is off."""
- if not enabled() or not _process_marks_admitted():
- return len(marks)
- runtime = _get_runtime(retry_failed=True)
- return (runtime._safe(runtime.record_process_marks_saved, marks) or 0) if runtime is not None else 0
- def record_session_friction(signal: str, session_id: str, fallback_route: dict[str, str], turns: int = 1) -> None:
- """Record one friction signal in the active profile's runtime (callers already checked enabled())."""
- if not relay_runtime.relay_instrumentation_enabled():
- return
- runtime = _get_runtime(retry_failed=True)
- if runtime is not None:
- runtime._safe(runtime.record_friction, signal, session_id, fallback_route, turns)
- def record_session_tools(session_id: str, agent: Any, tools_for_api: list) -> None:
- """The tools one primary request sent (caller checked enabled()); never creates a runtime."""
- runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
- if isinstance(runtime, _Runtime):
- runtime._safe(runtime.record_session_tools, session_id, agent, tools_for_api)
- def record_known_cache_break(cause: str, route: dict[str, str], session_id: str) -> None:
- """A prompt-cache break Hermes caused (caller checked enabled())."""
- if not relay_runtime.relay_instrumentation_enabled():
- return
- runtime = _get_runtime(retry_failed=True)
- if runtime is not None:
- runtime._safe(runtime.record_known_cache_break, cause, route, session_id)
- def record_model_switch_after(session_id: str) -> None:
- """Count turns-before-switch for a session this process served; never creates a runtime (a
- runtime that does not exist saw no turns). Callers are already in the owning profile's scope."""
- if not session_id or not enabled() or not relay_runtime.relay_instrumentation_enabled():
- return
- runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
- if isinstance(runtime, _Runtime):
- runtime._safe(runtime.record_switch_after, session_id)
- def _run_task_hook(method: str, *, retry_failed: bool = False, **event: Any) -> None:
- if not enabled():
- return
- runtime = _get_runtime(retry_failed=retry_failed)
- if runtime is not None:
- runtime._safe(getattr(runtime, method), event)
- def _terminal_flags(result: dict[str, Any] | None, error: BaseException | None) -> dict[str, Any]:
- """Bounded completed/failed/interrupted/turn_exit_reason for a task's return or raise."""
- if error is not None:
- interrupted = (
- isinstance(error, (KeyboardInterrupt, InterruptedError))
- or type(error).__name__ == "CancelledError"
- )
- if interrupted:
- reason = "interrupted_by_user"
- else:
- reason = "timed_out" if isinstance(error, TimeoutError) else "system_aborted"
- return {
- "completed": False, "failed": not interrupted, "interrupted": interrupted,
- "turn_exit_reason": reason, "failure_class": "exception",
- }
- terminal = result if isinstance(result, dict) else {}
- failed = terminal.get("failed") is True
- reason = str(terminal.get("turn_exit_reason") or terminal.get("failure_reason") or "")
- return {
- "completed": terminal.get("completed") is True,
- "failed": failed,
- "interrupted": terminal.get("interrupted") is True,
- "turn_exit_reason": reason or ("failed" if failed else "unknown"),
- # Classified in the contract against a closed set; the raw string never leaves.
- "failure_reason": str(terminal.get("failure_reason") or ""),
- }
- def _get_runtime(
- *, retry_failed: bool = False, host: relay_runtime.RelayRuntime | None = None
- ) -> _Runtime | None:
- profile_key = relay_runtime.current_profile_key()
- with _RUNTIME_LOCK:
- runtime = _RUNTIMES.get(profile_key)
- if isinstance(runtime, _Runtime):
- if host is None or runtime.host is host:
- return runtime
- runtime.deactivate()
- elif runtime is _RUNTIME_FAILED and not retry_failed:
- return None
- try:
- _RUNTIMES[profile_key] = runtime = _Runtime(host=host)
- except Exception:
- logger.warning("Hermes shared metrics initialization failed", exc_info=True)
- _RUNTIMES[profile_key] = _RUNTIME_FAILED
- return None
- return runtime
- relay_runtime.SESSION_COORDINATOR.register_session_initializer(
- SUBSCRIBER_NAME, _prepare_core_session
- )
- def _reset_for_tests() -> None:
- """Reset all profile-scoped shared-metrics state for isolated tests."""
- with _RUNTIME_LOCK:
- runtimes = list(_RUNTIMES.values())
- _RUNTIMES.clear()
- for runtime in runtimes:
- if isinstance(runtime, _Runtime):
- runtime.shutdown()
|