relay_shared_metrics.py 77 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675
  1. """Direct NeMo Relay integration for Hermes shared client metrics."""
  2. from __future__ import annotations
  3. import atexit
  4. import contextlib
  5. import contextvars
  6. import logging
  7. import threading
  8. import uuid
  9. from collections import OrderedDict, deque
  10. from dataclasses import dataclass, field
  11. from time import monotonic_ns
  12. from typing import Any, Callable, Iterator
  13. from agent import relay_runtime
  14. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  15. from agent.portal_tags import get_conversation_context
  16. from hermes_cli.version_info import get_version_info
  17. from .shared_metrics import SharedMetricsStore
  18. from . import shared_metrics_contract as contract
  19. from . import shared_metrics_connectors as connectors_
  20. from . import shared_metrics_efficiency as eff
  21. from . import shared_metrics_engagement as engagement_
  22. from . import shared_metrics_fields as fields_
  23. from . import shared_metrics_model as model_
  24. from .shared_metrics_contract import MODEL_CALL_SCOPE, SUBSCRIBER_NAME, TASK_SCOPE
  25. from .shared_metrics_subscriber import SharedMetricsSubscriber
  26. logger = logging.getLogger(__name__)
  27. _RUNTIME_FAILED = object()
  28. _RUNTIMES: dict[str, _Runtime | object] = {}
  29. _RUNTIME_LOCK = threading.RLock()
  30. _ABORTED = {"failed": True, "turn_exit_reason": "system_aborted"}
  31. # The store latch allows one snapshot per 24h; re-reading it hourly keeps task starts cheap.
  32. _SNAPSHOT_RECHECK_NS = 3_600 * 1_000_000_000
  33. def _text(event: dict[str, Any], key: str) -> str:
  34. return str(event.get(key) or "")
  35. def _session_pair(event: dict[str, Any], key: str) -> tuple[str, str] | None:
  36. """(session_id, event[key]) when both are non-empty."""
  37. session_id, value = _text(event, "session_id"), _text(event, key)
  38. return (session_id, value) if session_id and value else None
  39. def _retry_ordinal(event: dict[str, Any]) -> int:
  40. """Hermes's provider-local retry ordinal; 0 when absent or malformed."""
  41. value = event.get("retry_count")
  42. return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0
  43. def _forget(index: dict[Any, _MetricsSession], key: Any, owner: _MetricsSession) -> None:
  44. """Drop ``key`` from ``index`` only while it still points at ``owner``."""
  45. if index.get(key) is owner:
  46. index.pop(key, None)
  47. def _task_parent_handle(session: _MetricsSession, task_id: str) -> Any:
  48. """The active turn's handle when it owns this exact task, else the session handle."""
  49. active_turn = relay_runtime.active_turn(session.session_id)
  50. if (
  51. active_turn is not None
  52. and active_turn.lease.session_id == session.session_id
  53. and active_turn.task_id == task_id
  54. and active_turn.handle is not None
  55. ):
  56. return active_turn.handle
  57. return session.relay_session.handle
  58. # Session sources a Hermes dispatcher stamps on the finite CLI child it spawns: a kanban worker
  59. # (``kanban_db_dispatch``) and an A2A forward (``--source a2a``). Nobody is at either keyboard.
  60. _DISPATCHED_SOURCES = frozenset({"a2a", "kanban"})
  61. def _with_launch_entrypoint(event: dict[str, Any]) -> dict[str, Any]:
  62. """Declare the entrypoint of a finite CLI run (``hermes -z``, ``chat -q`` off a TTY, ``-Q``): the
  63. ``HERMES_SINGLE_QUERY_SESSION`` marker those paths set (the same one ``cache_ttl: auto`` and the session
  64. source read). ``background`` for a dispatcher-spawned child, else ``one_shot``; the REPL, other surfaces,
  65. delegated children and an already-declared entrypoint are untouched."""
  66. if (event.get("entrypoint") or event.get("parent_task_id") or event.get("parent_session_id")
  67. or contract.execution_surface(event) != "cli"):
  68. return event
  69. from agent.oneshot_footprint import is_single_query_session
  70. if not is_single_query_session():
  71. return event
  72. from gateway.session_context import get_session_env
  73. source = str(get_session_env("HERMES_SESSION_SOURCE", "") or "").strip().lower()
  74. return {**event, "entrypoint": "background" if source in _DISPATCHED_SOURCES else "one_shot"}
  75. def _elapsed_ms(started_ns: int) -> int:
  76. return max(0, (monotonic_ns() - started_ns) // 1_000_000)
  77. def _scope_handle(session: _MetricsSession, task: _TaskRun | None) -> Any:
  78. return task.handle if task is not None else session.relay_session.handle
  79. def _sole(items: Any) -> Any:
  80. """The single distinct element of ``items`` (identity-deduplicated), else None."""
  81. unique = {id(item): item for item in items}
  82. return next(iter(unique.values())) if len(unique) == 1 else None
  83. def _identities_compatible(candidate: tuple[str, str, str], observed: tuple[str, str, str]) -> bool:
  84. """Match partial hook context without crossing known call boundaries."""
  85. if not observed[2] or candidate[2] != observed[2]:
  86. return False
  87. return all(
  88. not left or not right or left == right
  89. for left, right in zip(candidate[:2], observed[:2], strict=True)
  90. )
  91. def _compatible_tool_call_keys(
  92. session: _MetricsSession, task_id: str, identity: tuple[str, str, str]
  93. ) -> list[tuple[str, str, str, str]]:
  94. return [
  95. key
  96. for key in session.tool_calls
  97. if key[0] == task_id and _identities_compatible(key[1:], identity)
  98. ]
  99. @dataclass
  100. class _ModelCall:
  101. handle: Any
  102. task_id: str
  103. fields: dict[str, str]
  104. error_class: str = "none"
  105. ttft_bucket: str = "unknown"
  106. started_ns: int = field(default_factory=monotonic_ns)
  107. @dataclass
  108. class _ToolCall:
  109. handle: Any
  110. category: str
  111. tool_name: str
  112. started_ns: int
  113. approval_outcome: str = "not_required"
  114. @dataclass
  115. class _TaskRun:
  116. task_id: str
  117. handle: Any
  118. context: contextvars.Context
  119. started_ns: int
  120. start_fields: dict[str, str]
  121. model_call_ids: set[str] = field(default_factory=set)
  122. tool_call_ids: set[tuple[str, str, str]] = field(default_factory=set)
  123. turn_ids: set[str] = field(default_factory=set)
  124. retired_turn_ids: frozenset[str] = field(default_factory=frozenset)
  125. completed_tool_call_ids: set[tuple[str, str, str]] = field(default_factory=set)
  126. unidentified_tool_calls: int = 0
  127. retry_count: int = 0
  128. model_route: dict[str, str] | None = None
  129. # The route the turn was sent on (its first request): provider failover does not change it.
  130. selected_route: dict[str, str] | None = None
  131. cost: eff.TurnCost = field(default_factory=eff.TurnCost)
  132. @dataclass
  133. class _MetricsSession:
  134. session_id: str
  135. relay_session: relay_runtime.RelaySession
  136. lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
  137. closing: bool = False
  138. model_calls: dict[tuple[str, str], _ModelCall] = field(default_factory=dict)
  139. tasks: dict[str, _TaskRun] = field(default_factory=dict)
  140. tool_calls: dict[tuple[str, str, str, str], _ToolCall] = field(default_factory=dict)
  141. retired_turn_ids: deque[str] = field(default_factory=lambda: deque(maxlen=256))
  142. # Session-level aggregation, emitted once as hermes.session.count when the session closes.
  143. start_fields: dict[str, str] | None = None
  144. turns: int = 0
  145. failed_turns: int = 0
  146. last_outcome: str = "unknown"
  147. first_turn_ns: int = 0
  148. last_turn_ns: int = 0
  149. model_state: model_.ModelSessionState = field(default_factory=model_.ModelSessionState)
  150. # Compression rotated this id away: close once its in-flight turn ends.
  151. retiring: bool = False
  152. efficiency: eff.SessionEfficiency = field(default_factory=eff.SessionEfficiency)
  153. connectors: connectors_.ConnectorSessionState = field(default_factory=connectors_.ConnectorSessionState)
  154. route_run: engagement_.RouteRun = field(default_factory=engagement_.RouteRun)
  155. api_calls: int = 0
  156. tool_call_total: int = 0
  157. replies: int = 0
  158. @dataclass
  159. class _PeakLineage:
  160. """Open segments of one conversation (compression rotates the session id), their merged peak and
  161. session tally, the conversation's current-model turn run and its spent user turns."""
  162. open: set[str] = field(default_factory=set)
  163. peak: model_.ModelSessionState = field(default_factory=model_.ModelSessionState)
  164. tools: eff.ToolUsage = field(default_factory=eff.ToolUsage)
  165. tally: engagement_.SessionTally = field(default_factory=engagement_.SessionTally)
  166. run: engagement_.RouteRun = field(default_factory=engagement_.RouteRun)
  167. spent: deque | None = None
  168. connectors: connectors_.ConnectorSessionState | None = None
  169. # Backstop: conversations whose rotated-to segment a surface never closed are flushed past this many.
  170. _MAX_LINEAGES = 512
  171. def _absorb_peak(into: model_.ModelSessionState, segment: model_.ModelSessionState) -> None:
  172. """Merge one closed segment's context peak (fullest fill wins, a limit hit anywhere sticks)."""
  173. if segment.peak_route is None:
  174. return
  175. into.limit_hit = into.limit_hit or segment.limit_hit
  176. fuller = segment.peak_window is not None and (
  177. into.peak_window is None or segment.peak_tokens / segment.peak_window > into.peak_tokens / into.peak_window
  178. )
  179. if into.peak_route is None or fuller:
  180. into.peak_route, into.peak_tokens, into.peak_window = segment.peak_route, segment.peak_tokens, segment.peak_window
  181. class _Runtime:
  182. """Own shared-metrics state layered on the Hermes core Relay host."""
  183. def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None:
  184. resolved_host = host or relay_runtime.get_runtime()
  185. if resolved_host is None:
  186. raise RuntimeError("Hermes core Relay runtime is unavailable")
  187. self.host: relay_runtime.RelayRuntime = resolved_host
  188. self.relay = self.host.relay
  189. self._active = True
  190. self._sessions: dict[str, _MetricsSession] = {}
  191. self._task_sessions: dict[tuple[str, str], _MetricsSession] = {}
  192. self._turn_sessions: dict[tuple[str, str], _MetricsSession] = {}
  193. self._sessions_lock = threading.RLock()
  194. # Leaf lock (nothing is acquired under it): taken while a session.lock is held. Lineages are
  195. # keyed by the conversation's first segment id; _lineage_of maps every open segment id to it.
  196. self._lineages: dict[str, _PeakLineage] = {}
  197. self._lineage_of: dict[str, str] = {}
  198. self._lineage_lock = threading.Lock()
  199. # Leaf lock: conversations whose next cold cache read Hermes already announced.
  200. self._cold_expected: OrderedDict[str, None] = OrderedDict()
  201. self._cold_lock = threading.Lock()
  202. self._task_creation_lock = threading.RLock()
  203. self._task_sessions_lock = threading.RLock()
  204. # Guards the opt-in send pass: at most one in flight per process.
  205. self._send_lock = threading.RLock()
  206. self._send_thread: threading.Thread | None = None
  207. # Flushing is a process-wide Relay barrier. Keep it off interactive turn threads while
  208. # coalescing concurrent finish hooks into one worker.
  209. self._flush_lock = threading.RLock()
  210. self._flush_pending = False
  211. self._flush_thread: threading.Thread | None = None
  212. self._snapshot_checked_ns: int | None = None
  213. self._subscriber_name = f"{SUBSCRIBER_NAME}.{self.host.runtime_id}"
  214. self.subscriber = SharedMetricsSubscriber(
  215. SharedMetricsStore(), get_version_info().base_version, runtime_id=self.host.runtime_id
  216. )
  217. self.relay.subscribers.register(self._subscriber_name, self.subscriber)
  218. self.host.retain_managed_execution(self._subscriber_name)
  219. self._registered = True
  220. atexit.register(self.shutdown)
  221. def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None:
  222. session_id = _text(event, "session_id")
  223. if not session_id:
  224. return None
  225. with self._sessions_lock:
  226. if not self._active:
  227. return None
  228. relay_session = self.host.ensure_session(event)
  229. if relay_session is None:
  230. return None
  231. session = self._sessions.get(session_id)
  232. if session is None:
  233. session = _MetricsSession(session_id=session_id, relay_session=relay_session)
  234. self._sessions[session_id] = session
  235. self._share_spent(session)
  236. with session.lock:
  237. return None if session.closing else session
  238. def record_client_active(self, event: dict[str, Any]) -> None:
  239. """Emit one payload-free activation attempt under the session scope."""
  240. session = self.ensure_session(event)
  241. if session is not None:
  242. self._emit_client_active(session)
  243. def _emit_client_active(self, session: _MetricsSession) -> None:
  244. with session.lock:
  245. if not session.closing:
  246. self._mark(session, None, contract.CLIENT_ACTIVE_MARK, {})
  247. self._safe(self._emit_install_snapshot_if_due, session)
  248. def _emit_install_snapshot_if_due(self, session: _MetricsSession) -> None:
  249. now = monotonic_ns()
  250. if self._snapshot_checked_ns is not None and now - self._snapshot_checked_ns < _SNAPSHOT_RECHECK_NS:
  251. return
  252. self._snapshot_checked_ns = now
  253. if not self.subscriber.store.install_snapshot_due():
  254. return
  255. from .shared_metrics_snapshot import collect_install_snapshot
  256. fields = collect_install_snapshot(_raw_config())
  257. self._mark(session, None, contract.INSTALL_SNAPSHOT_MARK, fields)
  258. def _mark(
  259. self, session: _MetricsSession, task: _TaskRun | None, name: str, data: dict[str, str]
  260. ) -> None:
  261. """Emit one Relay mark under the task scope when given, else the session scope."""
  262. self._run_scoped(
  263. session, task, self.relay.scope.event, name,
  264. handle=_scope_handle(session, task), data=data, metadata=self._event_metadata(),
  265. )
  266. def start_task(self, event: dict[str, Any]) -> _TaskRun | None:
  267. """Open one Relay function scope for a Hermes task run."""
  268. task_key = _session_pair(event, "task_id")
  269. if task_key is None:
  270. return None
  271. _, task_id = task_key
  272. with self._task_creation_lock:
  273. owner = self._task_session(event)
  274. if owner is not None:
  275. with owner.lock:
  276. if owner.closing:
  277. return None
  278. task = owner.tasks.get(task_id)
  279. if task is not None and not self._admits(owner, task, event):
  280. return None
  281. return task
  282. session = self.ensure_session(event)
  283. if session is None:
  284. return None
  285. with session.lock:
  286. turn_id = _text(event, "turn_id")
  287. if (
  288. session.closing
  289. or (turn_id and turn_id in session.retired_turn_ids)
  290. or session.relay_session.context is None
  291. ):
  292. return None
  293. self._emit_client_active(session)
  294. task_context = session.relay_session.context.copy()
  295. start_fields = contract.task_start_fields(_with_launch_entrypoint(event))
  296. handle = task_context.run(
  297. self._with_scope_stack, self.relay.scope.push,
  298. TASK_SCOPE, self.relay.ScopeType.Function,
  299. handle=_task_parent_handle(session, task_id), input=start_fields,
  300. metadata=self._event_metadata(),
  301. )
  302. task = _TaskRun(
  303. task_id=task_id,
  304. handle=handle,
  305. context=task_context,
  306. started_ns=monotonic_ns(),
  307. start_fields=start_fields,
  308. retired_turn_ids=frozenset(session.retired_turn_ids),
  309. )
  310. session.tasks[task_id] = task
  311. with self._task_sessions_lock:
  312. self._task_sessions[task_key] = session
  313. self._remember_turn(session, task, event)
  314. return task
  315. def start_user_turn(self, event: dict[str, Any]) -> None:
  316. """pre_llm_call: a user message started this task (Hermes-owned forks never fire it)."""
  317. task = self.start_task(event)
  318. if task is not None:
  319. task.cost.user_turn = True
  320. def _run_in_task(
  321. self, task: _TaskRun, callback: Callable[..., Any], *args: Any, **kwargs: Any
  322. ) -> Any:
  323. return task.context.copy().run(self._with_scope_stack, callback, *args, **kwargs)
  324. def _with_scope_stack(self, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
  325. self.relay.get_scope_stack()
  326. return callback(*args, **kwargs)
  327. def start_model_call(self, event: dict[str, Any]) -> None:
  328. task_id = _text(event, "task_id")
  329. session, task = self._task_pair(event, start=True, allow_task_id_fallback=True)
  330. if task_id and task is None:
  331. return
  332. session = session or self.ensure_session(event)
  333. if session is None:
  334. return
  335. request_id = _text(event, "api_request_id")
  336. if not request_id:
  337. return
  338. model_call_key = (task_id, request_id)
  339. fields = contract.model_call_fields(event)
  340. with session.lock:
  341. if session.closing:
  342. return
  343. if task is not None and not self._admits(session, task, event, current=True):
  344. return
  345. existing = session.model_calls.get(model_call_key)
  346. if existing is not None:
  347. existing.fields = fields
  348. if task is not None:
  349. # Every repeated start for one logical request is another physical
  350. # attempt. Provider fallback resets Hermes's provider-local retry
  351. # ordinal, so ordinal deltas are not a reliable task-level counter.
  352. task.retry_count += 1
  353. return
  354. if task is not None:
  355. task.model_route = fields
  356. task.selected_route = task.selected_route or fields
  357. task.model_call_ids.add(request_id)
  358. if _retry_ordinal(event) > 0:
  359. # A real Hermes retry can advance api_request_id while carrying the
  360. # retry ordinal. Count that physical attempt.
  361. task.retry_count += 1
  362. handle = self._run_scoped(
  363. session, task, self.relay.llm.call, MODEL_CALL_SCOPE, self.relay.LLMRequest({}, {}),
  364. handle=_scope_handle(session, task), metadata=self._event_metadata(),
  365. model_name=contract.MODEL_CALL_PROFILE_MODEL,
  366. )
  367. session.model_calls[model_call_key] = _ModelCall(handle, task_id, fields)
  368. def update_model_call(self, event: dict[str, Any], *, finish: bool) -> None:
  369. """Refresh the located model call's fields from ``event``; ``finish`` closes it.
  370. ``api_request_error`` retains the latest attempt error without closing the logical
  371. call; ``post_api_request`` closes it.
  372. """
  373. session = self._any_session(event)
  374. if session is None:
  375. return
  376. with session.lock:
  377. if session.closing:
  378. return
  379. model_call_key = self._existing_model_call_key(session, event)
  380. model_call = session.model_calls.get(model_call_key) if model_call_key else None
  381. if model_call is None:
  382. return
  383. model_call.fields = contract.model_call_fields(event)
  384. if finish:
  385. task = session.tasks.get(model_call.task_id)
  386. session.replies += task is not None and task.cost.user_turn
  387. session.model_state.observe_call(model_call.fields, event.get("usage"), event.get("context_length"))
  388. self._observe_turn_call(session, session.tasks.get(model_call.task_id), model_call, event)
  389. model_call.ttft_bucket = fields_.ttft_bucket(event)
  390. self._finish_model_call(session, model_call_key, "success")
  391. tokens = fields_.model_token_fields(
  392. event.get("usage"), call_role="primary", **model_call.fields
  393. )
  394. if tokens is not None:
  395. self._guarded(
  396. "Hermes shared-metrics token mark failed", self._mark,
  397. session, session.tasks.get(model_call.task_id), contract.MODEL_TOKENS_MARK, tokens,
  398. )
  399. else:
  400. model_call.error_class = contract.model_error_class(event)
  401. session.model_state.observe_error(model_call.fields, model_call.error_class)
  402. def start_tool_call(self, event: dict[str, Any]) -> None:
  403. """Open one privacy-safe Relay tool lifecycle under its task."""
  404. task_id = _text(event, "task_id")
  405. session, task = self._task_pair(event, start=True, allow_task_id_fallback=True)
  406. if session is None or task is None or not _text(event, "tool_call_id"):
  407. return
  408. if connectors_.is_connector_batch(event):
  409. return # each entry opens its own lifecycle; the wrapper counts only if no entry ran
  410. identity = self._tool_call_identity(event)
  411. with session.lock:
  412. if not self._admits(session, task, event):
  413. return
  414. key = (task_id, *identity)
  415. if identity in task.completed_tool_call_ids or key in session.tool_calls:
  416. return
  417. task.tool_call_ids.add(identity)
  418. session.tool_calls[key] = self._open_tool_call(task, event)
  419. def record_approval(self, event: dict[str, Any]) -> None:
  420. """Record one bounded approval result without approval text or commands."""
  421. session, task = self._approval_task(event)
  422. if session is None or task is None:
  423. return
  424. outcome = contract.tool_approval_outcome(event)
  425. attribution = "unattributed"
  426. with session.lock:
  427. if session.closing or not self._event_matches_task_turn(task, event):
  428. return
  429. if _text(event, "tool_call_id"):
  430. identity = self._tool_call_identity(event)
  431. tool_call = session.tool_calls.get((task.task_id, *identity))
  432. if tool_call is None:
  433. key = _sole(_compatible_tool_call_keys(session, task.task_id, identity))
  434. tool_call = session.tool_calls[key] if key is not None else None
  435. if tool_call is not None:
  436. tool_call.approval_outcome = outcome
  437. attribution = "tool_call"
  438. self._mark(
  439. session, task, contract.TOOL_APPROVAL_MARK,
  440. {"attribution": attribution, "outcome": outcome},
  441. )
  442. def record_tool_call(self, event: dict[str, Any]) -> None:
  443. """Close and count one unique privacy-safe tool lifecycle."""
  444. task_id = _text(event, "task_id")
  445. session, task = self._task_pair(event, allow_task_id_fallback=True)
  446. if session is None or task is None:
  447. return
  448. with session.lock:
  449. if not self._admits(session, task, event):
  450. return
  451. tool_call = None
  452. if _text(event, "tool_call_id"):
  453. observed_identity = self._tool_call_identity(event)
  454. if self._settle_connector_batch(session, task, observed_identity, event):
  455. return
  456. if observed_identity in task.completed_tool_call_ids:
  457. return
  458. identity = observed_identity
  459. tool_call = session.tool_calls.pop((task_id, *identity), None)
  460. if tool_call is None:
  461. if any(
  462. _identities_compatible(completed, observed_identity)
  463. for completed in task.completed_tool_call_ids
  464. ):
  465. return
  466. matching_keys = _compatible_tool_call_keys(session, task_id, observed_identity)
  467. if len(matching_keys) > 1:
  468. # Partial context cannot safely choose between concurrent calls
  469. # that reused the provider-local ID.
  470. return
  471. if matching_keys:
  472. identity = matching_keys[0][1:]
  473. tool_call = session.tool_calls.pop(matching_keys[0])
  474. task.completed_tool_call_ids.update({identity, observed_identity})
  475. task.tool_call_ids.add(identity)
  476. else:
  477. task.unidentified_tool_calls += 1
  478. if tool_call is None:
  479. tool_call = self._open_tool_call(task, event)
  480. if task.cost.user_turn:
  481. session.efficiency.tools.used.add(eff.toolset_metric_name(event.get("toolset")))
  482. self._finish_tool_call(task, tool_call, event)
  483. chain = session.connectors.observe(event)
  484. if chain is not None:
  485. self._mark(session, task, contract.CONNECTOR_CHAIN_MARK, chain)
  486. def _settle_connector_batch(
  487. self, session: _MetricsSession, task: _TaskRun, wrapper: tuple[str, str, str], event: dict[str, Any],
  488. ) -> bool:
  489. """A connector batch wrapper's post is not a call of its own once an entry started: entries
  490. still open (the executor timed out or abandoned the batch) end with the wrapper's outcome.
  491. True when the wrapper was settled this way; a batch refused before any entry counts as itself."""
  492. if not connectors_.is_connector_batch(event):
  493. return False
  494. started = [key for key in session.tool_calls
  495. if key[0] == task.task_id and connectors_.is_batch_entry(key[1:], wrapper)]
  496. if not started and not connectors_.batch_entry_ran(task.completed_tool_call_ids, wrapper):
  497. return False
  498. for key in started:
  499. task.completed_tool_call_ids.add(key[1:])
  500. self._finish_tool_call(task, session.tool_calls.pop(key), event)
  501. task.completed_tool_call_ids.add(wrapper)
  502. return True
  503. def record_skill_lifecycle(self, event: dict[str, Any]) -> None:
  504. """Emit one allowlisted skill fact without its local identity."""
  505. if _text(event, "action").strip().lower() == "loaded":
  506. mark, fields = contract.SKILL_LOAD_MARK, contract.skill_load_fields(event)
  507. else:
  508. mark, fields = contract.SKILL_LIFECYCLE_MARK, contract.skill_lifecycle_fields(event)
  509. if fields is None:
  510. return
  511. session_id, task_id = _text(event, "session_id"), _text(event, "task_id")
  512. session, task = self._task_pair(event, allow_task_id_fallback=not session_id)
  513. if session is None:
  514. if not (session_id and task_id):
  515. # No owning task: a bare process-level mark.
  516. self._with_scope_stack(
  517. self.relay.scope.event, mark, data=fields, metadata=self._event_metadata()
  518. )
  519. return
  520. if task is None:
  521. return
  522. with session.lock:
  523. if (
  524. not session.closing
  525. and session.tasks.get(task.task_id) is task
  526. and self._event_matches_task_turn(task, event)
  527. ):
  528. self._mark(session, task, mark, fields)
  529. def finish_task(self, event: dict[str, Any]) -> None:
  530. """Close one task scope exactly once with bounded terminal fields."""
  531. session = self._any_session(event)
  532. if session is None:
  533. return
  534. with session.lock:
  535. finished = not session.closing and self._finish_task(
  536. session, _text(event, "task_id"), event
  537. )
  538. retired = finished and session.retiring and not session.tasks
  539. if retired:
  540. self.close_session({"session_id": session.session_id})
  541. elif finished:
  542. self._schedule_flush_and_export("Hermes shared-metrics task flush failed")
  543. def close_session(self, event: dict[str, Any]) -> None:
  544. session = self._session(event)
  545. if session is None:
  546. self._close_unseen_segment(_text(event, "session_id"))
  547. return
  548. if not self._abort_session(
  549. session, {**event, **_ABORTED, "completed": False, "interrupted": False}
  550. ):
  551. return
  552. self._emit_session_summary(session)
  553. self._schedule_flush_and_export(
  554. f"Hermes shared-metrics session {session.session_id} flush failed"
  555. )
  556. with self._sessions_lock:
  557. _forget(self._sessions, session.session_id, session)
  558. def shutdown(self) -> None:
  559. # Callers (the atexit hook, an os._exit flush, a multiplex host's teardown) hold the launch
  560. # profile's scope: this profile's own collection gate and send consent decide its exit.
  561. with self._owner_scope():
  562. if not enabled(): # revoked since its last turn: drop the data, never export it
  563. if self._registered:
  564. self.deactivate()
  565. return
  566. self._close_and_export()
  567. def _close_and_export(self) -> None:
  568. with self._sessions_lock:
  569. self._active = False
  570. session_ids = list(self._sessions)
  571. for session_id in session_ids:
  572. self._safe(self.close_session, {"session_id": session_id})
  573. with self._lineage_lock:
  574. unseen = list(self._lineage_of)
  575. for session_id in unseen:
  576. self._safe(self._close_unseen_segment, session_id)
  577. if not self._registered:
  578. return
  579. self._flush_and_export("Hermes shared-metrics shutdown flush failed")
  580. self._deregister()
  581. self._release()
  582. @contextlib.contextmanager
  583. def _owner_scope(self) -> Iterator[None]:
  584. """Bind this runtime's profile home for work that can run outside its own turns."""
  585. token = set_hermes_home_override(self.host.profile_key)
  586. try:
  587. yield
  588. finally:
  589. reset_hermes_home_override(token)
  590. def _deregister(self) -> None:
  591. self._safe(self.relay.subscribers.deregister, self._subscriber_name)
  592. self.host.release_managed_execution(self._subscriber_name)
  593. self._registered = False
  594. def deactivate(self) -> None:
  595. """Stop collection without exporting locally aggregated metrics."""
  596. with self._sessions_lock:
  597. self._active = False
  598. self.subscriber.deactivate()
  599. if self._registered:
  600. self._deregister()
  601. with self._sessions_lock:
  602. sessions = list(self._sessions.values())
  603. for session in sessions:
  604. self._abort_session(session, {"session_id": session.session_id, **_ABORTED})
  605. with self._sessions_lock:
  606. self._sessions.clear()
  607. with self._task_sessions_lock:
  608. self._task_sessions.clear()
  609. self._turn_sessions.clear()
  610. with self._lineage_lock:
  611. self._lineages.clear()
  612. self._lineage_of.clear()
  613. self._release()
  614. def _release(self) -> None:
  615. """Let an in-flight send finish briefly, then drop the atexit hook.
  616. A short-lived CLI process would otherwise exit and kill the daemon send thread
  617. mid-request — the common case for this feature's one cadence.
  618. """
  619. self._join_send_thread()
  620. with contextlib.suppress(Exception):
  621. atexit.unregister(self.shutdown)
  622. def _join_send_thread(self, timeout: float = 2.0) -> None:
  623. """Bounded on purpose: pending packages stay in SQLite and go out next run, so
  624. blocking on a slow network is the wrong trade; the daemon thread dies with the process."""
  625. with self._send_lock:
  626. thread = self._send_thread
  627. if thread is not None and thread.is_alive():
  628. try:
  629. thread.join(timeout)
  630. except Exception:
  631. logger.debug("Shared-metrics send thread join failed", exc_info=True)
  632. def _session(self, event: dict[str, Any]) -> _MetricsSession | None:
  633. with self._sessions_lock:
  634. return self._sessions.get(_text(event, "session_id"))
  635. def _any_session(self, event: dict[str, Any]) -> _MetricsSession | None:
  636. """Owner session by task/turn correlation, else by session_id."""
  637. return self._task_session(event, allow_task_id_fallback=True) or self._session(event)
  638. def _task_pair(
  639. self, event: dict[str, Any], *, start: bool = False, **lookup: Any
  640. ) -> tuple[_MetricsSession | None, _TaskRun | None]:
  641. """Resolve (session, task) for a task-scoped hook; ``start`` opens a missing task."""
  642. session = self._task_session(event, **lookup)
  643. task = session.tasks.get(_text(event, "task_id")) if session is not None else None
  644. if task is None and start:
  645. task = self.start_task(event)
  646. session = self._task_session(event) if task is not None else None
  647. return session, task
  648. def _run_scoped(
  649. self, session: _MetricsSession, task: _TaskRun | None, callback: Callable[..., Any],
  650. *args: Any, **kwargs: Any,
  651. ) -> Any:
  652. """Run under the task context when the call belongs to a task, else the session."""
  653. if task is not None:
  654. return self._run_in_task(task, callback, *args, **kwargs)
  655. return self.host.run_in_session(session.relay_session, callback, *args, **kwargs)
  656. def _schedule_flush_and_export(self, failure_message: str) -> None:
  657. """Schedule the process-wide Relay flush without blocking the finishing turn.
  658. ``flush()`` also waits for every managed tool/LLM call in flight in ANY session, so on a
  659. turn thread one session finishing stalls behind another session's long tool. One worker
  660. per runtime (= per profile); triggers landing while it runs coalesce into one more pass.
  661. """
  662. from agent.memory_provider import spawn_context_thread
  663. with self._flush_lock:
  664. self._flush_pending = True
  665. if self._flush_thread is not None:
  666. return
  667. # Context copy: the export reads the profile's send/consent config via contextvars.
  668. thread = spawn_context_thread(
  669. self._run_flush_worker, name="hermes-shared-metrics-flush", args=(failure_message,)
  670. )
  671. # The worker's first act is taking this lock, so it cannot clear the slot before it is set.
  672. thread.start()
  673. self._flush_thread = thread
  674. def _run_flush_worker(self, failure_message: str) -> None:
  675. while True:
  676. with self._flush_lock:
  677. if not self._flush_pending:
  678. # Cleared under the lock that schedulers check: a trigger can never land on a
  679. # worker that has already decided to exit.
  680. self._flush_thread = None
  681. return
  682. self._flush_pending = False
  683. self._flush_and_export(failure_message)
  684. def _flush_and_export(self, failure_message: str) -> None:
  685. """Flush the Relay subscriber, then export under this profile's scope (its send consent,
  686. whoever triggered the flush); a failed flush skips the export."""
  687. try:
  688. self.relay.subscribers.flush()
  689. except Exception:
  690. logger.warning(failure_message, exc_info=True)
  691. else:
  692. with self._owner_scope():
  693. self._export()
  694. def _abort_session(self, session: _MetricsSession, base_event: dict[str, Any]) -> bool:
  695. """Mark the session closing and system-abort its open tasks; False if already closing."""
  696. with session.lock:
  697. if session.closing:
  698. return False
  699. session.closing = True
  700. for task_id in list(session.tasks):
  701. self._finish_task(session, task_id, {**base_event, "task_id": task_id})
  702. self._end_pending_model_calls(session, base_event)
  703. return True
  704. def _task_session(
  705. self, event: dict[str, Any], *, allow_task_id_fallback: bool = False
  706. ) -> _MetricsSession | None:
  707. """Owner session by (session, turn), then (session, task), then unique task_id."""
  708. session_id, task_id = _text(event, "session_id"), _text(event, "task_id")
  709. if not task_id:
  710. return None
  711. with self._task_sessions_lock:
  712. owner = self._turn_sessions.get(_session_pair(event, "turn_id"))
  713. if owner is None and session_id:
  714. owner = self._task_sessions.get((session_id, task_id))
  715. if owner is not None or not allow_task_id_fallback:
  716. return owner
  717. return _sole(
  718. session for (_, tid), session in self._task_sessions.items() if tid == task_id
  719. )
  720. def _remember_turn(
  721. self, session: _MetricsSession, task: _TaskRun, event: dict[str, Any]
  722. ) -> None:
  723. turn_id = _text(event, "turn_id")
  724. if turn_id:
  725. task.turn_ids.add(turn_id)
  726. with self._task_sessions_lock:
  727. self._turn_sessions[(session.session_id, turn_id)] = session
  728. @staticmethod
  729. def _tool_call_identity(event: dict[str, Any]) -> tuple[str, str, str]:
  730. """Identify one provider-local tool call without exporting its IDs."""
  731. return _text(event, "api_request_id"), _text(event, "turn_id"), _text(event, "tool_call_id")
  732. @staticmethod
  733. def _event_matches_task_turn(task: _TaskRun, event: dict[str, Any]) -> bool:
  734. """Reject delayed hooks from a prior run that reused the task ID."""
  735. turn_id = _text(event, "turn_id")
  736. if not turn_id:
  737. return True
  738. return turn_id not in task.retired_turn_ids and (
  739. not task.turn_ids or turn_id in task.turn_ids
  740. )
  741. def _admits(
  742. self,
  743. session: _MetricsSession,
  744. task: _TaskRun,
  745. event: dict[str, Any],
  746. *,
  747. current: bool = False,
  748. ) -> bool:
  749. """Whether ``event`` may act on ``task`` (caller holds ``session.lock``).
  750. Rejects closing sessions and stale turns; with ``current`` also requires ``task`` to
  751. still be the session's live run for its ID. Admitted events have their turn remembered.
  752. """
  753. if (
  754. session.closing
  755. or not self._event_matches_task_turn(task, event)
  756. or (current and session.tasks.get(task.task_id) is not task)
  757. ):
  758. return False
  759. self._remember_turn(session, task, event)
  760. return True
  761. def _approval_task(
  762. self, event: dict[str, Any]
  763. ) -> tuple[_MetricsSession | None, _TaskRun | None]:
  764. """Resolve approval correlation without guessing across ambiguous turns."""
  765. active = relay_runtime.active_turn()
  766. if active is not None:
  767. session, task = self._task_pair(
  768. {**event, "session_id": active.lease.session_id, "task_id": active.task_id}
  769. )
  770. if task is not None:
  771. return session, task
  772. session, task = self._task_pair(event)
  773. if task is not None:
  774. return session, task
  775. turn_id = _text(event, "turn_id")
  776. session = None
  777. if turn_id:
  778. with self._task_sessions_lock:
  779. session = _sole(
  780. candidate
  781. for (owner_id, candidate_turn_id), candidate in self._turn_sessions.items()
  782. if candidate_turn_id == turn_id and self._sessions.get(owner_id) is candidate
  783. )
  784. if session is None:
  785. return None, None
  786. task = _sole(task for task in session.tasks.values() if turn_id in task.turn_ids)
  787. return (None, None) if task is None else (session, task)
  788. def _open_tool_call(self, task: _TaskRun, event: dict[str, Any]) -> _ToolCall:
  789. handle = self._run_in_task(
  790. task, self.relay.tools.call, contract.TOOL_CALL_SCOPE, {},
  791. handle=task.handle, metadata=self._event_metadata(),
  792. )
  793. return _ToolCall(
  794. handle, contract.tool_category(event), contract.tool_metric_name(event), monotonic_ns()
  795. )
  796. def _finish_tool_call(
  797. self, task: _TaskRun, tool_call: _ToolCall, event: dict[str, Any]
  798. ) -> None:
  799. fields = contract.tool_terminal_fields(
  800. event, category=tool_call.category, approval_outcome=tool_call.approval_outcome,
  801. fallback_duration_ms=_elapsed_ms(tool_call.started_ns), tool_name=tool_call.tool_name,
  802. )
  803. self._guarded(
  804. "Hermes shared-metrics tool call close failed",
  805. lambda: self._run_in_task(
  806. task, self.relay.tools.call_end, tool_call.handle,
  807. self.relay.ToolExecutionResult(fields),
  808. metadata=self._event_metadata(),
  809. ),
  810. )
  811. def _end_pending_tool_calls(
  812. self, session: _MetricsSession, task: _TaskRun, event: dict[str, Any]
  813. ) -> None:
  814. task_outcome, _, _ = contract.task_terminal_state(event)
  815. status = {"cancelled": "cancelled", "timed_out": "timeout"}.get(task_outcome, "error")
  816. for key in [key for key in session.tool_calls if key[0] == task.task_id]:
  817. self._finish_tool_call(task, session.tool_calls.pop(key), {**event, "status": status})
  818. def _finish_model_call(
  819. self, session: _MetricsSession, model_call_key: tuple[str, str], outcome: str
  820. ) -> None:
  821. model_call = session.model_calls.pop(model_call_key, None)
  822. if model_call is None:
  823. return
  824. error_class = model_call.error_class
  825. if outcome == "failed" and error_class == "none":
  826. error_class = "unknown"
  827. fields = contract.model_route_fields(
  828. model_call.fields, call_role="primary", outcome=outcome, error_class=error_class,
  829. ttft_bucket=model_call.ttft_bucket,
  830. )
  831. self._guarded(
  832. "Hermes shared-metrics model call close failed",
  833. self._run_scoped, session, session.tasks.get(model_call.task_id),
  834. self.relay.llm.call_end, model_call.handle, fields,
  835. metadata=self._event_metadata(),
  836. )
  837. def _end_pending_model_calls(self, session: _MetricsSession, event: dict[str, Any]) -> None:
  838. """Close calls that never saw a successful response as failed or cancelled."""
  839. task_id = _text(event, "task_id")
  840. cancelled = contract.task_terminal_state(event)[0] == "cancelled"
  841. pending = [k for k, c in session.model_calls.items() if not task_id or c.task_id == task_id]
  842. for key in pending:
  843. self._finish_model_call(session, key, "cancelled" if cancelled else "failed")
  844. @staticmethod
  845. def _existing_model_call_key(
  846. session: _MetricsSession, event: dict[str, Any]
  847. ) -> tuple[str, str] | None:
  848. """(task_id, request_id) of an open call; a task-less event may match by request alone."""
  849. request_id = _text(event, "api_request_id")
  850. if not request_id:
  851. return None
  852. key = (_text(event, "task_id"), request_id)
  853. if key in session.model_calls or key[0]:
  854. return key if key in session.model_calls else None
  855. candidates = [candidate for candidate in session.model_calls if candidate[1] == request_id]
  856. return candidates[0] if len(candidates) == 1 else None
  857. def _finish_task(self, session: _MetricsSession, task_id: str, event: dict[str, Any]) -> bool:
  858. task = session.tasks.get(task_id)
  859. if task is None:
  860. return False
  861. self._end_pending_tool_calls(session, task, event)
  862. self._end_pending_model_calls(session, {**event, "task_id": task_id})
  863. fields = contract.task_terminal_fields(
  864. {**task.start_fields, **event},
  865. duration_ms=_elapsed_ms(task.started_ns),
  866. model_call_count=len(task.model_call_ids),
  867. tool_call_count=len(task.tool_call_ids) + task.unidentified_tool_calls,
  868. retry_count=task.retry_count,
  869. )
  870. if task.cost.user_turn: # background review forks share the session id and never fire pre_llm_call
  871. self._count_session_turn(session, task, fields["outcome"])
  872. if not session.closing:
  873. self._observe_model_turn(session, task, fields)
  874. try:
  875. popped = self._guarded(
  876. "Hermes shared-metrics task close failed",
  877. self._run_in_task, task, relay_runtime.pop_relay_scope_if_top, self.relay, task.handle,
  878. output=fields, metadata=self._event_metadata(),
  879. )
  880. if popped is False:
  881. logger.debug("Left shared-metrics task scope %s under a concurrent turn's scope; session close drains it", task_id)
  882. finally:
  883. session.tasks.pop(task_id, None)
  884. session.retired_turn_ids.extend(task.turn_ids)
  885. with self._task_sessions_lock:
  886. _forget(self._task_sessions, (session.session_id, task_id), session)
  887. for turn_id in task.turn_ids:
  888. _forget(self._turn_sessions, (session.session_id, turn_id), session)
  889. return True
  890. @staticmethod
  891. def _count_session_turn(session: _MetricsSession, task: _TaskRun, outcome: str) -> None:
  892. if session.start_fields is None:
  893. session.start_fields = dict(task.start_fields)
  894. session.first_turn_ns = task.started_ns
  895. session.turns += 1
  896. session.failed_turns += outcome == "failed"
  897. session.api_calls += len(task.model_call_ids)
  898. session.tool_call_total += len(task.tool_call_ids) + task.unidentified_tool_calls
  899. session.last_outcome = outcome
  900. session.last_turn_ns = monotonic_ns()
  901. def _observe_model_turn(self, session: _MetricsSession, task: _TaskRun, fields: dict[str, str]) -> None:
  902. """A turn the user saw end (session-close aborts excluded): trailing-failure state and interrupts."""
  903. route = task.model_route or session.model_state.last_route
  904. session.model_state.observe_turn(fields["outcome"], route, session.last_turn_ns)
  905. if task.selected_route is not None and task.cost.user_turn:
  906. self._with_route_run(session.session_id, session.route_run, lambda run: run.observe(task.selected_route))
  907. if task.model_route is not None and engagement_.engaged_turn(task.start_fields, task.cost.user_turn):
  908. self._guarded(
  909. "Hermes shared-metrics engagement mark failed", self._mark,
  910. session, None, contract.ENGAGEMENT_TURN_MARK, dict(task.model_route),
  911. )
  912. if fields["end_reason"] == "user_cancelled" and route is not None and model_.attended(task.start_fields):
  913. self._guarded(
  914. "Hermes shared-metrics friction mark failed", self._mark,
  915. session, None, contract.MODEL_FRICTION_MARK, model_.friction_fields("interrupt", route),
  916. )
  917. if task.cost.user_turn and model_.attended(task.start_fields):
  918. rows = session.efficiency.finish_turn(
  919. task.cost, fields, route=route, model_calls=len(task.model_call_ids),
  920. tool_calls=len(task.tool_call_ids) + task.unidentified_tool_calls,
  921. )
  922. self._emit_rows(session, rows)
  923. def _emit_rows(self, session: _MetricsSession | None, rows: list[tuple[str, dict[str, str]]]) -> None:
  924. for mark, data in rows:
  925. if session is None:
  926. self._guarded("Hermes shared-metrics efficiency mark failed", self.record_process_mark, mark, data)
  927. else:
  928. self._guarded("Hermes shared-metrics efficiency mark failed", self._mark, session, None, mark, data)
  929. def _observe_turn_call(
  930. self, session: _MetricsSession, task: _TaskRun | None, model_call: _ModelCall, event: dict[str, Any]
  931. ) -> None:
  932. """One finished primary call of a user turn: its tokens, and a cold cache read nobody announced."""
  933. if task is None or not task.cost.user_turn:
  934. return
  935. route, usage = model_call.fields, event.get("usage")
  936. task.cost.add_call(route, usage)
  937. expected = self._consume_cold(self._cold_key(session.session_id))
  938. cause = session.efficiency.observe_cache(route, usage, model_call.started_ns, monotonic_ns(), expected=expected)
  939. if cause is not None:
  940. self._emit_rows(session, [eff.cache_break_row(cause, route)])
  941. @staticmethod
  942. def _cold_key(session_id: str) -> str:
  943. """The conversation (lineage root survives compression rotation), else the session."""
  944. return get_conversation_context() or session_id
  945. def _announce_cold(self, key: str) -> bool:
  946. """Expect the next read of ``key`` to be cold; False when a break was already announced."""
  947. with self._cold_lock:
  948. if key in self._cold_expected:
  949. return False
  950. self._cold_expected[key] = None
  951. while len(self._cold_expected) > 256:
  952. self._cold_expected.popitem(last=False)
  953. return True
  954. def _consume_cold(self, key: str) -> bool:
  955. with self._cold_lock:
  956. return self._cold_expected.pop(key, False) is None
  957. def record_known_cache_break(self, cause: str, route: dict[str, str], session_id: str) -> None:
  958. """Hermes invalidated the prefix itself: count it, and don't recount the cold read it causes.
  959. Several causes before one cold read are one break (the first cause names it)."""
  960. if self._announce_cold(self._cold_key(session_id)):
  961. self._emit_rows(None, [eff.cache_break_row(cause, route)])
  962. def record_session_tools(self, session_id: str, agent: Any, tools_for_api: list) -> None:
  963. session = self._session({"session_id": session_id}) if session_id else None
  964. if session is None:
  965. return
  966. key = (id(getattr(agent, "tools", None)), id(tools_for_api), len(tools_for_api))
  967. with session.lock:
  968. if session.closing or session.efficiency.tools_key == key:
  969. return
  970. snapshot = eff.tool_snapshot(agent, tools_for_api) # registry + estimator: outside the lock
  971. with session.lock:
  972. if session.closing:
  973. return
  974. session.efficiency.tools_key = key
  975. changed = session.efficiency.tools.observe(*snapshot)
  976. route = model_.model_route(getattr(agent, "provider", None), getattr(agent, "model", None))
  977. if changed and self._announce_cold(self._cold_key(session_id)):
  978. self._emit_rows(session, [eff.cache_break_row("toolset_change", route)])
  979. def rotate_segment(self, old_id: str, new_id: str) -> None:
  980. """Compression rotated ``old_id`` to ``new_id``: the new id continues the same conversation.
  981. Only this explicit hand-off links segments (a reset, /new or /branch parent is a new
  982. conversation). The old segment closes into the lineage now, or once its in-flight turn ends."""
  983. old = self._session({"session_id": old_id})
  984. with self._lineage_lock:
  985. key = self._lineage_of.get(old_id)
  986. if key is None:
  987. if old is None:
  988. return # a segment this process never served: the new id is its own conversation
  989. key = old_id
  990. self._lineages[key] = _PeakLineage(
  991. open={old_id}, run=old.route_run, spent=old.efficiency.spent, connectors=old.connectors)
  992. self._lineage_of[old_id] = key
  993. self._lineages[key].open.add(new_id)
  994. self._lineage_of[new_id] = key
  995. overflow = list(self._lineages)[:-_MAX_LINEAGES]
  996. new = self._session({"session_id": new_id})
  997. if new is not None:
  998. self._share_spent(new)
  999. if old is not None:
  1000. with old.lock:
  1001. old.retiring = not old.closing
  1002. idle = old.retiring and not old.tasks
  1003. if idle:
  1004. self.close_session({"session_id": old_id})
  1005. for stale in overflow:
  1006. self._flush_lineage(stale)
  1007. def _lineage_spent(self, session_id: str) -> deque | None:
  1008. with self._lineage_lock:
  1009. lineage = self._lineages.get(self._lineage_of.get(session_id, ""))
  1010. return lineage.spent if lineage is not None else None
  1011. def _share_spent(self, session: _MetricsSession) -> None:
  1012. """A rotated-to segment's /undo and /retry reach the turns the earlier segments spent, and its
  1013. connector chain continues the earlier segments' searches and descriptions."""
  1014. spent = self._lineage_spent(session.session_id)
  1015. if spent is not None:
  1016. session.efficiency.spent = spent
  1017. with self._lineage_lock:
  1018. lineage = self._lineages.get(self._lineage_of.get(session.session_id, ""))
  1019. if lineage is not None and lineage.connectors is not None:
  1020. session.connectors = lineage.connectors
  1021. def _with_route_run(
  1022. self, session_id: str, own: engagement_.RouteRun | None, update: Callable[[engagement_.RouteRun], Any],
  1023. ) -> Any:
  1024. """Apply ``update`` to the conversation's model-turn run (the lineage's, shared by its segments)."""
  1025. with self._lineage_lock:
  1026. lineage = self._lineages.get(self._lineage_of.get(session_id, ""))
  1027. run = lineage.run if lineage is not None else own
  1028. return update(run) if run is not None else None
  1029. def _close_segment(
  1030. self, session_id: str, state: model_.ModelSessionState | None, tally: engagement_.SessionTally | None,
  1031. tools: eff.ToolUsage,
  1032. ) -> tuple[model_.ModelSessionState | None, engagement_.SessionTally | None, eff.ToolUsage | None]:
  1033. """This segment's context peak, tally and tool usage, or, for a lineage segment, the
  1034. conversation's merged ones once its last open segment closes (None before that)."""
  1035. with self._lineage_lock:
  1036. key = self._lineage_of.pop(session_id, None)
  1037. lineage = self._lineages.get(key) if key is not None else None
  1038. if lineage is None:
  1039. return state, tally, tools
  1040. lineage.open.discard(session_id)
  1041. if state is not None and tally is not None:
  1042. _absorb_peak(lineage.peak, state)
  1043. lineage.tally.absorb(tally)
  1044. lineage.tools.absorb(tools)
  1045. if lineage.open:
  1046. return None, None, None
  1047. del self._lineages[key]
  1048. return lineage.peak, lineage.tally, lineage.tools
  1049. def _close_unseen_segment(self, session_id: str) -> None:
  1050. """A rotated-to segment closed (or the process ended) before it served a turn here."""
  1051. if session_id:
  1052. self._emit_summary_rows(None, *self._close_segment(session_id, None, None, eff.ToolUsage()))
  1053. def _flush_lineage(self, key: str) -> None:
  1054. """Backstop for a conversation a surface never closed: report what it holds, detach its segments."""
  1055. with self._lineage_lock:
  1056. lineage = self._lineages.pop(key, None)
  1057. for segment in lineage.open if lineage is not None else ():
  1058. self._lineage_of.pop(segment, None)
  1059. if lineage is not None:
  1060. self._emit_summary_rows(None, lineage.peak, lineage.tally, lineage.tools)
  1061. def _emit_summary_rows(
  1062. self, session: _MetricsSession | None, peak: model_.ModelSessionState | None,
  1063. tally: engagement_.SessionTally | None, tools: eff.ToolUsage | None, *extra: tuple[str, Any],
  1064. ) -> None:
  1065. marks = [
  1066. (contract.CONTEXT_PEAK_MARK, peak.context_peak_fields() if peak is not None else None),
  1067. (contract.SESSION_MARK, tally.fields() if tally is not None else None),
  1068. *(tools.rows() if tools is not None else ()),
  1069. *extra,
  1070. ]
  1071. self._emit_rows(session, [(mark, data) for mark, data in marks if data is not None])
  1072. def _emit_session_summary(self, session: _MetricsSession) -> None:
  1073. """One hermes.session.count and context-peak row per closed top-level conversation (delegated
  1074. children excluded; compression-rotated segments merged), its tool usage, plus a quick-abandon friction."""
  1075. start = session.start_fields
  1076. counted = bool(session.turns) and start is not None and start.get("entrypoint") != "delegated"
  1077. tally = engagement_.SessionTally(
  1078. session.start_fields, session.turns, session.failed_turns, session.last_outcome,
  1079. session.first_turn_ns, session.last_turn_ns, session.api_calls, session.tool_call_total, session.replies,
  1080. ) if counted else None
  1081. tools = session.efficiency.tools if counted and model_.attended(start) else eff.ToolUsage()
  1082. tools.surface = (start or {}).get("execution_surface", "unknown")
  1083. merged = self._close_segment(session.session_id, session.model_state if counted else None, tally, tools)
  1084. # A segment compression retired is not the user walking away.
  1085. abandoned = session.model_state.quick_abandon_route(monotonic_ns()) if counted and not session.retiring else None
  1086. friction = (
  1087. [(contract.MODEL_FRICTION_MARK, model_.friction_fields("quick_abandon", abandoned))]
  1088. if abandoned is not None and model_.attended(start) else []
  1089. )
  1090. self._emit_summary_rows(session, *merged, *friction)
  1091. def record_switch_after(self, session_id: str) -> None:
  1092. """A counted /model switch: how many turns the model being left served in this conversation."""
  1093. session = self._session({"session_id": session_id})
  1094. with session.lock if session is not None else contextlib.nullcontext():
  1095. taken = self._with_route_run(
  1096. session_id, session.route_run if session is not None else None, lambda run: run.take()
  1097. )
  1098. if taken is not None:
  1099. self.record_process_mark(contract.MODEL_SWITCH_AFTER_MARK, engagement_.switch_after_fields(*taken))
  1100. def record_friction(self, signal: str, session_id: str, fallback_route: dict[str, str], turns: int = 1) -> None:
  1101. """A user friction action, attributed to the session's last primary model when known; an
  1102. /undo or /retry also counts the tokens of the turns it threw away."""
  1103. session = self._session({"session_id": session_id}) if session_id else None
  1104. route, wasted = None, []
  1105. # A session this process never saw (restart, remote host) still counts, with tokens unknown.
  1106. with session.lock if session is not None else contextlib.nullcontext():
  1107. if session is not None:
  1108. route = session.model_state.last_route
  1109. if signal in contract.WASTE_REASONS:
  1110. efficiency = session.efficiency if session is not None else eff.SessionEfficiency()
  1111. if session is None and (spent := self._lineage_spent(session_id)) is not None:
  1112. efficiency.spent = spent # a rotated-to segment that has not served a turn yet
  1113. wasted = efficiency.discard_turns(signal, turns, route or fallback_route)
  1114. data = model_.friction_fields(signal, route or fallback_route)
  1115. if data is not None:
  1116. self.record_process_mark(contract.MODEL_FRICTION_MARK, data)
  1117. self._emit_rows(None, wasted)
  1118. def record_process_mark(self, mark: str, data: dict[str, Any]) -> None:
  1119. """A process-level fact with no owning task (setup, installs, commands, switches)."""
  1120. # No flush here: the next task close or the atexit shutdown drains the subscriber.
  1121. self._with_scope_stack(self.relay.scope.event, mark, data=data, metadata=self._event_metadata())
  1122. def record_process_marks_saved(self, marks: list[tuple[str, dict[str, Any]]]) -> int:
  1123. """Emit ``marks`` and wait for the store; how many settled without a store error."""
  1124. ticket = uuid.uuid4().hex
  1125. metadata = {**self._event_metadata(), contract.COMMIT_TICKET_KEY: ticket}
  1126. for mark, data in marks:
  1127. self._with_scope_stack(self.relay.scope.event, mark, data=data, metadata=metadata)
  1128. self.relay.subscribers.flush()
  1129. return self.subscriber.take_saved(ticket)
  1130. def record_auxiliary_tokens(self, event: dict[str, Any]) -> None:
  1131. tokens = fields_.model_token_fields(
  1132. event.get("usage"), call_role="auxiliary", aux_task=event.get("aux_task"),
  1133. model=event.get("response_model") or event.get("model"), provider=event.get("provider"),
  1134. )
  1135. if tokens is not None:
  1136. self._with_scope_stack(
  1137. self.relay.scope.event, contract.MODEL_TOKENS_MARK, data=tokens,
  1138. metadata=self._event_metadata(),
  1139. )
  1140. def _export(self) -> None:
  1141. exported = self._safe(self.subscriber.store.create_and_export_package_if_due)
  1142. # Sending must never delay the caller: _export runs on finish_task, the user's
  1143. # interactive path. The thread is about latency, not correctness.
  1144. if exported is not None:
  1145. self._safe(self._send_exported_packages)
  1146. def _send_exported_packages(self) -> None:
  1147. try:
  1148. resolved = _resolved_send_config()
  1149. except Exception:
  1150. logger.debug("Unable to read shared-metrics send policy", exc_info=True)
  1151. return
  1152. # Observe the consent EDGE before deciding whether to send: the dominant revocation
  1153. # case is "sending turned off while no pass is running", invisible to the send loop.
  1154. # Failures never break the export hook but log at warning (privacy-relevant).
  1155. self._guarded(
  1156. "Unable to record a shared-metrics consent transition",
  1157. _reconcile_store_consent, self.subscriber.store, resolved.send,
  1158. )
  1159. if not resolved.send:
  1160. return
  1161. with self._send_lock:
  1162. # One in-flight pass per process; the next hook fire picks up what is pending.
  1163. if self._send_thread is None or not self._send_thread.is_alive():
  1164. from agent.memory_provider import spawn_context_thread
  1165. # Context copy: the pass re-checks consent (still_consented) in THIS profile's scope.
  1166. self._send_thread = spawn_context_thread(
  1167. self._run_send_pass, name="hermes-shared-metrics-send", args=(resolved.endpoint,)
  1168. )
  1169. self._send_thread.start()
  1170. def _run_send_pass(self, endpoint: str) -> None:
  1171. from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender
  1172. def still_consented() -> bool:
  1173. """Re-read consent so revoking `send` stops an in-flight pass."""
  1174. resolved = _resolved_send_config()
  1175. return resolved.send and resolved.endpoint == endpoint
  1176. sender = SharedMetricsSender(self.subscriber.store, endpoint, consent_check=still_consented)
  1177. self._guarded("Shared-metrics send pass failed", sender.send_pending)
  1178. def _event_metadata(self) -> dict[str, str]:
  1179. return {
  1180. contract.SCHEMA_KEY: contract.SCHEMA_VERSION,
  1181. relay_runtime.RUNTIME_INSTANCE_KEY: self.host.runtime_id,
  1182. }
  1183. @staticmethod
  1184. def _guarded(message: str, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
  1185. """Run ``callback``; log-and-swallow any exception, returning None."""
  1186. try:
  1187. return callback(*args, **kwargs)
  1188. except Exception:
  1189. logger.warning(message, exc_info=True)
  1190. return None
  1191. @classmethod
  1192. def _safe(cls, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
  1193. return cls._guarded("Hermes shared metrics operation failed", callback, *args, **kwargs)
  1194. def _raw_config() -> dict[str, Any]:
  1195. """Read-only config snapshot (lazy import: tests patch ``hermes_cli.config``).
  1196. Collection consent is profile-owned: managed overlays cannot opt a profile in or out.
  1197. The read-only path matters because this gate runs 2-3x per agent turn and the mutable
  1198. read_raw_config() paid a full config deepcopy on every call.
  1199. """
  1200. from hermes_cli.config import read_raw_config_readonly
  1201. return read_raw_config_readonly() or {}
  1202. def _resolved_send_config():
  1203. """Resolve the opt-in send policy from the read-only config snapshot."""
  1204. from hermes_cli.observability.shared_metrics_send_config import resolve_send_config
  1205. return resolve_send_config(_raw_config())
  1206. def _reconcile_store_consent(store: SharedMetricsStore, send_enabled: bool) -> None:
  1207. from hermes_cli.observability.shared_metrics_sender import reconcile_send_consent
  1208. from hermes_cli.sqlite_util import write_txn
  1209. with store._connection() as connection:
  1210. with write_txn(connection):
  1211. reconcile_send_consent(connection, send_enabled)
  1212. def enabled() -> bool:
  1213. """Return the shared-metrics policy for the active Hermes profile."""
  1214. profile_key = relay_runtime.current_profile_key()
  1215. try:
  1216. config: Any = _raw_config()
  1217. except Exception:
  1218. logger.debug("Unable to read Hermes shared-metrics policy", exc_info=True)
  1219. config = None
  1220. for key in ("telemetry", "shared_metrics"):
  1221. config = config.get(key) if isinstance(config, dict) else None
  1222. if isinstance(config, dict) and config.get("enabled") is True:
  1223. return True
  1224. if profile_key not in _RUNTIMES: # the opted-out hot path: nothing to tear down, no lock
  1225. return False
  1226. with _RUNTIME_LOCK:
  1227. runtime = _RUNTIMES.pop(profile_key, None)
  1228. if isinstance(runtime, _Runtime):
  1229. runtime.deactivate()
  1230. return False
  1231. def handles_hook(hook_name: str) -> bool:
  1232. return hook_name in HANDLED_HOOKS and enabled()
  1233. _consent_reconcile_done = False
  1234. def _reconcile_send_consent_once() -> None:
  1235. """Reconcile consent windows with config, once per process.
  1236. Runs BEFORE and INDEPENDENT of the collection gate, so a user with ``enabled: false``
  1237. still gets send-consent windows reconciled. Skipped only when there is no store on disk
  1238. AND consent is off: nothing to protect, and creating ``~/.hermes/telemetry`` for every
  1239. fully-disabled user would be the wrong behaviour change.
  1240. """
  1241. global _consent_reconcile_done
  1242. if _consent_reconcile_done:
  1243. return
  1244. _consent_reconcile_done = True
  1245. try:
  1246. # Lazy: tests patch ``shared_metrics.SharedMetricsStore`` at its origin.
  1247. from hermes_cli.observability.shared_metrics import SharedMetricsStore
  1248. from hermes_constants import get_hermes_home
  1249. resolved = _resolved_send_config()
  1250. # Probe WITHOUT constructing a store: the constructor creates the directory and
  1251. # schema as a side effect, which would make the skip below dead code.
  1252. default_path = get_hermes_home() / "telemetry" / "shared_metrics" / "metrics.sqlite3"
  1253. if not resolved.send and not default_path.exists():
  1254. return
  1255. _reconcile_store_consent(SharedMetricsStore(), resolved.send)
  1256. except Exception:
  1257. logger.warning("Unable to reconcile shared-metrics send consent", exc_info=True)
  1258. # Hooks whose only effect is a process mark (no task, no scope): gated like record_process_mark.
  1259. _PROCESS_MARK_HOOKS = frozenset({"post_auxiliary_call"})
  1260. def _process_marks_admitted() -> bool:
  1261. """Whether the inherited turn lets a process-level mark through.
  1262. A process mark has no owning task and opens no scope, so a turn that has CLOSED does not gate
  1263. it: background work spawned under the turn (a turn-end compaction pass, title generation)
  1264. inherits that turn through its copied context and still reports after it ends. A turn Relay
  1265. skipped (a concurrent turn on the session) keeps gating, open or closed."""
  1266. turn = relay_runtime.current_turn()
  1267. return turn is None or turn.relay_enabled
  1268. def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
  1269. """Project one Hermes lifecycle event into the core Relay integration."""
  1270. _reconcile_send_consent_once()
  1271. if not handles_hook(hook_name):
  1272. return
  1273. if not (
  1274. _process_marks_admitted() if hook_name in _PROCESS_MARK_HOOKS
  1275. else relay_runtime.relay_instrumentation_enabled()
  1276. ):
  1277. return
  1278. runtime = _get_runtime()
  1279. if runtime is None:
  1280. return
  1281. try:
  1282. _HOOK_HANDLERS[hook_name](runtime, kwargs)
  1283. except Exception:
  1284. logger.warning("Hermes shared metrics hook failed: %s", hook_name, exc_info=True)
  1285. def _with_runtime_toolset(event: dict[str, Any]) -> dict[str, Any]:
  1286. """Attach the toolset already declared by Hermes's runtime registry."""
  1287. tool_name = _text(event, "tool_name")
  1288. if event.get("toolset") or not tool_name:
  1289. return event
  1290. try:
  1291. from model_tools import get_toolset_for_tool
  1292. toolset = get_toolset_for_tool(tool_name)
  1293. except Exception:
  1294. toolset = None
  1295. return {**event, "toolset": toolset or "other"}
  1296. def _close_child_session(runtime: _Runtime, kwargs: dict[str, Any]) -> None:
  1297. child_session_id = _text(kwargs, "child_session_id")
  1298. if child_session_id:
  1299. runtime.close_session({"session_id": child_session_id})
  1300. _HOOK_HANDLERS: dict[str, Callable[[_Runtime, dict[str, Any]], Any]] = {
  1301. "on_session_start": lambda rt, kw: rt.record_client_active(kw),
  1302. "pre_llm_call": lambda rt, kw: rt.start_user_turn(kw),
  1303. "pre_api_request": lambda rt, kw: rt.start_model_call(kw),
  1304. "pre_tool_call": lambda rt, kw: rt.start_tool_call(_with_runtime_toolset(kw)),
  1305. "post_tool_call": lambda rt, kw: rt.record_tool_call(_with_runtime_toolset(kw)),
  1306. "post_approval_response": lambda rt, kw: rt.record_approval(kw),
  1307. "on_skill_lifecycle": lambda rt, kw: rt.record_skill_lifecycle(kw),
  1308. "post_api_request": lambda rt, kw: rt.update_model_call(kw, finish=True),
  1309. "post_auxiliary_call": lambda rt, kw: rt.record_auxiliary_tokens(kw),
  1310. "api_request_error": lambda rt, kw: rt.update_model_call(kw, finish=False),
  1311. "on_session_end": lambda rt, kw: rt.finish_task(kw),
  1312. "subagent_stop": _close_child_session,
  1313. "on_session_finalize": lambda rt, kw: rt.close_session(kw),
  1314. "on_session_reset": lambda rt, kw: rt.close_session(kw),
  1315. }
  1316. HANDLED_HOOKS = frozenset(_HOOK_HANDLERS)
  1317. def _prepare_core_session(host: relay_runtime.RelayRuntime, context: dict[str, Any]) -> None:
  1318. """Prepare the profile subscriber before the coordinator opens a scope."""
  1319. del context
  1320. if host.profile_key == relay_runtime.current_profile_key() and enabled():
  1321. _get_runtime(retry_failed=True, host=host)
  1322. def start_task_run(
  1323. *, session_id: str, task_id: str, platform: str, parent_session_id: str = ""
  1324. ) -> None:
  1325. """Start task metrics at the outer Hermes execution boundary."""
  1326. _run_task_hook(
  1327. "start_task", retry_failed=True, session_id=session_id, task_id=task_id,
  1328. platform=platform, parent_session_id=parent_session_id,
  1329. )
  1330. def shutdown_runtimes() -> None:
  1331. """Run each live runtime's exit flush now, for a process that leaves through ``os._exit`` (the
  1332. ``hermes -z`` exit) and so never reaches the atexit hook that closes sessions and writes
  1333. ``task_run.finished`` / ``session.count``. Each runtime binds its own profile while it does."""
  1334. with _RUNTIME_LOCK:
  1335. runtimes = [runtime for runtime in _RUNTIMES.values() if isinstance(runtime, _Runtime)]
  1336. for runtime in runtimes:
  1337. runtime._safe(runtime.shutdown)
  1338. def close_session_run(session_id: str) -> None:
  1339. """Emit the summary of a session a surface retired without a finalize hook (gateway
  1340. auto-reset, idle expiry). Never creates a runtime; unknown ids are a no-op."""
  1341. if not session_id or not enabled():
  1342. return
  1343. runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
  1344. if isinstance(runtime, _Runtime):
  1345. runtime._safe(runtime.close_session, {"session_id": session_id})
  1346. def rotate_segment(old_session_id: str, new_session_id: str) -> None:
  1347. """Compression rotated a conversation's session id (the one seam that continues a conversation
  1348. under a new id). Never creates a runtime: a runtime that does not exist served no segment."""
  1349. if not old_session_id or not new_session_id or old_session_id == new_session_id or not enabled():
  1350. return
  1351. runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
  1352. if isinstance(runtime, _Runtime):
  1353. runtime._safe(runtime.rotate_segment, old_session_id, new_session_id)
  1354. def finish_task_run(
  1355. *, session_id: str, task_id: str, platform: str,
  1356. result: dict[str, Any] | None = None, error: BaseException | None = None,
  1357. ) -> None:
  1358. """Finish task metrics for every return or exception path."""
  1359. _run_task_hook(
  1360. "finish_task", session_id=session_id, task_id=task_id, platform=platform,
  1361. **_terminal_flags(result, error),
  1362. )
  1363. def record_process_mark(mark: str, data: dict[str, Any]) -> None:
  1364. """Emit one process-level decision mark when shared metrics are on (never raises)."""
  1365. if not enabled() or not _process_marks_admitted():
  1366. return
  1367. runtime = _get_runtime(retry_failed=True)
  1368. if runtime is not None:
  1369. runtime._safe(runtime.record_process_mark, mark, data)
  1370. def record_process_marks_saved(marks: list[tuple[str, dict[str, Any]]]) -> int:
  1371. """``record_process_mark`` for facts recovered from a file the caller deletes only once they are
  1372. saved (a busy store would otherwise lose them for good). How many rows are settled: saved, rejected
  1373. by the contract, or nothing to record because collection is off."""
  1374. if not enabled() or not _process_marks_admitted():
  1375. return len(marks)
  1376. runtime = _get_runtime(retry_failed=True)
  1377. return (runtime._safe(runtime.record_process_marks_saved, marks) or 0) if runtime is not None else 0
  1378. def record_session_friction(signal: str, session_id: str, fallback_route: dict[str, str], turns: int = 1) -> None:
  1379. """Record one friction signal in the active profile's runtime (callers already checked enabled())."""
  1380. if not relay_runtime.relay_instrumentation_enabled():
  1381. return
  1382. runtime = _get_runtime(retry_failed=True)
  1383. if runtime is not None:
  1384. runtime._safe(runtime.record_friction, signal, session_id, fallback_route, turns)
  1385. def record_session_tools(session_id: str, agent: Any, tools_for_api: list) -> None:
  1386. """The tools one primary request sent (caller checked enabled()); never creates a runtime."""
  1387. runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
  1388. if isinstance(runtime, _Runtime):
  1389. runtime._safe(runtime.record_session_tools, session_id, agent, tools_for_api)
  1390. def record_known_cache_break(cause: str, route: dict[str, str], session_id: str) -> None:
  1391. """A prompt-cache break Hermes caused (caller checked enabled())."""
  1392. if not relay_runtime.relay_instrumentation_enabled():
  1393. return
  1394. runtime = _get_runtime(retry_failed=True)
  1395. if runtime is not None:
  1396. runtime._safe(runtime.record_known_cache_break, cause, route, session_id)
  1397. def record_model_switch_after(session_id: str) -> None:
  1398. """Count turns-before-switch for a session this process served; never creates a runtime (a
  1399. runtime that does not exist saw no turns). Callers are already in the owning profile's scope."""
  1400. if not session_id or not enabled() or not relay_runtime.relay_instrumentation_enabled():
  1401. return
  1402. runtime = _RUNTIMES.get(relay_runtime.current_profile_key())
  1403. if isinstance(runtime, _Runtime):
  1404. runtime._safe(runtime.record_switch_after, session_id)
  1405. def _run_task_hook(method: str, *, retry_failed: bool = False, **event: Any) -> None:
  1406. if not enabled():
  1407. return
  1408. runtime = _get_runtime(retry_failed=retry_failed)
  1409. if runtime is not None:
  1410. runtime._safe(getattr(runtime, method), event)
  1411. def _terminal_flags(result: dict[str, Any] | None, error: BaseException | None) -> dict[str, Any]:
  1412. """Bounded completed/failed/interrupted/turn_exit_reason for a task's return or raise."""
  1413. if error is not None:
  1414. interrupted = (
  1415. isinstance(error, (KeyboardInterrupt, InterruptedError))
  1416. or type(error).__name__ == "CancelledError"
  1417. )
  1418. if interrupted:
  1419. reason = "interrupted_by_user"
  1420. else:
  1421. reason = "timed_out" if isinstance(error, TimeoutError) else "system_aborted"
  1422. return {
  1423. "completed": False, "failed": not interrupted, "interrupted": interrupted,
  1424. "turn_exit_reason": reason, "failure_class": "exception",
  1425. }
  1426. terminal = result if isinstance(result, dict) else {}
  1427. failed = terminal.get("failed") is True
  1428. reason = str(terminal.get("turn_exit_reason") or terminal.get("failure_reason") or "")
  1429. return {
  1430. "completed": terminal.get("completed") is True,
  1431. "failed": failed,
  1432. "interrupted": terminal.get("interrupted") is True,
  1433. "turn_exit_reason": reason or ("failed" if failed else "unknown"),
  1434. # Classified in the contract against a closed set; the raw string never leaves.
  1435. "failure_reason": str(terminal.get("failure_reason") or ""),
  1436. }
  1437. def _get_runtime(
  1438. *, retry_failed: bool = False, host: relay_runtime.RelayRuntime | None = None
  1439. ) -> _Runtime | None:
  1440. profile_key = relay_runtime.current_profile_key()
  1441. with _RUNTIME_LOCK:
  1442. runtime = _RUNTIMES.get(profile_key)
  1443. if isinstance(runtime, _Runtime):
  1444. if host is None or runtime.host is host:
  1445. return runtime
  1446. runtime.deactivate()
  1447. elif runtime is _RUNTIME_FAILED and not retry_failed:
  1448. return None
  1449. try:
  1450. _RUNTIMES[profile_key] = runtime = _Runtime(host=host)
  1451. except Exception:
  1452. logger.warning("Hermes shared metrics initialization failed", exc_info=True)
  1453. _RUNTIMES[profile_key] = _RUNTIME_FAILED
  1454. return None
  1455. return runtime
  1456. relay_runtime.SESSION_COORDINATOR.register_session_initializer(
  1457. SUBSCRIBER_NAME, _prepare_core_session
  1458. )
  1459. def _reset_for_tests() -> None:
  1460. """Reset all profile-scoped shared-metrics state for isolated tests."""
  1461. with _RUNTIME_LOCK:
  1462. runtimes = list(_RUNTIMES.values())
  1463. _RUNTIMES.clear()
  1464. for runtime in runtimes:
  1465. if isinstance(runtime, _Runtime):
  1466. runtime.shutdown()