shared_metrics_gateway.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. """Call-site API for messaging-gateway and cron shared metrics.
  2. Platform connection health, outbound delivery outcomes, first-reply latency and cron runs happen on
  3. the gateway event loop, adapter tasks or the cron ticker, never inside a model or tool call. Every
  4. recorder takes RAW runtime objects, derives only closed classes / buckets / public platform names,
  5. and hands the recording to one background worker so the caller's loop never waits on the first
  6. runtime build or a config read. The worker runs in a copy of the caller's context, so the profile
  7. the caller was scoped to (gateway multiplex, cron ``_profile_cron_scope``) owns the row.
  8. This module is imported by ``gateway/platforms/base.py`` at class-definition time: keep imports lazy.
  9. """
  10. from __future__ import annotations
  11. import contextvars
  12. import functools
  13. import logging
  14. import threading
  15. import time
  16. from collections import OrderedDict
  17. from concurrent.futures import Future, ThreadPoolExecutor
  18. from datetime import datetime, timezone, UTC
  19. from typing import Any, Callable
  20. logger = logging.getLogger(__name__)
  21. _executor: ThreadPoolExecutor | None = None
  22. _executor_lock = threading.Lock()
  23. def _submit(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Future | None:
  24. """Fire-and-forget: run ``fn`` on the metrics worker in the caller's context (profile scope)."""
  25. global _executor
  26. try:
  27. with _executor_lock:
  28. if _executor is None:
  29. _executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="hermes-gateway-metrics")
  30. return _executor.submit(contextvars.copy_context().run, _guarded, fn, *args, **kwargs)
  31. except Exception: # interpreter shutdown refuses new work: the metric is simply dropped
  32. logger.debug("Gateway shared metric not scheduled", exc_info=True)
  33. return None
  34. def _guarded(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
  35. try:
  36. fn(*args, **kwargs)
  37. except Exception:
  38. logger.debug("Gateway shared metric not recorded", exc_info=True)
  39. def drain(timeout: float = 5.0) -> None:
  40. """Wait for queued recordings (tests and live probes; the worker is FIFO)."""
  41. future = _submit(lambda: None)
  42. if future is not None:
  43. future.result(timeout=timeout)
  44. def _emit(mark_name: str, build: Callable[..., dict[str, str] | None], **raw: Any) -> None:
  45. from . import shared_metrics_contract as contract
  46. from .shared_metrics_events import _emit as emit
  47. emit(getattr(contract, mark_name), build, **raw)
  48. def _platform(value: Any) -> str:
  49. from .shared_metrics_contract import adapter_platform
  50. return adapter_platform(getattr(value, "platform", value))
  51. # ---- classification (closed classes from exception types, HTTP statuses and Hermes's own codes) ----
  52. _STATUS_CLASSES = {401: "auth", 403: "forbidden", 413: "too_long", 429: "rate_limited"}
  53. # Adapter fatal codes are Hermes-authored identifiers (``telegram_auth_error``); first match wins, so
  54. # "missing credentials" reads as configuration, not a rejected credential.
  55. _CODE_CLASSES = (
  56. (("missing", "dependency", "npm", "config", "conflict", "lock", "bind"), "config"),
  57. (("auth", "credential", "token", "unauthor", "forbidden", "login"), "auth"),
  58. (("ratelimit", "rate_limit", "flood", "too_many", "toomany", "429"), "rate_limited"),
  59. (("network", "connect", "timeout", "socket", "dns", "ssl", "lost", "exited", "stream"), "network"),
  60. )
  61. _SEND_KIND_CLASSES = {
  62. "too_long": "too_long", "forbidden": "forbidden", "rate_limited": "rate_limited", "transient": "network",
  63. }
  64. def _status_code(exc: BaseException | None) -> int | None:
  65. # Stored values only, never a property: a deprecated one warns (websockets' ``code``), and a
  66. # classifier must not run library code against the adapter's live exception.
  67. from inspect import getattr_static
  68. for holder in (exc, getattr_static(exc, "response", None)):
  69. for attr in ("status_code", "status", "code"):
  70. value = getattr_static(holder, attr, None)
  71. if isinstance(value, int) and not isinstance(value, bool) and 100 <= value <= 599:
  72. return value
  73. return None
  74. def _code_class(code: Any) -> str | None:
  75. text = str(code or "").strip().lower()
  76. if not text:
  77. return None
  78. return next((cls for needles, cls in _CODE_CLASSES if any(n in text for n in needles)), "other")
  79. def _exception_class(exc: BaseException | None) -> str:
  80. """Connection error class from the exception's type and HTTP status only (never its message)."""
  81. if exc is None:
  82. return "other"
  83. status = _status_code(exc)
  84. if status in (401, 403):
  85. return "auth"
  86. if status == 429:
  87. return "rate_limited"
  88. if isinstance(exc, ImportError):
  89. return "config"
  90. if isinstance(exc, (TimeoutError, ConnectionError)):
  91. return "network"
  92. names = "".join(cls.__name__.lower() for cls in type(exc).__mro__[:-1])
  93. # Type names say auth / rate / network; "configuration" is only ever a Hermes fatal code.
  94. return next((cls for needles, cls in _CODE_CLASSES[1:] if any(n in names for n in needles)), "other")
  95. def connect_error_class(*, exc: BaseException | None = None, fatal_code: Any = None) -> str:
  96. if fatal_code:
  97. return _code_class(fatal_code) or "other"
  98. return _exception_class(exc)
  99. def delivery_failure_class(result: Any = None, exc: BaseException | None = None) -> str:
  100. """Why an outbound delivery failed, from the SendResult's typed fields or the exception type."""
  101. if exc is not None:
  102. status = _STATUS_CLASSES.get(_status_code(exc) or 0)
  103. if status is not None:
  104. return status
  105. cls = _exception_class(exc)
  106. return cls if cls in {"auth", "network", "rate_limited"} else "other"
  107. raw = getattr(result, "raw_response", None)
  108. status = raw.get("status_code", raw.get("status")) if isinstance(raw, dict) else None
  109. if isinstance(status, int) and status in _STATUS_CLASSES:
  110. return _STATUS_CLASSES[status]
  111. if getattr(result, "retry_after", None) is not None:
  112. return "rate_limited"
  113. kind = getattr(result, "error_kind", None)
  114. if kind is None:
  115. from gateway.platforms.base import classify_send_error
  116. kind = classify_send_error(None, str(getattr(result, "error", "") or ""))
  117. if kind in _SEND_KIND_CLASSES:
  118. return _SEND_KIND_CLASSES[kind]
  119. return "network" if getattr(result, "retryable", False) else "other"
  120. # ---- hermes.platform.health ----
  121. def platform_health_fields(*, platform: Any, event: str, error_class: str) -> dict[str, str] | None:
  122. from . import shared_metrics_contract as contract
  123. if event not in contract.PLATFORM_HEALTH_EVENTS:
  124. return None
  125. ok = event in {"connect_ok", "reconnect"}
  126. return {
  127. "error_class": "none" if ok else error_class if error_class in contract.PLATFORM_ERROR_CLASSES else "other",
  128. "event": event,
  129. "platform": _platform(platform),
  130. }
  131. def _record_health(platform: Any, event: str, *, exc: BaseException | None = None, fatal_code: Any = None) -> None:
  132. # Classified on the worker: whatever an exception carries, the caller re-raising it never sees us.
  133. error_class = "none" if event in {"connect_ok", "reconnect"} else connect_error_class(exc=exc, fatal_code=fatal_code)
  134. _emit("PLATFORM_HEALTH_MARK", platform_health_fields, platform=platform, event=event, error_class=error_class)
  135. def _in_home(home: Any, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
  136. """Run ``fn`` scoped to ``home`` (the profile that owns the row), or the submitting context's scope."""
  137. if home is None:
  138. fn(*args, **kwargs)
  139. return
  140. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  141. token = set_hermes_home_override(str(home))
  142. try:
  143. fn(*args, **kwargs)
  144. finally:
  145. reset_hermes_home_override(token)
  146. # (profile home, platform) -> UTC day of the open failed-connect episode. Touched only on the FIFO worker.
  147. _failing_connects: dict[tuple[str, str], str] = {}
  148. def _record_connect(platform: Any, event: str, *, exc: BaseException | None, fatal_code: Any, terminal: bool) -> None:
  149. from hermes_constants import get_hermes_home
  150. from .relay_shared_metrics import enabled
  151. # Keyed by the real adapter (two custom adapters both emit "plugin"); projected only at emission.
  152. key = (str(get_hermes_home()), str(getattr(platform, "value", platform)))
  153. day = datetime.now(UTC).date().isoformat()
  154. if event != "connect_failed" or terminal:
  155. counted = _failing_connects.pop(key, None) == day
  156. elif not enabled(): # nothing is recorded, so nothing is latched: opting in mid-outage counts it
  157. return
  158. else:
  159. counted = _failing_connects.get(key) == day
  160. _failing_connects[key] = day
  161. if not (counted and event == "connect_failed"): # else: a watcher retry already counted today
  162. _record_health(platform, event, exc=exc, fatal_code=fatal_code)
  163. def record_platform_connect(
  164. adapter: Any, platform: Any, *, is_reconnect: bool, ok: bool, exc: BaseException | None = None,
  165. ) -> None:
  166. """One adapter connect attempt (cold start or reconnect watcher), from the runner's single
  167. connect seam. ``connect_failed`` counts a failed EPISODE once per profile, platform and UTC day:
  168. the watcher's backoff retries are not new failures, so the next row is the success that ends
  169. it (or the next day it is still failing). A non-retryable failure ends its episode."""
  170. event = ("reconnect" if is_reconnect else "connect_ok") if ok else "connect_failed"
  171. fatal_code = None if ok or exc is not None else getattr(adapter, "fatal_error_code", None)
  172. terminal = not ok and exc is None and bool(fatal_code) and not getattr(adapter, "fatal_error_retryable", True)
  173. _submit(_record_connect, platform, event, exc=exc, fatal_code=fatal_code, terminal=terminal)
  174. # The user turned the platform off (relay opt-out revokes its credential): not a lost connection.
  175. _USER_CHOSEN_FATAL_CODES = frozenset({"relay_disabled"})
  176. def record_platform_disconnect(adapter: Any, *, hermes_home: Any = None) -> None:
  177. """A live adapter lost its connection after startup (its fatal-error handler owned it).
  178. ``hermes_home``: the owning profile, for handlers that run outside its scope (multiplexed
  179. secondaries)."""
  180. fatal_code = getattr(adapter, "fatal_error_code", None)
  181. if fatal_code in _USER_CHOSEN_FATAL_CODES:
  182. return
  183. _submit(_in_home, hermes_home, _record_health, adapter, "disconnect", fatal_code=fatal_code)
  184. # ---- hermes.platform.delivery + hermes.gateway.reply_latency ----
  185. def delivery_fields(*, platform: Any, failure_class: str) -> dict[str, str]:
  186. from . import shared_metrics_contract as contract
  187. return {
  188. "failure_class": failure_class if failure_class in contract.DELIVERY_FAILURE_CLASSES else "other",
  189. "outcome": "sent" if failure_class == "none" else "failed",
  190. "platform": _platform(platform),
  191. }
  192. def records_delivery(send: Callable[..., Any]) -> Callable[..., Any]:
  193. """Count one logical outbound delivery (all retries and the plain-text fallback included) per
  194. call of the wrapped ``_send_with_retry`` (replies, busy acks, command replies alike)."""
  195. @functools.wraps(send)
  196. async def wrapper(self, *args: Any, **kwargs: Any) -> Any:
  197. chat_id = kwargs.get("chat_id", args[0] if args else None)
  198. # The final send runs after the routed profile scope is reset: the row goes to the profile
  199. # whose turn last started in this chat, not the launch profile.
  200. home = _chat_home(self, chat_id)
  201. try:
  202. result = await send(self, *args, **kwargs)
  203. except Exception as exc:
  204. _submit(_in_home, home, _record_delivery, self, exc=exc, chat_id=chat_id)
  205. raise
  206. if isinstance(getattr(result, "success", None), bool):
  207. _submit(_in_home, home, _record_delivery, self, result, chat_id=chat_id)
  208. return result
  209. return wrapper
  210. def _record_delivery(
  211. adapter: Any, result: Any = None, *, exc: BaseException | None = None, chat_id: Any = None,
  212. ) -> None:
  213. failure_class = "none" if exc is None and result.success else delivery_failure_class(result, exc)
  214. _emit("PLATFORM_DELIVERY_MARK", delivery_fields, platform=_source_platform(adapter, chat_id),
  215. failure_class=failure_class)
  216. def stops_reply_clock(send_or_edit: Callable[..., Any]) -> Callable[..., Any]:
  217. """Stream consumer transport: the first delivered visible text (frame, draft or message)."""
  218. @functools.wraps(send_or_edit)
  219. async def wrapper(self, *args: Any, **kwargs: Any) -> Any:
  220. ok = await send_or_edit(self, *args, **kwargs)
  221. if ok is True and getattr(self, "_last_sent_text", None):
  222. stop_reply_clock(getattr(self, "adapter", None), getattr(self, "chat_id", None))
  223. return ok
  224. return wrapper
  225. _REPLY_THRESHOLDS = ((2.0, "lt_2s"), (5.0, "2s_to_5s"), (15.0, "5s_to_15s"), (60.0, "15s_to_60s"))
  226. _REPLY_CLOCK_MAX = 1024
  227. _REPLY_CLOCK_MAX_AGE = 3600.0
  228. # (platform value, chat id) -> (monotonic start, owning Hermes home). Keyed without the profile: the
  229. # send side may run outside the turn's scope, so the start side records whose row it is.
  230. _reply_clocks: OrderedDict[tuple[str, str], tuple[float, str]] = OrderedDict()
  231. # (platform value, chat id) -> the Hermes home whose turn last started there; outlives the clock so
  232. # later sends (busy acks, follow-ups) in the chat are attributed too.
  233. _chat_homes: OrderedDict[tuple[str, str], str] = OrderedDict()
  234. _reply_lock = threading.Lock()
  235. # One connector adapter fronting several platforms: the turn started under the platform the message
  236. # came in on (``discord``), its reply leaves through this adapter.
  237. _FRONTING_PLATFORMS = frozenset({"relay"})
  238. def _source_platform(adapter: Any, chat_id: Any) -> Any:
  239. """The platform a fronting adapter's chat really lives on (the relay connector learned it from
  240. the inbound, else the one platform it fronts); the adapter's own platform otherwise. Read-only."""
  241. platform = getattr(adapter, "platform", None)
  242. if str(getattr(platform, "value", platform) or "") not in _FRONTING_PLATFORMS or chat_id in (None, ""):
  243. return adapter
  244. resolve = getattr(adapter, "_metrics_platform", None)
  245. try:
  246. return (resolve(str(chat_id)) if callable(resolve) else None) or adapter
  247. except Exception:
  248. return adapter
  249. def _clock_key(platform: Any, chat_id: Any) -> tuple[str, str] | None:
  250. value = str(getattr(platform, "value", platform) or "")
  251. return (value, str(chat_id)) if value and chat_id not in (None, "") else None
  252. def start_reply_clock(source: Any, *, internal: bool = False) -> None:
  253. """An inbound user message was accepted as a new agent turn (internal events never are)."""
  254. key = None if internal else _clock_key(getattr(source, "platform", None), getattr(source, "chat_id", None))
  255. if key is None:
  256. return
  257. from hermes_constants import get_hermes_home
  258. home = str(get_hermes_home())
  259. with _reply_lock:
  260. for table, value in ((_reply_clocks, (time.monotonic(), home)), (_chat_homes, home)):
  261. table.pop(key, None)
  262. table[key] = value
  263. while len(table) > _REPLY_CLOCK_MAX:
  264. table.popitem(last=False)
  265. def _chat_home(adapter: Any, chat_id: Any) -> str | None:
  266. """The home owning ``chat_id``'s latest turn (a fronting adapter matches the one fronted chat),
  267. else None (the caller's scope)."""
  268. key = _clock_key(getattr(adapter, "platform", None), chat_id)
  269. if key is None:
  270. return None
  271. with _reply_lock:
  272. if key in _chat_homes:
  273. return _chat_homes[key]
  274. if key[0] in _FRONTING_PLATFORMS:
  275. fronted = [home for k, home in _chat_homes.items() if k[1] == key[1]]
  276. return fronted[0] if len(fronted) == 1 else None
  277. return None
  278. def stop_reply_clock(adapter: Any, chat_id: Any, result: Any = None) -> None:
  279. """First outbound text for the chat's pending turn (stream first chunk or the final reply; busy
  280. acks and command replies never stop it). ``result``: a failed send is not a reply."""
  281. key = _clock_key(getattr(adapter, "platform", None), chat_id)
  282. if key is None or (result is not None and getattr(result, "success", False) is not True):
  283. return
  284. with _reply_lock:
  285. started, started_key = _reply_clocks.pop(key, None), key
  286. if started is None and key[0] in _FRONTING_PLATFORMS:
  287. fronted = [k for k in _reply_clocks if k[1] == key[1]]
  288. started_key = fronted[0] if len(fronted) == 1 else key
  289. started = _reply_clocks.pop(started_key) if len(fronted) == 1 else None
  290. # A turn that ended without a reply leaves its clock behind; an unrelated send hours later
  291. # must not be read as that turn's reply.
  292. if started is not None and time.monotonic() - started[0] <= _REPLY_CLOCK_MAX_AGE:
  293. # Labelled by the platform the message came in on, not the connector that carried the reply.
  294. platform = started_key[0] if started_key[0] not in _FRONTING_PLATFORMS else None
  295. _submit(_in_home, started[1], _emit_reply_latency, adapter, chat_id, platform, time.monotonic() - started[0])
  296. def _emit_reply_latency(adapter: Any, chat_id: Any, platform: Any, seconds: float) -> None:
  297. _emit("REPLY_LATENCY_MARK", reply_latency_fields,
  298. platform=platform or _source_platform(adapter, chat_id), seconds=seconds)
  299. def reply_latency_fields(*, platform: Any, seconds: float) -> dict[str, str]:
  300. from .shared_metrics_contract import _bucket
  301. return {
  302. "first_response_bucket": _bucket(max(0.0, seconds), _REPLY_THRESHOLDS, "gte_60s"),
  303. "platform": _platform(platform),
  304. }
  305. # ---- hermes.cron.run ----
  306. _CRON_NOTES_MAX = 512
  307. # execution id -> delivery kind, noted where the job dict is in hand; popped once at the terminal write.
  308. _cron_delivery: OrderedDict[str, str] = OrderedDict()
  309. _cron_skipped: set[str] = set()
  310. _cron_lock = threading.Lock()
  311. _LOCAL_DELIVERY = frozenset({"local", ""})
  312. def cron_delivery_kind(job: Any) -> str:
  313. """Where a job's output goes, as a closed kind; targets themselves never leave the machine."""
  314. if not isinstance(job, dict):
  315. return "other"
  316. deliver = job.get("deliver")
  317. parts = deliver if isinstance(deliver, (list, tuple)) else str(deliver or "").split(",")
  318. kinds = {str(p).strip().lower().split(":", 1)[0] for p in parts} - {""}
  319. if not kinds or kinds <= _LOCAL_DELIVERY:
  320. return "local"
  321. if kinds & {"none", "off"}:
  322. return "none"
  323. return "webhook" if kinds == {"webhook"} else "platform"
  324. def note_cron_execution(job: Any) -> None:
  325. """Remember the delivery kind of an execution about to run (or be refused) in this process."""
  326. execution_id = job.get("execution_id") if isinstance(job, dict) else None
  327. if not execution_id:
  328. return
  329. with _cron_lock:
  330. _cron_delivery[str(execution_id)] = cron_delivery_kind(job)
  331. while len(_cron_delivery) > _CRON_NOTES_MAX:
  332. _cron_delivery.popitem(last=False)
  333. def note_cron_skipped(job: Any) -> None:
  334. """A gate (pre-run script ``wakeAgent=false`` / no output) decided this run does no agent work."""
  335. execution_id = job.get("execution_id") if isinstance(job, dict) else None
  336. if execution_id:
  337. with _cron_lock:
  338. if len(_cron_skipped) < _CRON_NOTES_MAX:
  339. _cron_skipped.add(str(execution_id))
  340. def _iso_seconds(start: Any, end: Any) -> float:
  341. try:
  342. return max(0.0, (datetime.fromisoformat(str(end)) - datetime.fromisoformat(str(start))).total_seconds())
  343. except (TypeError, ValueError):
  344. return 0.0
  345. def cron_run_fields(*, outcome: str, delivery_kind: str, seconds: float) -> dict[str, str]:
  346. from . import shared_metrics_contract as contract
  347. return {
  348. "delivery_kind": delivery_kind if delivery_kind in contract.CRON_DELIVERY_KINDS else "other",
  349. "duration_bucket": contract.duration_bucket(int(seconds * 1000)),
  350. "outcome": outcome if outcome in contract.CRON_RUN_OUTCOMES else "failed",
  351. }
  352. def record_cron_finish(record: Any, delivery_outcome: Any = None) -> None:
  353. """One execution reached its (write-once) terminal row. A row that never started running was
  354. refused before any side effect (claim lost, dispatch limit, executor shutdown): ``skipped``,
  355. unless a failure notice was delivered for it."""
  356. if not isinstance(record, dict) or not record.get("id"):
  357. return
  358. execution_id = str(record["id"])
  359. with _cron_lock:
  360. delivery_kind = _cron_delivery.pop(execution_id, "other")
  361. gated = execution_id in _cron_skipped
  362. _cron_skipped.discard(execution_id)
  363. started = record.get("started_at")
  364. success = record.get("status") == "completed"
  365. if record.get("status") == "cancelled": # an operator stop, not a failure
  366. outcome = "cancelled"
  367. elif gated or (not started and not success and delivery_outcome is None):
  368. outcome = "skipped"
  369. else:
  370. outcome = "success" if success else "failed"
  371. _submit(_emit, "CRON_RUN_MARK", cron_run_fields, outcome=outcome, delivery_kind=delivery_kind,
  372. seconds=_iso_seconds(started, record.get("finished_at")) if started else 0.0)
  373. def record_cron_missed(job: Any) -> None:
  374. """A scheduled occurrence the scheduler dropped without running it (catch-up disabled after
  375. downtime, or a one-shot past its grace window)."""
  376. _submit(_emit, "CRON_RUN_MARK", cron_run_fields, outcome="missed",
  377. delivery_kind=cron_delivery_kind(job), seconds=0.0)