otlp_exporter.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. """Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
  2. Maps gateway monitoring events to OTel spans for the operator-configured ``monitoring.export.otlp``
  3. endpoint (no default destination ships) and hosts the OTLP plumbing shared with
  4. ``gateway_health_export``. The OTel SDK is an optional extra (``hermes-agent[otlp]``) imported
  5. lazily; ``headers_env`` values are read at export time and never logged or stored. The streaming
  6. subscriber runs fail-isolated on the emitter thread; ``event_filter`` keeps other planes off it.
  7. """
  8. from __future__ import annotations
  9. from pm import install_hint
  10. import importlib
  11. import logging
  12. import os
  13. import re
  14. from contextlib import suppress
  15. from typing import Any, Callable, Dict, Iterable, List, Optional
  16. from agent.monitoring.gateway_health import _safe_instance_id
  17. from agent.monitoring.redaction import redact_bounded
  18. logger = logging.getLogger(__name__)
  19. class OTLPUnavailable(RuntimeError):
  20. """Raised when the optional OpenTelemetry SDK isn't installed."""
  21. # ── SDK loading ──────────────────────────────────────────────────────────────
  22. _SDK_MODULES: dict[str, tuple[str, ...]] = {
  23. "opentelemetry.sdk.trace": ("TracerProvider",),
  24. "opentelemetry.sdk.trace.export": ("BatchSpanProcessor",),
  25. "opentelemetry.sdk.resources": ("Resource",),
  26. "opentelemetry.exporter.otlp.proto.http.trace_exporter": ("OTLPSpanExporter",),
  27. "opentelemetry.exporter.otlp.proto.http._log_exporter": ("OTLPLogExporter",),
  28. "opentelemetry.exporter.otlp.proto.http.metric_exporter": ("OTLPMetricExporter",),
  29. "opentelemetry.trace": ("SpanKind", "INVALID_SPAN_ID", "INVALID_TRACE_ID", "TraceFlags"),
  30. "opentelemetry.metrics": ("Observation",),
  31. "opentelemetry._logs": ("LogRecord",),
  32. "opentelemetry._logs.severity": ("SeverityNumber",),
  33. "opentelemetry.sdk._logs": ("LoggerProvider",),
  34. "opentelemetry.sdk._logs.export": ("BatchLogRecordProcessor",),
  35. "opentelemetry.sdk.metrics": ("MeterProvider",),
  36. "opentelemetry.sdk.metrics.export": ("PeriodicExportingMetricReader",),
  37. }
  38. _SDK_SYMBOLS: dict[str, str] = {name: module for module, names in _SDK_MODULES.items() for name in names}
  39. _SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind")
  40. def _require_sdk(names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True) -> dict[str, Any]:
  41. """Import the named OTel SDK symbols, lazily installing the extra on first use.
  42. Routes through pm.ensure_import('otlp') so a missing SDK triggers the
  43. standard venv install flow — same as every other optional backend — gated by
  44. security.allow_lazy_installs. Any lazy-install failure falls through to the
  45. import attempt, which raises OTLPUnavailable with a manual install hint.
  46. """
  47. if auto_install:
  48. try:
  49. from pm import ensure_import as _lazy_ensure
  50. _lazy_ensure("otlp")
  51. except Exception:
  52. pass
  53. try:
  54. return {name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name) for name in names}
  55. except Exception as e: # ImportError or partial install
  56. raise OTLPUnavailable(
  57. "OTLP export requires the optional dependency. Install with:\n"
  58. f" {install_hint('otlp')}\n"
  59. f"(import error: {e})"
  60. )
  61. # ── config + connection plumbing ─────────────────────────────────────────────
  62. def _monitoring_section(config: dict[str, Any], *path: str) -> dict[str, Any]:
  63. node: Any = (config or {}).get("monitoring") or {}
  64. for key in path:
  65. node = node.get(key) or {}
  66. return node
  67. def _otlp_config(config: dict[str, Any]) -> dict[str, Any]:
  68. return _monitoring_section(config, "export", "otlp")
  69. def _resolve_headers(headers_env: Optional[dict[str, str]]) -> dict[str, str]:
  70. """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value}; missing vars skipped."""
  71. resolved: dict[str, str] = {}
  72. for header_name, env_name in (headers_env or {}).items():
  73. val = os.environ.get(str(env_name))
  74. if val:
  75. resolved[str(header_name)] = val
  76. else:
  77. logger.debug("OTLP header %s: env var %s not set; skipping", header_name, env_name)
  78. return resolved
  79. _SIGNAL_SUFFIXES = ("/v1/traces", "/v1/metrics")
  80. def _signal_endpoint(endpoint: str, signal: str) -> str:
  81. """Rewrite a traces/metrics OTLP path to ``/v1/<signal>``; other paths pass through."""
  82. target = f"/v1/{signal}"
  83. for suffix in _SIGNAL_SUFFIXES:
  84. if suffix != target and endpoint.endswith(suffix):
  85. return endpoint[: -len(suffix)] + target
  86. return endpoint
  87. _RESOURCE_ATTRIBUTE_KEYS = frozenset({
  88. "service.name", "service.namespace", "service.version", "service.instance.id",
  89. "deployment.environment.name", "cloud.provider", "cloud.platform", "cloud.region", "telemetry.scope",
  90. })
  91. _SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$")
  92. def _install_id(config: dict[str, Any]) -> str:
  93. try:
  94. from agent.monitoring.policy import ensure_install_id
  95. return str(ensure_install_id(config))
  96. except Exception:
  97. return "unknown"
  98. def _safe_resource_attributes(raw: Any) -> dict[str, str]:
  99. """Allowlist bounded resource labels and reject values changed by redaction."""
  100. attrs: dict[str, str] = {}
  101. for key, value in (raw.items() if isinstance(raw, dict) else ()):
  102. key = str(key)
  103. if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None:
  104. continue
  105. text = str(value)
  106. if key == "service.instance.id":
  107. attrs[key] = _safe_instance_id(value)
  108. elif _SAFE_RESOURCE_VALUE.fullmatch(text) and redact_bounded(text, limit=128) == text:
  109. attrs[key] = text
  110. return attrs
  111. def _runtime_resource_attributes(config: dict[str, Any], *, telemetry_scope: str) -> dict[str, str]:
  112. """Build the safe OTLP resource shared by spans, metrics and diagnostic logs."""
  113. attrs = _safe_resource_attributes(_monitoring_section(config, "gateway_health_export").get("resource_attributes"))
  114. attrs["service.name"] = "hermes-gateway"
  115. attrs["service.instance.id"] = _safe_instance_id(_install_id(config))
  116. attrs["telemetry.scope"] = telemetry_scope
  117. return attrs
  118. def build_exporter(config: dict[str, Any]):
  119. """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
  120. sdk = _require_sdk()
  121. otlp = _otlp_config(config)
  122. endpoint = otlp.get("endpoint")
  123. if not endpoint:
  124. raise ValueError("monitoring.export.otlp.endpoint is not set")
  125. return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=_resolve_headers(otlp.get("headers_env")) or None)
  126. def _resource_attributes(config: dict[str, Any]) -> dict[str, str]:
  127. return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring")
  128. def _make_provider(config: dict[str, Any]):
  129. sdk = _require_sdk()
  130. provider = sdk["TracerProvider"](resource=sdk["Resource"].create(_resource_attributes(config)))
  131. processor = sdk["BatchSpanProcessor"](build_exporter(config))
  132. provider.add_span_processor(processor)
  133. return provider, processor
  134. # ── event -> span attribute mapping ──────────────────────────────────────────
  135. # Per-kind attribute allowlists: everything else (profile, install_id, ...) never egresses.
  136. _KEEP_BY_KIND: dict[str, tuple[str, ...]] = {
  137. "gateway_health": (
  138. "name", "gateway_state", "old_state", "new_state", "exit_reason", "restart_requested", "active_agents",
  139. "gateway_busy", "gateway_drainable", "platform_count", "fatal_platform_count", "version", "supervision_mode", "pid",
  140. ),
  141. "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "platform", "old_state", "new_state", "version", "severity"),
  142. "cron_execution": ("status", "job_key", "source", "duration_ms", "delivery_outcome", "error_class"),
  143. }
  144. def _allowlisted_attrs(ev: dict[str, Any], keys: Iterable[str]) -> dict[str, Any]:
  145. """``hermes.<key>`` attributes for present keys; string values are redacted and bounded."""
  146. attrs: dict[str, Any] = {}
  147. for col in keys:
  148. v = ev.get(col)
  149. if v is not None:
  150. attrs[f"hermes.{col}"] = redact_bounded(v) if isinstance(v, str) else v
  151. return attrs
  152. def _span_attrs(ev: dict[str, Any]) -> dict[str, Any]:
  153. """Span attributes for a monitoring event (content-free by construction)."""
  154. kind = ev.get("event")
  155. attrs: dict[str, Any] = {"hermes.event": kind or "unknown"}
  156. attrs.update(_allowlisted_attrs(ev, _KEEP_BY_KIND.get(kind, ()))) # type: ignore[arg-type]
  157. return attrs
  158. def export_batch(provider, batch: list[dict[str, Any]]) -> int:
  159. """Map a batch of events to OTel spans. Returns spans created."""
  160. tracer = provider.get_tracer("hermes.monitoring")
  161. n = 0
  162. for ev in batch:
  163. try:
  164. tracer.start_span(f"hermes.{ev.get('event', 'event')}", attributes=_span_attrs(ev)).end()
  165. n += 1
  166. except Exception:
  167. logger.debug("OTLP span map failed", exc_info=True)
  168. return n
  169. # ── continuous streaming subscribers ─────────────────────────────────────────
  170. class EmitterStreamer:
  171. """Base for emitter subscribers owning an OTel provider + batch processor.
  172. Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter."""
  173. _provider: Any
  174. _processor: Any
  175. exported: int = 0
  176. def shutdown(self) -> None:
  177. with suppress(Exception):
  178. from agent.monitoring.emitter import get_emitter
  179. get_emitter().unsubscribe(self)
  180. with suppress(Exception):
  181. self._processor.force_flush()
  182. self._provider.shutdown()
  183. class OTLPStreamer(EmitterStreamer):
  184. """A live subscriber that pushes each emitter batch to OTLP as spans."""
  185. def __init__(self, config: dict[str, Any], *, event_filter: Optional[Callable[[dict[str, Any]], bool]] = None):
  186. self._provider, self._processor = _make_provider(config)
  187. self._event_filter = event_filter
  188. self.exported = 0
  189. def __call__(self, batch: list[dict[str, Any]]) -> None:
  190. if self._event_filter is not None:
  191. batch = [ev for ev in batch if self._event_filter(ev)]
  192. if not batch:
  193. return
  194. self.exported += export_batch(self._provider, batch)
  195. def is_available() -> bool:
  196. """True when the OTel SDK is already importable (pure check, no auto-install)."""
  197. try:
  198. _require_sdk(auto_install=False)
  199. return True
  200. except OTLPUnavailable:
  201. return False
  202. def is_enabled(config: dict[str, Any]) -> bool:
  203. otlp = _otlp_config(config)
  204. return bool(otlp.get("enabled") and otlp.get("endpoint"))
  205. def start_streaming(
  206. config: dict[str, Any], *, event_filter: Optional[Callable[[dict[str, Any]], bool]] = None,
  207. ) -> Optional[OTLPStreamer]:
  208. """If OTLP is enabled, attach a streamer to the singleton emitter.
  209. ``event_filter`` scopes the exporter to its plane. Startup is non-interactive: a
  210. configured-but-missing SDK is lazily installed once (gated by
  211. security.allow_lazy_installs); if it still can't load, log and no-op — never raise into startup.
  212. """
  213. if not is_enabled(config):
  214. return None
  215. try:
  216. _require_sdk()
  217. except OTLPUnavailable:
  218. logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
  219. "be installed/imported; install 'hermes-agent[otlp]'")
  220. return None
  221. from agent.monitoring.emitter import get_emitter
  222. streamer = OTLPStreamer(config, event_filter=event_filter)
  223. get_emitter().subscribe(streamer)
  224. return streamer
  225. __all__ = [
  226. "OTLPStreamer",
  227. "OTLPUnavailable",
  228. "build_exporter",
  229. "export_batch",
  230. "is_available",
  231. "is_enabled",
  232. "start_streaming",
  233. ]