| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293 |
- """Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
- Maps gateway monitoring events to OTel spans for the operator-configured ``monitoring.export.otlp``
- endpoint (no default destination ships) and hosts the OTLP plumbing shared with
- ``gateway_health_export``. The OTel SDK is an optional extra (``hermes-agent[otlp]``) imported
- lazily; ``headers_env`` values are read at export time and never logged or stored. The streaming
- subscriber runs fail-isolated on the emitter thread; ``event_filter`` keeps other planes off it.
- """
- from __future__ import annotations
- from pm import install_hint
- import importlib
- import logging
- import os
- import re
- from contextlib import suppress
- from typing import Any, Callable, Dict, Iterable, List, Optional
- from agent.monitoring.gateway_health import _safe_instance_id
- from agent.monitoring.redaction import redact_bounded
- logger = logging.getLogger(__name__)
- class OTLPUnavailable(RuntimeError):
- """Raised when the optional OpenTelemetry SDK isn't installed."""
- # ── SDK loading ──────────────────────────────────────────────────────────────
- _SDK_MODULES: dict[str, tuple[str, ...]] = {
- "opentelemetry.sdk.trace": ("TracerProvider",),
- "opentelemetry.sdk.trace.export": ("BatchSpanProcessor",),
- "opentelemetry.sdk.resources": ("Resource",),
- "opentelemetry.exporter.otlp.proto.http.trace_exporter": ("OTLPSpanExporter",),
- "opentelemetry.exporter.otlp.proto.http._log_exporter": ("OTLPLogExporter",),
- "opentelemetry.exporter.otlp.proto.http.metric_exporter": ("OTLPMetricExporter",),
- "opentelemetry.trace": ("SpanKind", "INVALID_SPAN_ID", "INVALID_TRACE_ID", "TraceFlags"),
- "opentelemetry.metrics": ("Observation",),
- "opentelemetry._logs": ("LogRecord",),
- "opentelemetry._logs.severity": ("SeverityNumber",),
- "opentelemetry.sdk._logs": ("LoggerProvider",),
- "opentelemetry.sdk._logs.export": ("BatchLogRecordProcessor",),
- "opentelemetry.sdk.metrics": ("MeterProvider",),
- "opentelemetry.sdk.metrics.export": ("PeriodicExportingMetricReader",),
- }
- _SDK_SYMBOLS: dict[str, str] = {name: module for module, names in _SDK_MODULES.items() for name in names}
- _SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind")
- def _require_sdk(names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True) -> dict[str, Any]:
- """Import the named OTel SDK symbols, lazily installing the extra on first use.
- Routes through pm.ensure_import('otlp') so a missing SDK triggers the
- standard venv install flow — same as every other optional backend — gated by
- security.allow_lazy_installs. Any lazy-install failure falls through to the
- import attempt, which raises OTLPUnavailable with a manual install hint.
- """
- if auto_install:
- try:
- from pm import ensure_import as _lazy_ensure
- _lazy_ensure("otlp")
- except Exception:
- pass
- try:
- return {name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name) for name in names}
- except Exception as e: # ImportError or partial install
- raise OTLPUnavailable(
- "OTLP export requires the optional dependency. Install with:\n"
- f" {install_hint('otlp')}\n"
- f"(import error: {e})"
- )
- # ── config + connection plumbing ─────────────────────────────────────────────
- def _monitoring_section(config: dict[str, Any], *path: str) -> dict[str, Any]:
- node: Any = (config or {}).get("monitoring") or {}
- for key in path:
- node = node.get(key) or {}
- return node
- def _otlp_config(config: dict[str, Any]) -> dict[str, Any]:
- return _monitoring_section(config, "export", "otlp")
- def _resolve_headers(headers_env: Optional[dict[str, str]]) -> dict[str, str]:
- """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value}; missing vars skipped."""
- resolved: dict[str, str] = {}
- for header_name, env_name in (headers_env or {}).items():
- val = os.environ.get(str(env_name))
- if val:
- resolved[str(header_name)] = val
- else:
- logger.debug("OTLP header %s: env var %s not set; skipping", header_name, env_name)
- return resolved
- _SIGNAL_SUFFIXES = ("/v1/traces", "/v1/metrics")
- def _signal_endpoint(endpoint: str, signal: str) -> str:
- """Rewrite a traces/metrics OTLP path to ``/v1/<signal>``; other paths pass through."""
- target = f"/v1/{signal}"
- for suffix in _SIGNAL_SUFFIXES:
- if suffix != target and endpoint.endswith(suffix):
- return endpoint[: -len(suffix)] + target
- return endpoint
- _RESOURCE_ATTRIBUTE_KEYS = frozenset({
- "service.name", "service.namespace", "service.version", "service.instance.id",
- "deployment.environment.name", "cloud.provider", "cloud.platform", "cloud.region", "telemetry.scope",
- })
- _SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$")
- def _install_id(config: dict[str, Any]) -> str:
- try:
- from agent.monitoring.policy import ensure_install_id
- return str(ensure_install_id(config))
- except Exception:
- return "unknown"
- def _safe_resource_attributes(raw: Any) -> dict[str, str]:
- """Allowlist bounded resource labels and reject values changed by redaction."""
- attrs: dict[str, str] = {}
- for key, value in (raw.items() if isinstance(raw, dict) else ()):
- key = str(key)
- if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None:
- continue
- text = str(value)
- if key == "service.instance.id":
- attrs[key] = _safe_instance_id(value)
- elif _SAFE_RESOURCE_VALUE.fullmatch(text) and redact_bounded(text, limit=128) == text:
- attrs[key] = text
- return attrs
- def _runtime_resource_attributes(config: dict[str, Any], *, telemetry_scope: str) -> dict[str, str]:
- """Build the safe OTLP resource shared by spans, metrics and diagnostic logs."""
- attrs = _safe_resource_attributes(_monitoring_section(config, "gateway_health_export").get("resource_attributes"))
- attrs["service.name"] = "hermes-gateway"
- attrs["service.instance.id"] = _safe_instance_id(_install_id(config))
- attrs["telemetry.scope"] = telemetry_scope
- return attrs
- def build_exporter(config: dict[str, Any]):
- """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
- sdk = _require_sdk()
- otlp = _otlp_config(config)
- endpoint = otlp.get("endpoint")
- if not endpoint:
- raise ValueError("monitoring.export.otlp.endpoint is not set")
- return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=_resolve_headers(otlp.get("headers_env")) or None)
- def _resource_attributes(config: dict[str, Any]) -> dict[str, str]:
- return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring")
- def _make_provider(config: dict[str, Any]):
- sdk = _require_sdk()
- provider = sdk["TracerProvider"](resource=sdk["Resource"].create(_resource_attributes(config)))
- processor = sdk["BatchSpanProcessor"](build_exporter(config))
- provider.add_span_processor(processor)
- return provider, processor
- # ── event -> span attribute mapping ──────────────────────────────────────────
- # Per-kind attribute allowlists: everything else (profile, install_id, ...) never egresses.
- _KEEP_BY_KIND: dict[str, tuple[str, ...]] = {
- "gateway_health": (
- "name", "gateway_state", "old_state", "new_state", "exit_reason", "restart_requested", "active_agents",
- "gateway_busy", "gateway_drainable", "platform_count", "fatal_platform_count", "version", "supervision_mode", "pid",
- ),
- "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "platform", "old_state", "new_state", "version", "severity"),
- "cron_execution": ("status", "job_key", "source", "duration_ms", "delivery_outcome", "error_class"),
- }
- def _allowlisted_attrs(ev: dict[str, Any], keys: Iterable[str]) -> dict[str, Any]:
- """``hermes.<key>`` attributes for present keys; string values are redacted and bounded."""
- attrs: dict[str, Any] = {}
- for col in keys:
- v = ev.get(col)
- if v is not None:
- attrs[f"hermes.{col}"] = redact_bounded(v) if isinstance(v, str) else v
- return attrs
- def _span_attrs(ev: dict[str, Any]) -> dict[str, Any]:
- """Span attributes for a monitoring event (content-free by construction)."""
- kind = ev.get("event")
- attrs: dict[str, Any] = {"hermes.event": kind or "unknown"}
- attrs.update(_allowlisted_attrs(ev, _KEEP_BY_KIND.get(kind, ()))) # type: ignore[arg-type]
- return attrs
- def export_batch(provider, batch: list[dict[str, Any]]) -> int:
- """Map a batch of events to OTel spans. Returns spans created."""
- tracer = provider.get_tracer("hermes.monitoring")
- n = 0
- for ev in batch:
- try:
- tracer.start_span(f"hermes.{ev.get('event', 'event')}", attributes=_span_attrs(ev)).end()
- n += 1
- except Exception:
- logger.debug("OTLP span map failed", exc_info=True)
- return n
- # ── continuous streaming subscribers ─────────────────────────────────────────
- class EmitterStreamer:
- """Base for emitter subscribers owning an OTel provider + batch processor.
- Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter."""
- _provider: Any
- _processor: Any
- exported: int = 0
- def shutdown(self) -> None:
- with suppress(Exception):
- from agent.monitoring.emitter import get_emitter
- get_emitter().unsubscribe(self)
- with suppress(Exception):
- self._processor.force_flush()
- self._provider.shutdown()
- class OTLPStreamer(EmitterStreamer):
- """A live subscriber that pushes each emitter batch to OTLP as spans."""
- def __init__(self, config: dict[str, Any], *, event_filter: Optional[Callable[[dict[str, Any]], bool]] = None):
- self._provider, self._processor = _make_provider(config)
- self._event_filter = event_filter
- self.exported = 0
- def __call__(self, batch: list[dict[str, Any]]) -> None:
- if self._event_filter is not None:
- batch = [ev for ev in batch if self._event_filter(ev)]
- if not batch:
- return
- self.exported += export_batch(self._provider, batch)
- def is_available() -> bool:
- """True when the OTel SDK is already importable (pure check, no auto-install)."""
- try:
- _require_sdk(auto_install=False)
- return True
- except OTLPUnavailable:
- return False
- def is_enabled(config: dict[str, Any]) -> bool:
- otlp = _otlp_config(config)
- return bool(otlp.get("enabled") and otlp.get("endpoint"))
- def start_streaming(
- config: dict[str, Any], *, event_filter: Optional[Callable[[dict[str, Any]], bool]] = None,
- ) -> Optional[OTLPStreamer]:
- """If OTLP is enabled, attach a streamer to the singleton emitter.
- ``event_filter`` scopes the exporter to its plane. Startup is non-interactive: a
- configured-but-missing SDK is lazily installed once (gated by
- security.allow_lazy_installs); if it still can't load, log and no-op — never raise into startup.
- """
- if not is_enabled(config):
- return None
- try:
- _require_sdk()
- except OTLPUnavailable:
- logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
- "be installed/imported; install 'hermes-agent[otlp]'")
- return None
- from agent.monitoring.emitter import get_emitter
- streamer = OTLPStreamer(config, event_filter=event_filter)
- get_emitter().subscribe(streamer)
- return streamer
- __all__ = [
- "OTLPStreamer",
- "OTLPUnavailable",
- "build_exporter",
- "export_batch",
- "is_available",
- "is_enabled",
- "start_streaming",
- ]
|