cron_health.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180
  1. """Content-free cron service-health and execution telemetry projection."""
  2. from __future__ import annotations
  3. import logging
  4. import re
  5. from dataclasses import dataclass
  6. from datetime import datetime
  7. from typing import Any, Callable, Optional
  8. from agent.monitoring.events import CronExecutionEvent
  9. from agent.monitoring.gateway_health import GatewayMetric, _contains_any, _safe_instance_id
  10. from cron.jobs import (
  11. _compute_grace_seconds,
  12. get_ticker_heartbeat_age,
  13. get_ticker_success_age,
  14. load_jobs,
  15. )
  16. from cron.occurrences import get_catch_up_occurrence_count
  17. from cron.scheduler import get_running_job_ids
  18. from cron.store_health import degraded_records
  19. from hermes_time import now as _now
  20. logger = logging.getLogger(__name__)
  21. # ``cancelled``: an operator stopped the run; ``missed``: the scheduler skipped past its slots.
  22. _KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown", "cancelled", "missed"}
  23. _KNOWN_SOURCES = {"builtin", "direct", "external"}
  24. _KNOWN_DELIVERY_OUTCOMES = {"queued", "delivered", "failed", "suppressed", "suppressed_acked", "not_configured"}
  25. _TERMINAL_STATUSES = {"completed", "failed", "unknown", "cancelled", "missed"}
  26. @dataclass(frozen=True, slots=True)
  27. class CronHealthSnapshot:
  28. metrics: list[GatewayMetric]
  29. events: list[CronExecutionEvent]
  30. # Opaque job key: same sha256:<24 hex> shape as service.instance.id — never the raw id.
  31. _job_key = _safe_instance_id
  32. _AUTH_RE = re.compile(
  33. r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden|bearer|401|403)\b"
  34. r"|\b(?:access|api|refresh) token\b"
  35. )
  36. # Ordered (predicate, class) rules; first match wins. Auth uses word boundaries so
  37. # "oauth"/"tokenizer"/"HTTP 4015" do not false-positive.
  38. _CRON_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = (
  39. (lambda s: _AUTH_RE.search(s) is not None, "auth_failed"),
  40. (_contains_any("rate limit", "429", "quota"), "rate_limited"),
  41. (_contains_any("timeout", "timed out"), "timeout"),
  42. (_contains_any("network", "connection", "dns", "socket", "unreachable"), "network_error"),
  43. (_contains_any("dispatch", "executor"), "dispatch_failed"),
  44. (_contains_any("interrupt", "owner exited", "restarted"), "interrupted"),
  45. (_contains_any("empty response"), "empty_response"),
  46. (_contains_any("config", "missing", "invalid"), "invalid_config"),
  47. )
  48. def classify_cron_error(raw: Any) -> str:
  49. text = str(raw or "").lower()
  50. return next((label for match, label in _CRON_ERROR_RULES if match(text)), "unknown")
  51. def _parse_time(raw: Any) -> Optional[datetime]:
  52. try:
  53. return datetime.fromisoformat(str(raw)) if raw else None
  54. except (TypeError, ValueError):
  55. return None
  56. def _duration_ms(record: dict[str, Any]) -> Optional[int]:
  57. start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at"))
  58. finish = _parse_time(record.get("finished_at"))
  59. if start is None or finish is None:
  60. return None
  61. try:
  62. duration = int((finish - start).total_seconds() * 1000)
  63. except (TypeError, ValueError):
  64. return None
  65. return max(0, duration)
  66. def project_execution_event(record: dict[str, Any], *, delivery_outcome: Optional[str] = None) -> CronExecutionEvent:
  67. status = str(record.get("status") or "unknown").lower()
  68. source = str(record.get("source") or "unknown").lower()
  69. outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None
  70. return CronExecutionEvent(
  71. status=status if status in _KNOWN_STATUSES else "unknown",
  72. job_key=_job_key(record.get("job_id")),
  73. # Unknown non-empty sources are bucketed as "external" (not dropped to unknown).
  74. source=source if source in _KNOWN_SOURCES or source == "unknown" else "external",
  75. duration_ms=_duration_ms(record),
  76. delivery_outcome=outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None,
  77. error_class=classify_cron_error(record.get("error")) if status in {"failed", "unknown"} else None,
  78. )
  79. def emit_execution_state(record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None) -> None:
  80. """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier."""
  81. if not record:
  82. return
  83. try:
  84. from agent.monitoring import emitter
  85. event = project_execution_event(record, delivery_outcome=delivery_outcome)
  86. target = emitter.get_emitter()
  87. target.emit(event)
  88. if event.status in _TERMINAL_STATUSES:
  89. target.flush(timeout=1.0)
  90. except Exception:
  91. logger.debug("cron execution telemetry emit failed", exc_info=True)
  92. def _is_overdue(job: dict[str, Any], now: datetime) -> bool:
  93. if not job.get("enabled", True):
  94. return False
  95. next_run = _parse_time(job.get("next_run_at"))
  96. schedule = job.get("schedule")
  97. if next_run is None or not isinstance(schedule, dict):
  98. return False
  99. try:
  100. if next_run.tzinfo is None and now.tzinfo is not None:
  101. next_run = next_run.replace(tzinfo=now.tzinfo)
  102. # A per-job grace override moves the scheduler's own overdue line; the metric must
  103. # agree with it or the dashboard flags jobs the ticker considers on time.
  104. return (now - next_run).total_seconds() > _compute_grace_seconds(
  105. schedule, job.get("grace_minutes"))
  106. except (TypeError, ValueError):
  107. return False
  108. def _job_metrics(metrics: list[GatewayMetric]) -> None:
  109. enabled = [job for job in load_jobs() if job.get("enabled", True)]
  110. metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {}))
  111. metrics.append(GatewayMetric("hermes.cron.jobs.overdue", sum(1 for job in enabled if _is_overdue(job, _now())), {}))
  112. def _freshness_metric(name: str, reader: Callable[[], Optional[float]]) -> Callable[[list[GatewayMetric]], None]:
  113. def build(metrics: list[GatewayMetric]) -> None:
  114. value = reader()
  115. if value is not None:
  116. metrics.append(GatewayMetric(name, max(0.0, float(value)), {}))
  117. return build
  118. def _single_metric(name: str, reader: Callable[[], Any]) -> Callable[[list[GatewayMetric]], None]:
  119. return lambda metrics: metrics.append(GatewayMetric(name, reader(), {}))
  120. # Each group is independently fail-open so one unavailable source never hides the rest.
  121. # Readers are wrapped in lambdas so monkeypatching this module's names still takes effect.
  122. _METRIC_GROUPS: tuple[tuple[Callable[[list[GatewayMetric]], None], str], ...] = (
  123. (_freshness_metric("hermes.cron.scheduler.heartbeat_age_seconds", lambda: get_ticker_heartbeat_age()), "cron freshness metric unavailable"),
  124. (_freshness_metric("hermes.cron.scheduler.last_success_age_seconds", lambda: get_ticker_success_age()), "cron freshness metric unavailable"),
  125. (_single_metric("hermes.cron.scheduler.catch_up_occurrences", lambda: get_catch_up_occurrence_count()), "cron catch-up metric unavailable"),
  126. (_job_metrics, "cron job metrics unavailable"),
  127. (_single_metric("hermes.cron.jobs.running", lambda: len(get_running_job_ids())), "cron running-job metric unavailable"),
  128. # Host-wide: the exporter runs in the launch profile's scope, but every served profile's ticker
  129. # shares this process, so a secondary profile's outage must not read as healthy.
  130. (_single_metric("hermes.cron.store.writable", lambda: int(not degraded_records())), "cron store metric unavailable"),
  131. (_single_metric("hermes.cron.store.skipped_runs", lambda: sum(r.skipped_runs for r in degraded_records())),
  132. "cron store metric unavailable"),
  133. )
  134. def build_cron_health_snapshot() -> CronHealthSnapshot:
  135. metrics: list[GatewayMetric] = []
  136. for build, failure_msg in _METRIC_GROUPS:
  137. try:
  138. build(metrics)
  139. except Exception:
  140. logger.debug(failure_msg, exc_info=True)
  141. return CronHealthSnapshot(metrics=metrics, events=[])
  142. __all__ = [
  143. "CronHealthSnapshot", "build_cron_health_snapshot", "classify_cron_error", "emit_execution_state",
  144. "project_execution_event",
  145. ]