| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293 |
- """Durable cron failure incidents with signature dedup and ack.
- The executions ledger records every attempt; this module groups the *failures* into incidents keyed
- by ``(job_id, error signature)`` so the same job failing with the same error does not re-ping the
- operator every run once acknowledged. Lifecycle: ``detected`` → ``alerted`` → ``closed``. The same
- job + same normalized error resolves to the SAME incident id, so a closed incident stays closed
- until the error text changes and mints a new one. ``alerted`` means a failure ping actually reached
- the operator (``alerted_at`` = when the latest one did; the scheduler withholds repeats until
- ``cron.failure_repeat_alert_hours`` have passed). Incidents share ``cron/executions.db`` with
- ``cron.executions`` (one ledger file).
- """
- from __future__ import annotations
- import hashlib
- import re
- import sqlite3
- import threading
- from contextlib import contextmanager
- from pathlib import Path
- from typing import Any, Dict, Iterator, List, Optional
- from cron import executions as _executions
- from hermes_constants import get_hermes_home
- from hermes_time import now as _hermes_now
- # Optional test override (mirrors ``cron.executions.EXECUTIONS_FILE``).
- EXECUTIONS_FILE: Optional[Path] = None
- # ``resolved``: the job ran OK after the failure (auto); ``closed``: the operator acked the signature
- # and wants it silent. Only ``closed`` is terminal; a repeat of a resolved error re-opens it.
- INCIDENT_STATES = ("detected", "alerted", "resolved", "closed")
- _FAILURE_TYPE_ORDER = (
- ("rate_limit", (r"\b429\b", "rate limit", "usage limit", "quota")),
- ("timeout", ("timeout", "timed out")),
- ("auth", (r"\b401\b", "unauthorized", "authentication", "auth")),
- ("delivery", ("delivery", "deliver", "delivering")),
- ("config", ("config", "configuration", "validation")),
- ("script", ("script", "no_agent")),
- ("agent", ("agent", "model", "provider", "inference")),
- )
- MAX_ERROR_CHARS = 500
- _MAX_SIGNATURE_ERROR_CHARS = 200
- # Measured durations ("idle for 603s", "retry in 12.5 seconds") differ run to run for the same
- # failure, so they are masked out of the signature. Status codes and other numbers still count.
- _DURATION_RE = re.compile(
- r"\b\d+(?:\.\d+)?\s*(?:ms|s|secs?|seconds?|m|mins?|minutes?|h|hrs?|hours?)\b"
- )
- _lock = threading.RLock()
- def _db_path() -> Path:
- """Shared cron DB path. The ``cron.executions`` override wins when installed so redirecting the
- executions ledger also redirects the incident table (they must stay in the SAME database); then
- this module's own override, then the canonical profile home."""
- for override in (_executions.EXECUTIONS_FILE, EXECUTIONS_FILE):
- if override is not None:
- return Path(override)
- return get_hermes_home().resolve() / "cron" / "executions.db"
- def _connect() -> sqlite3.Connection:
- # Late imports: a scheduler daemon that outlives an on-disk upgrade already has the OLD
- # ``hermes_cli.sqlite_util`` / ``cron.jobs`` cached, so new names must be resolved at call time,
- # not at import time (the guarantee cron/ledger.py used to carry, see e24c8499).
- from cron.jobs import _ensure_cron_dir
- from hermes_cli.sqlite_util import open_db
- path = _db_path()
- _ensure_cron_dir(path.parent)
- return open_db(path, db_label="cron/executions.db", synchronous_full=True, initialize=_initialize_schema)
- def _initialize_schema(conn: sqlite3.Connection) -> None:
- from hermes_cli.sqlite_util import add_column_if_missing
- conn.execute(
- """CREATE TABLE IF NOT EXISTS cron_incidents (
- id TEXT PRIMARY KEY,
- job_id TEXT NOT NULL,
- error_sig TEXT NOT NULL,
- state TEXT NOT NULL,
- failure_type TEXT NOT NULL DEFAULT 'unknown',
- first_seen_at TEXT NOT NULL,
- last_seen_at TEXT NOT NULL,
- acked_at TEXT,
- alerted_at TEXT,
- closed_at TEXT,
- error TEXT NOT NULL,
- output_file TEXT
- )"""
- )
- # Ledgers created before the alert-once gate lack ``alerted_at``; add it in place.
- add_column_if_missing(conn, "cron_incidents", "alerted_at", "alerted_at TEXT")
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_cron_incidents_job "
- "ON cron_incidents(job_id)"
- )
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_cron_incidents_state "
- "ON cron_incidents(state)"
- )
- @contextmanager
- def _transaction() -> Iterator[sqlite3.Connection]:
- from hermes_cli.sqlite_util import transaction
- with _lock, transaction(_connect()) as conn:
- yield conn
- def _normalize_error(error: str) -> str:
- """Strip whitespace and lowercase before signing (dedup normalization)."""
- return re.sub(r"\s+", " ", str(error or "")).strip().lower()
- def _redact_error(error: str) -> str:
- """Redact secrets (best-effort; the scheduler path never fails on it) then bound the length."""
- text = str(error or "")
- try:
- from agent.redact import redact_sensitive_text
- text = redact_sensitive_text(text, force=True) # persisted to disk: always scrub
- except Exception:
- pass
- return text[:MAX_ERROR_CHARS]
- def _error_signature(job_id: str, error: str) -> str:
- """Dedup key: stable for same job + same normalized error prefix."""
- normalized = _DURATION_RE.sub("#s", _normalize_error(error))[:_MAX_SIGNATURE_ERROR_CHARS]
- return hashlib.sha256(job_id.encode() + normalized.encode()).hexdigest()[:12]
- def _incident_id(job_id: str, error_sig: str) -> str:
- return f"{job_id[:6]}_{error_sig}"
- def _classify_failure_type(error: str) -> str:
- """Classify a failure from error-text keywords; ``unknown`` is the default."""
- text = _normalize_error(error)
- if not text:
- return "unknown"
- for kind, patterns in _FAILURE_TYPE_ORDER:
- for pattern in patterns:
- if pattern.startswith("\\b") and pattern.endswith("\\b"):
- if re.search(pattern, text):
- return kind
- elif pattern in text:
- return kind
- return "unknown"
- def upsert_incident(
- job_id: str, error: str, *, job_name: Optional[str] = None, failure_type: Optional[str] = None,
- output_file: Optional[str] = None,
- ) -> tuple[str, bool]:
- """Record (or refresh) the incident for ``job_id`` + ``error``; returns ``(incident_id,
- is_new)``. An existing row for the signature refreshes
- ``last_seen_at``/``error``/``output_file`` and keeps its state — a ``closed`` incident stays
- closed, while a ``resolved`` one (job recovered, then broke the same way again) re-opens as
- ``detected`` so the operator is alerted once more. A changed error text mints a new incident."""
- job_id = str(job_id or "")
- sig = _error_signature(job_id, error)
- stored_error = _redact_error(error)
- incident_id = _incident_id(job_id, sig)
- now = _hermes_now().isoformat()
- failure_type = failure_type or _classify_failure_type(error)
- output_file = str(output_file) if output_file is not None else None
- with _transaction() as conn:
- row = conn.execute(
- "SELECT id, state FROM cron_incidents WHERE id=?", (incident_id,)
- ).fetchone()
- if row is not None:
- reopen = row["state"] == "resolved"
- conn.execute(
- """UPDATE cron_incidents
- SET last_seen_at=?, error=?, output_file=?,
- state=CASE WHEN state='resolved' THEN 'detected' ELSE state END,
- alerted_at=CASE WHEN state='resolved' THEN NULL ELSE alerted_at END,
- closed_at=CASE WHEN state='resolved' THEN NULL ELSE closed_at END
- WHERE id=?""",
- (now, stored_error, output_file, incident_id),
- )
- return incident_id, reopen
- conn.execute(
- """INSERT INTO cron_incidents
- (id, job_id, error_sig, state, failure_type,
- first_seen_at, last_seen_at, error, output_file)
- VALUES (?, ?, ?, 'detected', ?, ?, ?, ?, ?)""",
- (incident_id, job_id, sig, failure_type, now, now,
- stored_error, output_file),
- )
- return incident_id, True
- def set_incident_state(incident_id: str, state: str) -> bool:
- """Transition an incident's lifecycle state; return whether it changed. ``closed`` is terminal
- for that signature (re-open happens by a changed error minting a NEW incident). Unknown states
- are rejected (no-op, ``False``). ``alerted`` also stamps ``alerted_at`` — every time, so the
- cooldown reminder (see ``cron.scheduler._upsert_incident_for_failure``) restarts its window."""
- if state not in INCIDENT_STATES:
- return False
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- row = conn.execute(
- "SELECT state FROM cron_incidents WHERE id=?", (incident_id,)
- ).fetchone()
- if row is None or row["state"] == "closed":
- return False
- if state == "alerted":
- conn.execute(
- "UPDATE cron_incidents SET state='alerted', alerted_at=? WHERE id=?",
- (now, incident_id),
- )
- return True
- if row["state"] == state:
- return False
- if state == "closed":
- conn.execute(
- """UPDATE cron_incidents
- SET state='closed', closed_at=?, acked_at=?
- WHERE id=? AND state != 'closed'""",
- (now, now, incident_id),
- )
- else:
- conn.execute(
- "UPDATE cron_incidents SET state=? WHERE id=?",
- (state, incident_id),
- )
- return True
- def ack_incident(incident_id: str) -> bool:
- """Acknowledge (close) an incident; ``False`` when missing or already closed."""
- return set_incident_state(incident_id, "closed")
- def close_incidents_for_recovered_job(job_id: str) -> int:
- """Mark every open incident for ``job_id`` ``resolved`` after a successful run; returns how many.
- Without this the ledger only ever grows: a one-off failure (a config drift skip, a provider
- outage) stayed ``detected``/``alerted`` forever after the job recovered, so ``hermes cron
- incidents`` showed dozens of "open" incidents for jobs that had been green for weeks (32 of 32 on
- one install). ``resolved`` is distinct from the operator's ``closed`` on purpose: a repeat of the
- same error re-opens a resolved incident and alerts again (see ``upsert_incident``), whereas
- ``closed`` keeps that signature silent."""
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- cursor = conn.execute(
- """UPDATE cron_incidents SET state='resolved', closed_at=?
- WHERE job_id=? AND state IN ('detected', 'alerted')""",
- (now, str(job_id or "")),
- )
- return int(cursor.rowcount or 0)
- def _state_filter(state: Optional[str]) -> tuple[str, tuple]:
- return ("", ()) if state is None else (" WHERE state=?", (state,))
- def list_incidents(state: Optional[str] = None) -> list[dict[str, Any]]:
- """Return incidents, newest-activity first, optionally filtered by state."""
- if state is not None and state not in INCIDENT_STATES:
- return []
- where, params = _state_filter(state)
- with _transaction() as conn:
- # last_seen_at carries a DST-varying offset: order by instant, not text.
- rows = conn.execute(
- "SELECT * FROM cron_incidents" + where
- + " ORDER BY julianday(last_seen_at) DESC, last_seen_at DESC, id DESC",
- params,
- ).fetchall()
- return [dict(row) for row in rows]
- def get_incident(incident_id: str) -> Optional[dict[str, Any]]:
- with _transaction() as conn:
- row = conn.execute(
- "SELECT * FROM cron_incidents WHERE id=?", (incident_id,)
- ).fetchone()
- return dict(row) if row is not None else None
- def count_incidents(state: Optional[str] = None) -> int:
- if state is not None and state not in INCIDENT_STATES:
- return 0
- where, params = _state_filter(state)
- with _transaction() as conn:
- row = conn.execute("SELECT COUNT(*) AS n FROM cron_incidents" + where, params).fetchone()
- return int(row["n"]) if row is not None else 0
|