incidents.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. """Durable cron failure incidents with signature dedup and ack.
  2. The executions ledger records every attempt; this module groups the *failures* into incidents keyed
  3. by ``(job_id, error signature)`` so the same job failing with the same error does not re-ping the
  4. operator every run once acknowledged. Lifecycle: ``detected`` → ``alerted`` → ``closed``. The same
  5. job + same normalized error resolves to the SAME incident id, so a closed incident stays closed
  6. until the error text changes and mints a new one. ``alerted`` means a failure ping actually reached
  7. the operator (``alerted_at`` = when the latest one did; the scheduler withholds repeats until
  8. ``cron.failure_repeat_alert_hours`` have passed). Incidents share ``cron/executions.db`` with
  9. ``cron.executions`` (one ledger file).
  10. """
  11. from __future__ import annotations
  12. import hashlib
  13. import re
  14. import sqlite3
  15. import threading
  16. from contextlib import contextmanager
  17. from pathlib import Path
  18. from typing import Any, Dict, Iterator, List, Optional
  19. from cron import executions as _executions
  20. from hermes_constants import get_hermes_home
  21. from hermes_time import now as _hermes_now
  22. # Optional test override (mirrors ``cron.executions.EXECUTIONS_FILE``).
  23. EXECUTIONS_FILE: Optional[Path] = None
  24. # ``resolved``: the job ran OK after the failure (auto); ``closed``: the operator acked the signature
  25. # and wants it silent. Only ``closed`` is terminal; a repeat of a resolved error re-opens it.
  26. INCIDENT_STATES = ("detected", "alerted", "resolved", "closed")
  27. _FAILURE_TYPE_ORDER = (
  28. ("rate_limit", (r"\b429\b", "rate limit", "usage limit", "quota")),
  29. ("timeout", ("timeout", "timed out")),
  30. ("auth", (r"\b401\b", "unauthorized", "authentication", "auth")),
  31. ("delivery", ("delivery", "deliver", "delivering")),
  32. ("config", ("config", "configuration", "validation")),
  33. ("script", ("script", "no_agent")),
  34. ("agent", ("agent", "model", "provider", "inference")),
  35. )
  36. MAX_ERROR_CHARS = 500
  37. _MAX_SIGNATURE_ERROR_CHARS = 200
  38. # Measured durations ("idle for 603s", "retry in 12.5 seconds") differ run to run for the same
  39. # failure, so they are masked out of the signature. Status codes and other numbers still count.
  40. _DURATION_RE = re.compile(
  41. r"\b\d+(?:\.\d+)?\s*(?:ms|s|secs?|seconds?|m|mins?|minutes?|h|hrs?|hours?)\b"
  42. )
  43. _lock = threading.RLock()
  44. def _db_path() -> Path:
  45. """Shared cron DB path. The ``cron.executions`` override wins when installed so redirecting the
  46. executions ledger also redirects the incident table (they must stay in the SAME database); then
  47. this module's own override, then the canonical profile home."""
  48. for override in (_executions.EXECUTIONS_FILE, EXECUTIONS_FILE):
  49. if override is not None:
  50. return Path(override)
  51. return get_hermes_home().resolve() / "cron" / "executions.db"
  52. def _connect() -> sqlite3.Connection:
  53. # Late imports: a scheduler daemon that outlives an on-disk upgrade already has the OLD
  54. # ``hermes_cli.sqlite_util`` / ``cron.jobs`` cached, so new names must be resolved at call time,
  55. # not at import time (the guarantee cron/ledger.py used to carry, see e24c8499).
  56. from cron.jobs import _ensure_cron_dir
  57. from hermes_cli.sqlite_util import open_db
  58. path = _db_path()
  59. _ensure_cron_dir(path.parent)
  60. return open_db(path, db_label="cron/executions.db", synchronous_full=True, initialize=_initialize_schema)
  61. def _initialize_schema(conn: sqlite3.Connection) -> None:
  62. from hermes_cli.sqlite_util import add_column_if_missing
  63. conn.execute(
  64. """CREATE TABLE IF NOT EXISTS cron_incidents (
  65. id TEXT PRIMARY KEY,
  66. job_id TEXT NOT NULL,
  67. error_sig TEXT NOT NULL,
  68. state TEXT NOT NULL,
  69. failure_type TEXT NOT NULL DEFAULT 'unknown',
  70. first_seen_at TEXT NOT NULL,
  71. last_seen_at TEXT NOT NULL,
  72. acked_at TEXT,
  73. alerted_at TEXT,
  74. closed_at TEXT,
  75. error TEXT NOT NULL,
  76. output_file TEXT
  77. )"""
  78. )
  79. # Ledgers created before the alert-once gate lack ``alerted_at``; add it in place.
  80. add_column_if_missing(conn, "cron_incidents", "alerted_at", "alerted_at TEXT")
  81. conn.execute(
  82. "CREATE INDEX IF NOT EXISTS idx_cron_incidents_job "
  83. "ON cron_incidents(job_id)"
  84. )
  85. conn.execute(
  86. "CREATE INDEX IF NOT EXISTS idx_cron_incidents_state "
  87. "ON cron_incidents(state)"
  88. )
  89. @contextmanager
  90. def _transaction() -> Iterator[sqlite3.Connection]:
  91. from hermes_cli.sqlite_util import transaction
  92. with _lock, transaction(_connect()) as conn:
  93. yield conn
  94. def _normalize_error(error: str) -> str:
  95. """Strip whitespace and lowercase before signing (dedup normalization)."""
  96. return re.sub(r"\s+", " ", str(error or "")).strip().lower()
  97. def _redact_error(error: str) -> str:
  98. """Redact secrets (best-effort; the scheduler path never fails on it) then bound the length."""
  99. text = str(error or "")
  100. try:
  101. from agent.redact import redact_sensitive_text
  102. text = redact_sensitive_text(text, force=True) # persisted to disk: always scrub
  103. except Exception:
  104. pass
  105. return text[:MAX_ERROR_CHARS]
  106. def _error_signature(job_id: str, error: str) -> str:
  107. """Dedup key: stable for same job + same normalized error prefix."""
  108. normalized = _DURATION_RE.sub("#s", _normalize_error(error))[:_MAX_SIGNATURE_ERROR_CHARS]
  109. return hashlib.sha256(job_id.encode() + normalized.encode()).hexdigest()[:12]
  110. def _incident_id(job_id: str, error_sig: str) -> str:
  111. return f"{job_id[:6]}_{error_sig}"
  112. def _classify_failure_type(error: str) -> str:
  113. """Classify a failure from error-text keywords; ``unknown`` is the default."""
  114. text = _normalize_error(error)
  115. if not text:
  116. return "unknown"
  117. for kind, patterns in _FAILURE_TYPE_ORDER:
  118. for pattern in patterns:
  119. if pattern.startswith("\\b") and pattern.endswith("\\b"):
  120. if re.search(pattern, text):
  121. return kind
  122. elif pattern in text:
  123. return kind
  124. return "unknown"
  125. def upsert_incident(
  126. job_id: str, error: str, *, job_name: Optional[str] = None, failure_type: Optional[str] = None,
  127. output_file: Optional[str] = None,
  128. ) -> tuple[str, bool]:
  129. """Record (or refresh) the incident for ``job_id`` + ``error``; returns ``(incident_id,
  130. is_new)``. An existing row for the signature refreshes
  131. ``last_seen_at``/``error``/``output_file`` and keeps its state — a ``closed`` incident stays
  132. closed, while a ``resolved`` one (job recovered, then broke the same way again) re-opens as
  133. ``detected`` so the operator is alerted once more. A changed error text mints a new incident."""
  134. job_id = str(job_id or "")
  135. sig = _error_signature(job_id, error)
  136. stored_error = _redact_error(error)
  137. incident_id = _incident_id(job_id, sig)
  138. now = _hermes_now().isoformat()
  139. failure_type = failure_type or _classify_failure_type(error)
  140. output_file = str(output_file) if output_file is not None else None
  141. with _transaction() as conn:
  142. row = conn.execute(
  143. "SELECT id, state FROM cron_incidents WHERE id=?", (incident_id,)
  144. ).fetchone()
  145. if row is not None:
  146. reopen = row["state"] == "resolved"
  147. conn.execute(
  148. """UPDATE cron_incidents
  149. SET last_seen_at=?, error=?, output_file=?,
  150. state=CASE WHEN state='resolved' THEN 'detected' ELSE state END,
  151. alerted_at=CASE WHEN state='resolved' THEN NULL ELSE alerted_at END,
  152. closed_at=CASE WHEN state='resolved' THEN NULL ELSE closed_at END
  153. WHERE id=?""",
  154. (now, stored_error, output_file, incident_id),
  155. )
  156. return incident_id, reopen
  157. conn.execute(
  158. """INSERT INTO cron_incidents
  159. (id, job_id, error_sig, state, failure_type,
  160. first_seen_at, last_seen_at, error, output_file)
  161. VALUES (?, ?, ?, 'detected', ?, ?, ?, ?, ?)""",
  162. (incident_id, job_id, sig, failure_type, now, now,
  163. stored_error, output_file),
  164. )
  165. return incident_id, True
  166. def set_incident_state(incident_id: str, state: str) -> bool:
  167. """Transition an incident's lifecycle state; return whether it changed. ``closed`` is terminal
  168. for that signature (re-open happens by a changed error minting a NEW incident). Unknown states
  169. are rejected (no-op, ``False``). ``alerted`` also stamps ``alerted_at`` — every time, so the
  170. cooldown reminder (see ``cron.scheduler._upsert_incident_for_failure``) restarts its window."""
  171. if state not in INCIDENT_STATES:
  172. return False
  173. now = _hermes_now().isoformat()
  174. with _transaction() as conn:
  175. row = conn.execute(
  176. "SELECT state FROM cron_incidents WHERE id=?", (incident_id,)
  177. ).fetchone()
  178. if row is None or row["state"] == "closed":
  179. return False
  180. if state == "alerted":
  181. conn.execute(
  182. "UPDATE cron_incidents SET state='alerted', alerted_at=? WHERE id=?",
  183. (now, incident_id),
  184. )
  185. return True
  186. if row["state"] == state:
  187. return False
  188. if state == "closed":
  189. conn.execute(
  190. """UPDATE cron_incidents
  191. SET state='closed', closed_at=?, acked_at=?
  192. WHERE id=? AND state != 'closed'""",
  193. (now, now, incident_id),
  194. )
  195. else:
  196. conn.execute(
  197. "UPDATE cron_incidents SET state=? WHERE id=?",
  198. (state, incident_id),
  199. )
  200. return True
  201. def ack_incident(incident_id: str) -> bool:
  202. """Acknowledge (close) an incident; ``False`` when missing or already closed."""
  203. return set_incident_state(incident_id, "closed")
  204. def close_incidents_for_recovered_job(job_id: str) -> int:
  205. """Mark every open incident for ``job_id`` ``resolved`` after a successful run; returns how many.
  206. Without this the ledger only ever grows: a one-off failure (a config drift skip, a provider
  207. outage) stayed ``detected``/``alerted`` forever after the job recovered, so ``hermes cron
  208. incidents`` showed dozens of "open" incidents for jobs that had been green for weeks (32 of 32 on
  209. one install). ``resolved`` is distinct from the operator's ``closed`` on purpose: a repeat of the
  210. same error re-opens a resolved incident and alerts again (see ``upsert_incident``), whereas
  211. ``closed`` keeps that signature silent."""
  212. now = _hermes_now().isoformat()
  213. with _transaction() as conn:
  214. cursor = conn.execute(
  215. """UPDATE cron_incidents SET state='resolved', closed_at=?
  216. WHERE job_id=? AND state IN ('detected', 'alerted')""",
  217. (now, str(job_id or "")),
  218. )
  219. return int(cursor.rowcount or 0)
  220. def _state_filter(state: Optional[str]) -> tuple[str, tuple]:
  221. return ("", ()) if state is None else (" WHERE state=?", (state,))
  222. def list_incidents(state: Optional[str] = None) -> list[dict[str, Any]]:
  223. """Return incidents, newest-activity first, optionally filtered by state."""
  224. if state is not None and state not in INCIDENT_STATES:
  225. return []
  226. where, params = _state_filter(state)
  227. with _transaction() as conn:
  228. # last_seen_at carries a DST-varying offset: order by instant, not text.
  229. rows = conn.execute(
  230. "SELECT * FROM cron_incidents" + where
  231. + " ORDER BY julianday(last_seen_at) DESC, last_seen_at DESC, id DESC",
  232. params,
  233. ).fetchall()
  234. return [dict(row) for row in rows]
  235. def get_incident(incident_id: str) -> Optional[dict[str, Any]]:
  236. with _transaction() as conn:
  237. row = conn.execute(
  238. "SELECT * FROM cron_incidents WHERE id=?", (incident_id,)
  239. ).fetchone()
  240. return dict(row) if row is not None else None
  241. def count_incidents(state: Optional[str] = None) -> int:
  242. if state is not None and state not in INCIDENT_STATES:
  243. return 0
  244. where, params = _state_filter(state)
  245. with _transaction() as conn:
  246. row = conn.execute("SELECT COUNT(*) AS n FROM cron_incidents" + where, params).fetchone()
  247. return int(row["n"]) if row is not None else 0