delivery_queue.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. """Profile-local durable handoff for cron delivery through live gateway adapters.
  2. A restart-safe cron worker executes outside the gateway cgroup. It cannot own
  3. relay/E2EE adapter objects, so it queues the final send here. A gateway claims
  4. each row at most once. If that gateway dies after claiming, the outcome is
  5. marked unknown and never retried: losing a delivery is safer than duplicating a
  6. possibly-completed send.
  7. """
  8. from __future__ import annotations
  9. import json
  10. import logging
  11. import os
  12. import sqlite3
  13. import threading
  14. import uuid
  15. from contextlib import contextmanager
  16. from pathlib import Path
  17. from typing import Any, Callable, Iterator, Optional
  18. from agent.deadline import poll_until
  19. from agent.redact import redact_sensitive_text
  20. from cron.executions import _owner_is_live, _process_start_time
  21. from hermes_constants import get_hermes_home
  22. from hermes_time import now as _hermes_now
  23. logger = logging.getLogger(__name__)
  24. DELIVERY_DB: Optional[Path] = None
  25. _PROCESS_ID = uuid.uuid4().hex
  26. _lock = threading.RLock()
  27. _ACTIVE_DELIVERIES: set[str] = set()
  28. _TERMINAL = ("delivered", "failed", "unknown", "suppressed")
  29. MAX_TERMINAL_DELIVERIES = 1000
  30. DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS = 300.0
  31. def _prune_terminal_unlocked(conn: sqlite3.Connection) -> None:
  32. """Redact terminal payloads and retain only bounded outcome metadata."""
  33. conn.execute(
  34. """UPDATE deliveries SET job_json='{}', content=''
  35. WHERE status IN ('delivered','failed','unknown','suppressed')
  36. AND (job_json != '{}' OR content != '')"""
  37. )
  38. keep = max(0, int(MAX_TERMINAL_DELIVERIES))
  39. terminal_count = int(
  40. conn.execute(
  41. "SELECT COUNT(*) FROM deliveries "
  42. "WHERE status IN ('delivered','failed','unknown','suppressed')"
  43. ).fetchone()[0]
  44. )
  45. excess = terminal_count - keep
  46. if excess > 0:
  47. conn.execute(
  48. """INSERT OR IGNORE INTO delivery_tombstones
  49. (execution_id, terminal_status, finished_at)
  50. SELECT execution_id, status, finished_at FROM deliveries
  51. WHERE status IN ('delivered','failed','unknown','suppressed')
  52. ORDER BY julianday(finished_at), finished_at,
  53. julianday(created_at), created_at, execution_id
  54. LIMIT ?""",
  55. (excess,),
  56. )
  57. conn.execute(
  58. """DELETE FROM deliveries WHERE execution_id IN (
  59. SELECT execution_id FROM deliveries
  60. WHERE status IN ('delivered','failed','unknown','suppressed')
  61. ORDER BY julianday(finished_at), finished_at,
  62. julianday(created_at), created_at, execution_id
  63. LIMIT ?
  64. )""",
  65. (excess,),
  66. )
  67. def queue_path(home: Optional[Path] = None) -> Path:
  68. """The queue file of ``home`` (the active home when None); a test override wins."""
  69. if DELIVERY_DB is not None:
  70. return DELIVERY_DB
  71. root = Path(home) if home is not None else get_hermes_home()
  72. return root.resolve() / "cron" / "deliveries.db"
  73. def _path() -> Path:
  74. return queue_path()
  75. def _initialize_schema(conn: sqlite3.Connection) -> None:
  76. from hermes_cli.sqlite_util import add_column_if_missing
  77. # SQLite cannot widen a CHECK in place. Preserve all old rows atomically,
  78. # including claimed sends, while admitting a distinct never-sent disposition.
  79. for table in ("deliveries", "delivery_tombstones"):
  80. row = conn.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
  81. if row and "'suppressed'" not in row[0]:
  82. conn.execute("SAVEPOINT notification_disposition")
  83. try:
  84. conn.execute(f"ALTER TABLE {table} RENAME TO {table}_old")
  85. conn.execute(row[0].replace("'unknown'", "'unknown','suppressed'"))
  86. conn.execute(f"INSERT INTO {table} SELECT * FROM {table}_old")
  87. conn.execute(f"DROP TABLE {table}_old")
  88. conn.execute("RELEASE notification_disposition")
  89. except BaseException:
  90. conn.execute("ROLLBACK TO notification_disposition")
  91. conn.execute("RELEASE notification_disposition")
  92. raise
  93. conn.execute(
  94. """CREATE TABLE IF NOT EXISTS deliveries (
  95. execution_id TEXT PRIMARY KEY,
  96. job_json TEXT NOT NULL,
  97. content TEXT NOT NULL,
  98. for_failure INTEGER NOT NULL DEFAULT 0,
  99. status TEXT NOT NULL CHECK(status IN
  100. ('pending','delivering','delivered','failed','unknown','suppressed')),
  101. owner_process_id TEXT,
  102. owner_pid INTEGER,
  103. owner_started_at INTEGER,
  104. created_at TEXT NOT NULL,
  105. finished_at TEXT,
  106. error TEXT
  107. )"""
  108. )
  109. conn.execute(
  110. """CREATE TABLE IF NOT EXISTS delivery_tombstones (
  111. execution_id TEXT PRIMARY KEY,
  112. terminal_status TEXT NOT NULL CHECK(terminal_status IN
  113. ('delivered','failed','unknown','suppressed')),
  114. finished_at TEXT
  115. )"""
  116. )
  117. add_column_if_missing(
  118. conn, "deliveries", "for_failure",
  119. "for_failure INTEGER NOT NULL DEFAULT 0",
  120. )
  121. def _connect() -> sqlite3.Connection:
  122. # Late imports: a scheduler daemon that outlives an on-disk upgrade already has the OLD
  123. # ``hermes_cli.sqlite_util`` / ``cron.jobs`` cached, so new names must be resolved at call time,
  124. # not at import time (the guarantee cron/ledger.py used to carry, see e24c8499).
  125. from hermes_cli.sqlite_util import open_db
  126. path = _path()
  127. conn = open_db(path, db_label="cron/deliveries.db", synchronous_full=True, initialize=_initialize_schema)
  128. try:
  129. path.chmod(0o600)
  130. except OSError:
  131. pass
  132. return conn
  133. @contextmanager
  134. def _transaction() -> Iterator[sqlite3.Connection]:
  135. # Pruning is done explicitly by the paths that create terminal
  136. # rows (_finish / recover_abandoned / _terminalize_wait_timeout);
  137. # read-only polls must not pay for a full-table UPDATE + COUNT.
  138. from hermes_cli.sqlite_util import transaction
  139. with _lock, transaction(_connect()) as conn:
  140. yield conn
  141. def enqueue(
  142. execution_id: str,
  143. job: dict,
  144. content: str,
  145. *,
  146. for_failure: bool = False,
  147. ) -> dict:
  148. """Persist one idempotent delivery request before the worker waits."""
  149. with _transaction() as conn:
  150. # Serialize the tombstone check and insert with retention in other
  151. # processes, which can move a terminal delivery into the tombstone table.
  152. conn.execute("BEGIN IMMEDIATE")
  153. tombstone = conn.execute(
  154. "SELECT terminal_status, finished_at FROM delivery_tombstones "
  155. "WHERE execution_id=?",
  156. (str(execution_id),),
  157. ).fetchone()
  158. if tombstone is not None:
  159. return {
  160. "execution_id": str(execution_id),
  161. "status": tombstone["terminal_status"],
  162. "finished_at": tombstone["finished_at"],
  163. }
  164. conn.execute(
  165. """INSERT OR IGNORE INTO deliveries
  166. (execution_id, job_json, content, for_failure, status, created_at)
  167. VALUES (?, ?, ?, ?, 'pending', ?)""",
  168. (
  169. str(execution_id),
  170. json.dumps(job, ensure_ascii=False, sort_keys=True),
  171. str(content),
  172. int(bool(for_failure)),
  173. _hermes_now().isoformat(),
  174. ),
  175. )
  176. row = conn.execute(
  177. "SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
  178. ).fetchone()
  179. return dict(row)
  180. def get_status(execution_id: str) -> Optional[dict]:
  181. with _transaction() as conn:
  182. row = conn.execute(
  183. "SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
  184. ).fetchone()
  185. if row is not None:
  186. return dict(row)
  187. tombstone = conn.execute(
  188. "SELECT execution_id, terminal_status, finished_at "
  189. "FROM delivery_tombstones WHERE execution_id=?",
  190. (str(execution_id),),
  191. ).fetchone()
  192. if tombstone is None:
  193. return None
  194. return {
  195. "execution_id": tombstone["execution_id"],
  196. "status": tombstone["terminal_status"],
  197. "finished_at": tombstone["finished_at"],
  198. "error": None,
  199. }
  200. def claim_next() -> Optional[dict]:
  201. """Atomically claim one pending send before touching the transport."""
  202. pid = os.getpid()
  203. started = _process_start_time(pid)
  204. with _transaction() as conn:
  205. # created_at carries a DST-varying offset: order by instant, not text.
  206. row = conn.execute(
  207. "SELECT execution_id FROM deliveries WHERE status='pending' "
  208. "ORDER BY julianday(created_at), created_at, execution_id LIMIT 1"
  209. ).fetchone()
  210. if row is None:
  211. return None
  212. cur = conn.execute(
  213. """UPDATE deliveries SET status='delivering', owner_process_id=?,
  214. owner_pid=?, owner_started_at=?
  215. WHERE execution_id=? AND status='pending'""",
  216. (_PROCESS_ID, pid, started, row["execution_id"]),
  217. )
  218. if cur.rowcount != 1:
  219. return None
  220. claimed = conn.execute(
  221. "SELECT * FROM deliveries WHERE execution_id=?", (row["execution_id"],)
  222. ).fetchone()
  223. _ACTIVE_DELIVERIES.add(row["execution_id"])
  224. result = dict(claimed)
  225. result["job"] = json.loads(result.pop("job_json"))
  226. return result
  227. def _finish(execution_id: str, *, error: Optional[str], suppressed: bool = False) -> bool:
  228. status = "failed" if error else "suppressed" if suppressed else "delivered"
  229. safe_error = (
  230. redact_sensitive_text(str(error), force=True, redact_url_credentials=True)
  231. if error
  232. else None
  233. )
  234. with _transaction() as conn:
  235. cur = conn.execute(
  236. """UPDATE deliveries SET status=?, finished_at=?, error=?
  237. WHERE execution_id=? AND status='delivering'
  238. AND owner_process_id=? AND owner_pid=?""",
  239. (
  240. status,
  241. _hermes_now().isoformat(),
  242. safe_error,
  243. execution_id,
  244. _PROCESS_ID,
  245. os.getpid(),
  246. ),
  247. )
  248. _prune_terminal_unlocked(conn)
  249. return cur.rowcount == 1
  250. def recover_abandoned() -> int:
  251. """Fence dead delivery owners as unknown; never replay uncertain sends.
  252. The firer settled long ago against a ``queued`` row, so each fenced row's outcome is written
  253. back here (job ``delivery_failed`` + error, ledger ``failed``) instead of leaving the run at
  254. ``delivery_queued`` forever."""
  255. fenced: list[tuple[Optional[str], str]] = []
  256. with _transaction() as conn:
  257. rows = conn.execute(
  258. "SELECT execution_id, job_json, owner_process_id, owner_pid, owner_started_at "
  259. "FROM deliveries WHERE status='delivering'"
  260. ).fetchall()
  261. for row in rows:
  262. same_process = row["owner_process_id"] == _PROCESS_ID
  263. if same_process:
  264. with _lock:
  265. if row["execution_id"] in _ACTIVE_DELIVERIES:
  266. continue
  267. elif _owner_is_live(int(row["owner_pid"]), row["owner_started_at"]):
  268. continue
  269. error = (
  270. "Gateway finished delivery but could not persist its outcome; "
  271. "send was not retried."
  272. if same_process
  273. else "Gateway exited during delivery; send outcome is unknown and was not retried."
  274. )
  275. cur = conn.execute(
  276. """UPDATE deliveries SET status='unknown', finished_at=?, error=?
  277. WHERE execution_id=? AND status='delivering'""",
  278. (
  279. _hermes_now().isoformat(),
  280. error,
  281. row["execution_id"],
  282. ),
  283. )
  284. if cur.rowcount:
  285. # Read the job id now: pruning below redacts a terminal row's job_json.
  286. fenced.append((json.loads(row["job_json"] or "{}").get("id"), row["execution_id"]))
  287. _prune_terminal_unlocked(conn)
  288. from cron.delivery_outcome import settle_quietly
  289. for job_id, execution_id in fenced:
  290. settle_quietly(job_id, execution_id)
  291. return len(fenced)
  292. def drain(
  293. send: Callable[[dict, str, bool], Optional[str]], *, limit: int = 20
  294. ) -> int:
  295. """Deliver pending rows through *send*, terminalizing every claimed row."""
  296. recover_abandoned()
  297. processed = 0
  298. for _ in range(max(0, limit)):
  299. row = claim_next()
  300. if row is None:
  301. break
  302. with _lock:
  303. _ACTIVE_DELIVERIES.add(row["execution_id"])
  304. try:
  305. try:
  306. error = send(
  307. row["job"], row["content"], bool(row["for_failure"])
  308. )
  309. except BaseException as exc:
  310. error = f"{type(exc).__name__}: {exc}"
  311. if _finish(row["execution_id"], error=error,
  312. suppressed=bool(row["job"].get("_notification_all_targets_suppressed"))):
  313. from cron.delivery_outcome import settle_quietly
  314. settle_quietly(row["job"].get("id"), row["execution_id"])
  315. finally:
  316. with _lock:
  317. _ACTIVE_DELIVERIES.discard(row["execution_id"])
  318. processed += 1
  319. return processed
  320. def _terminalize_wait_timeout(execution_id: str) -> str:
  321. """Fence a delivery whose worker can no longer wait for confirmation.
  322. A row still ``pending`` was provably never attempted, so it is left queued
  323. for whichever gateway comes up next (a restart that includes an update can
  324. easily exceed the worker's wait budget). That is a deferral, not a
  325. failure: report success so the job is not recorded ``delivery_failed`` for
  326. a message the drain will still send. Only a row caught mid-send is
  327. uncertain and gets fenced ``unknown``.
  328. """
  329. now = _hermes_now().isoformat()
  330. uncertain_error = (
  331. "timed out while gateway delivery was in progress; outcome is unknown and "
  332. "was not retried"
  333. )
  334. with _transaction() as conn:
  335. row = conn.execute(
  336. "SELECT status FROM deliveries WHERE execution_id=?",
  337. (str(execution_id),),
  338. ).fetchone()
  339. if row is not None and row["status"] == "pending":
  340. logger.warning(
  341. "Cron delivery %s: no live gateway within the wait budget; "
  342. "left queued for the next gateway",
  343. execution_id,
  344. )
  345. return ""
  346. conn.execute(
  347. """UPDATE deliveries SET status='unknown', finished_at=?, error=?
  348. WHERE execution_id=? AND status='delivering'""",
  349. (now, uncertain_error, str(execution_id)),
  350. )
  351. row = conn.execute(
  352. "SELECT status, error FROM deliveries WHERE execution_id=?",
  353. (str(execution_id),),
  354. ).fetchone()
  355. _prune_terminal_unlocked(conn)
  356. if row is None:
  357. return "timed out waiting for live gateway delivery"
  358. if row["status"] in {"delivered", "suppressed"}:
  359. return ""
  360. return str(row["error"] or f"delivery {row['status']}")
  361. def enqueue_and_wait(
  362. execution_id: str,
  363. job: dict,
  364. content: str,
  365. *,
  366. for_failure: bool = False,
  367. timeout: Optional[float] = None,
  368. ) -> Optional[str]:
  369. """Queue delivery and wait for a gateway's terminal at-most-once outcome."""
  370. queued = enqueue(execution_id, job, content, for_failure=for_failure)
  371. if queued["status"] in _TERMINAL:
  372. return None if queued["status"] in {"delivered", "suppressed"} else str(
  373. queued.get("error") or f"delivery {queued['status']}"
  374. )
  375. wait_timeout = (
  376. DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS if timeout is None else max(0.0, timeout)
  377. )
  378. def _terminal_row() -> Optional[dict]:
  379. row = get_status(execution_id)
  380. return row if row and row["status"] in _TERMINAL else None
  381. row = poll_until(_terminal_row, wait_timeout, 1.0)
  382. if row is None:
  383. return _terminalize_wait_timeout(execution_id) or None
  384. return None if row["status"] in {"delivered", "suppressed"} else str(
  385. row.get("error") or f"delivery {row['status']}"
  386. )