delivery_outcome.py 4.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394
  1. """Write a queued cron delivery's terminal outcome back onto the run that queued it.
  2. An agent run hands its send to ``cron.delivery_queue`` and books ``last_status =
  3. "delivery_queued"`` / ledger ``delivery_outcome = "queued"``. Once a gateway drain sends,
  4. refuses, suppresses or fences the send ``unknown``, the job record and the execution
  5. ledger must carry the real result (main's inline ``ok``/``delivered`` and
  6. ``delivery_failed``/``failed`` + ``last_delivery_error``); otherwise ``hermes cron list``
  7. says "delivery still in progress" forever and refused sends never surface.
  8. Both sides call :func:`settle_queued_delivery`: the drain right after it terminalizes a
  9. row, and the run right after its own bookkeeping (a drain can finish before the run's
  10. ``mark_job_run`` lands). Every write is fenced on the execution id, so the call is
  11. idempotent and never rewrites a later run's status.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. from typing import Optional
  16. logger = logging.getLogger(__name__)
  17. # Queue terminal status -> ledger delivery_outcome. ``unknown`` (owner died mid-send, never
  18. # retried) is surfaced as a failure: the user cannot be told the result arrived.
  19. _LEDGER_OUTCOME = {
  20. "delivered": "delivered", "suppressed": "suppressed", "failed": "failed", "unknown": "failed",
  21. }
  22. def _settle_job_record(job_id: str, execution_id: str, delivery_error: Optional[str]) -> bool:
  23. """Stamp the outcome onto the job only while *execution_id* is still its current run:
  24. the ledger's latest attempt for the job AND the job's latest recorded completion."""
  25. from cron.executions import latest_execution
  26. from cron.jobs import _with_job, save_jobs
  27. latest = latest_execution(job_id)
  28. if latest is None or latest.get("id") != execution_id:
  29. return False
  30. def apply(jobs, _i, job):
  31. completions = job.get("canonical_completions") or []
  32. if not completions or completions[-1] != execution_id:
  33. return False
  34. status = job.get("last_status")
  35. if status == "delivery_queued":
  36. status = "delivery_failed" if delivery_error else "ok"
  37. if (status, delivery_error) == (job.get("last_status"), job.get("last_delivery_error")):
  38. return True
  39. job["last_status"] = status
  40. job["last_delivery_error"] = delivery_error
  41. save_jobs(jobs)
  42. return True
  43. return bool(_with_job(job_id, apply, missing=False))
  44. def _terminal(execution_id: str) -> Optional[tuple[str, Optional[str]]]:
  45. """``(ledger_outcome, delivery_error)`` once the queue row is terminal, else None."""
  46. from cron.delivery_queue import get_status
  47. row = get_status(str(execution_id))
  48. if not row or row["status"] not in _LEDGER_OUTCOME:
  49. return None
  50. delivery_error = None
  51. if row["status"] in ("failed", "unknown"):
  52. delivery_error = str(row.get("error") or f"delivery {row['status']}")
  53. return _LEDGER_OUTCOME[row["status"]], delivery_error
  54. def settled_outcome(execution_id: str) -> Optional[str]:
  55. """The ledger outcome a drain already reached for *execution_id* (None while queued)."""
  56. terminal = _terminal(execution_id)
  57. return terminal[0] if terminal else None
  58. def settle_queued_delivery(job_id: Optional[str], execution_id: Optional[str]) -> bool:
  59. """Apply the queue's terminal outcome for *execution_id*; False while still in flight."""
  60. if not job_id or not execution_id:
  61. return False
  62. terminal = _terminal(str(execution_id))
  63. if terminal is None:
  64. return False
  65. from cron.executions import settle_delivery_outcome
  66. settle_delivery_outcome(str(execution_id), terminal[0])
  67. _settle_job_record(str(job_id), str(execution_id), terminal[1])
  68. return True
  69. def settle_quietly(job_id: Optional[str], execution_id: Optional[str]) -> None:
  70. """Bookkeeping failure must not undo a terminal at-most-once send or fail its run."""
  71. try:
  72. settle_queued_delivery(job_id, execution_id)
  73. except Exception:
  74. logger.warning("Cron delivery %s: outcome write-back failed", execution_id, exc_info=True)