occurrences.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121
  1. """Exact scheduled identities, independent of mutable jobs.json dispatch stamps, plus the
  2. profile-local stale-schedule catch-up counter marker."""
  3. from datetime import datetime, timedelta, timezone, UTC
  4. import logging
  5. logger = logging.getLogger(__name__)
  6. def scheduled_instant(value):
  7. """Canonicalize aware instants; legacy/ambiguous values carry no exact identity."""
  8. if not isinstance(value, str):
  9. return None
  10. try:
  11. instant = datetime.fromisoformat(value)
  12. if instant.tzinfo is None:
  13. return None
  14. return instant.astimezone(UTC).isoformat()
  15. except ValueError:
  16. return None
  17. def completed_occurrence(job, instant):
  18. """Unknown/failed/pruned attempts cannot prove completion: keep them eligible."""
  19. from cron.constants import FIRE_CLAIM_SKEW_SECONDS
  20. from cron.executions import _transaction
  21. instant = scheduled_instant(instant)
  22. if instant is None:
  23. return False
  24. # A skewed early fire (see claim_job_for_fire) legitimately completes just before its slot.
  25. earliest_real = datetime.fromisoformat(instant) - timedelta(seconds=FIRE_CLAIM_SKEW_SECONDS)
  26. try:
  27. with _transaction() as conn:
  28. rows = conn.execute(
  29. "SELECT id, finished_at, claimed_at FROM executions "
  30. "WHERE job_id=? AND scheduled_instant=? "
  31. "AND status='completed'", (str(job['id']), instant)
  32. ).fetchall()
  33. for row in rows:
  34. completed_at = scheduled_instant(row["finished_at"] or row["claimed_at"])
  35. # Legacy or malformed timestamps remain proof; only positively identified poison
  36. # rows — completions recorded before their claimed occurrence — are ignored.
  37. if completed_at is None or datetime.fromisoformat(completed_at) >= earliest_real:
  38. # Both dedup gates (due scan and fire claim) consume the slot on True without a
  39. # run or a ledger row, so this line is the only trace the skip leaves (#111414).
  40. logger.warning(
  41. "Job '%s' (%s): scheduled occurrence %s was already completed by execution "
  42. "%s (finished %s); skipping the due slot without a new run",
  43. job.get("name", job.get("id")), job.get("id"), instant, row["id"],
  44. row["finished_at"] or row["claimed_at"])
  45. return True
  46. return False
  47. except Exception:
  48. logger.warning("Cannot check completed occurrence for job %s", job['id'], exc_info=True)
  49. return False
  50. # --- Pending slot: the occurrence a tick took off the schedule but has not yet claimed ---
  51. #
  52. # The tick advances a recurring job's ``next_run_at`` BEFORE dispatch (at-most-once: a crash
  53. # mid-run must not re-fire on every restart). That leaves a window — advance done, fire claim
  54. # not yet taken (interpreter finalizing, executor refusing work, process killed) — in which the
  55. # process exiting loses the occurrence silently: the restarted scan sees a future ``next_run_at``
  56. # and nothing ever ran (#107485). ``pending_slot`` is the durable record of that window: the due
  57. # scan stamps it with the exact stored instant plus the stamping owner, ``claim_job_for_fire``
  58. # (the point after which side effects may exist) clears it, and any explicit rewrite of the
  59. # schedule drops it. A slot still pending once its owner is provably gone (or its lease has
  60. # expired) was never claimed, so it is restored ONCE as the due instant and then flows through
  61. # the ordinary late / fast-forward / ``cron.catch_up_missed`` policy — never N replays.
  62. def pending_slot_stamp(next_run, now):
  63. """Store value for a recurring occurrence about to be handed to the dispatcher."""
  64. from cron.jobs import _machine_id
  65. return {"scheduled_at": next_run, "at": now.isoformat(), "by": _machine_id()}
  66. def unclaimed_pending_slot(job, now):
  67. """Stored instant of a slot the dispatcher never claimed, or None.
  68. None for non-recurring jobs and malformed stamps (never a fire) and for a job still in this
  69. process's running set (its queued worker will claim and clear the slot itself). A stamp by
  70. THIS process on a job not running here is orphaned (dispatch refused). A stamp by another
  71. process is honoured while that owner may still be alive within the fire-claim lease — a
  72. second live gateway on the same store is mid-dispatch, not dead."""
  73. from cron.constants import FIRE_CLAIM_TTL_SECONDS
  74. from cron.jobs import _claim_is_live, _job_running_in_this_process, _machine_id
  75. pending = job.get("pending_slot")
  76. if not isinstance(pending, dict):
  77. return None
  78. slot = pending.get("scheduled_at")
  79. if job.get("schedule", {}).get("kind") not in {"cron", "interval"} or not isinstance(slot, str):
  80. return None
  81. try:
  82. datetime.fromisoformat(slot)
  83. except ValueError:
  84. return None
  85. if _job_running_in_this_process(str(job.get("id", ""))):
  86. return None
  87. if pending.get("by") != _machine_id() and _claim_is_live(pending, now, FIRE_CLAIM_TTL_SECONDS):
  88. return None
  89. return slot
  90. def get_catch_up_occurrence_count() -> int:
  91. """Return the profile-local stale-schedule catch-up count."""
  92. from cron.jobs import _current_cron_store
  93. path = _current_cron_store().cron_dir / "catch_up_occurrences"
  94. try:
  95. return max(0, int(path.read_text(encoding="utf-8-sig").strip()))
  96. except (OSError, ValueError):
  97. return 0
  98. def record_catch_up_occurrence() -> None:
  99. """Increment the profile-local stale-schedule catch-up counter, best effort."""
  100. from cron.jobs import _write_marker
  101. _write_marker("catch_up_occurrences", str(get_catch_up_occurrence_count() + 1), ".count_")