missed_runs.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159
  1. """What happens to occurrences a job could not fire on time (machine off, gateway stopped).
  2. Each job may carry ``catch_up``: ``true`` runs ONE catch-up for everything it missed, ``false``
  3. skips to the next slot. Absent, recurring jobs follow ``cron.catch_up_missed`` (default true)
  4. and one-shots keep the 120s grace rule (#93526: a stale reminder is skipped unless the job opts
  5. in). Either way the skipped or collapsed slots are booked as one ``missed ×N`` row in the
  6. execution ledger, so run history shows the gap instead of nothing.
  7. Called from the due scan in ``cron/jobs.py`` under the store lock; ledger writes ride the
  8. scan's ``on_saved`` hooks so an unwritable store cannot book the same miss every tick.
  9. """
  10. from __future__ import annotations
  11. import logging
  12. from datetime import datetime
  13. from typing import Any, Optional
  14. logger = logging.getLogger("cron.jobs")
  15. # Counting walks the schedule slot by slot; a once-a-minute job off for a month stops here.
  16. _MAX_COUNTED_SLOTS = 1000
  17. def job_catches_up(job: dict[str, Any]) -> bool:
  18. """The job's own ``catch_up`` wins; otherwise one-shots skip, and recurring jobs skip past an
  19. explicit ``grace_minutes`` or else follow ``cron.catch_up_missed``."""
  20. value = job.get("catch_up")
  21. if isinstance(value, bool):
  22. return value
  23. if (job.get("schedule") or {}).get("kind") == "once":
  24. return False
  25. from cron import jobs_time
  26. from cron.jobs import _cron_config_number
  27. # An explicit grace_minutes is the user's catch-up limit for THIS job: past it the miss is
  28. # skipped even while the global cron.catch_up_missed default still runs it once.
  29. if jobs_time.grace_seconds_override(job.get("grace_minutes")) is not None:
  30. return False
  31. return bool(_cron_config_number("catch_up_missed", True, lambda raw: raw is not False))
  32. def count_missed_slots(
  33. schedule: dict[str, Any], first: str, now: datetime, *, timezone: Any = None,
  34. ) -> int:
  35. """Occurrences from ``first`` up to ``now`` inclusive, capped at ``_MAX_COUNTED_SLOTS``.
  36. *timezone* is the job's pin (job record or zone name), as for ``compute_next_run``."""
  37. from cron.jobs import _instant_after, _parse_aware, compute_next_run
  38. count, cursor = 0, first
  39. while cursor and count < _MAX_COUNTED_SLOTS:
  40. at = _parse_aware(cursor)
  41. if at is None or _instant_after(at, now):
  42. break
  43. count += 1
  44. following = compute_next_run(schedule, cursor, timezone=timezone)
  45. cursor = following if following != cursor else None
  46. return count
  47. def _book_missed(
  48. job: dict[str, Any], count: int, first: Optional[str], detail: str, *, capped: bool,
  49. ) -> None:
  50. """``capped``: counting hit ``_MAX_COUNTED_SLOTS``, so ``count`` is a floor, not exact. Passed
  51. separately because the collapse branch books one fewer than it counted."""
  52. from cron.executions import record_missed_occurrences
  53. if capped:
  54. detail = f"{detail}; counting stopped at {_MAX_COUNTED_SLOTS}"
  55. try:
  56. record_missed_occurrences(str(job.get("id")), count=count, first_instant=first, detail=detail)
  57. except Exception:
  58. logger.warning("Job '%s': could not book its missed occurrences", job.get("id"), exc_info=True)
  59. def fast_forward_missed_recurring(d: Any, grace: int) -> bool:
  60. """Re-anchor accumulated misses; return whether catch-up is off for this job (skip the fire).
  61. The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run:
  62. it protects the crash window before mark_job_run and covers the external fire_due path, which
  63. never calls advance_next_run. mark_job_run re-anchors on completion, so the value is
  64. provisional.
  65. """
  66. from cron.jobs import _elapsed_seconds, _ensure_aware, _instant_after, record_cron_missed
  67. if _elapsed_seconds(d.scan.now, d.next_run_dt) <= grace:
  68. return False
  69. new_next = d.recompute_next()
  70. if not new_next:
  71. return False
  72. d.scan.persist(d.job["id"], next_run_at=new_next)
  73. missed = count_missed_slots(d.schedule, d.next_run, d.scan.now, timezone=d.job)
  74. capped = missed >= _MAX_COUNTED_SLOTS
  75. if (_instant_after(_ensure_aware(datetime.fromisoformat(new_next)), d.scan.now)
  76. and not job_catches_up(d.job)):
  77. logger.info(
  78. "Job '%s' missed its scheduled time (%s, grace=%ds). "
  79. "Skipping %d missed occurrence(s) because catch-up is off for this job; next run: %s",
  80. d.label, d.next_run, grace, missed, new_next)
  81. d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
  82. d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
  83. job, missed, first, f"skipped, catch-up off; next run {new_next}", capped=capped))
  84. return True
  85. logger.info(
  86. "Job '%s' missed its scheduled time (%s, grace=%ds). "
  87. "Running now; next run provisionally set to: %s (re-anchored on completion)",
  88. d.label, d.next_run, grace, new_next)
  89. from cron.occurrences import record_catch_up_occurrence
  90. d.scan.on_saved.append(record_catch_up_occurrence)
  91. if missed > 1:
  92. d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
  93. job, missed - 1, first, "collapsed into one catch-up run", capped=capped))
  94. return False
  95. def retire_expired_oneshot(d: Any) -> bool:
  96. """One-shot grace gate; True when the job must not fire this tick.
  97. A one-shot beyond the grace window fires once late only when the job opted in
  98. (``catch_up: true``); otherwise it never fires (create/update/resume reject such schedules and
  99. recovery never revives them). With no claim stamped, retire it with a diagnostic and a
  100. ``missed ×1`` ledger row (never silently delete). A claim may mean a run is still in flight
  101. elsewhere — skip but keep the record so its mark_job_run can land."""
  102. from cron import store_health
  103. from cron.jobs import (
  104. ONESHOT_GRACE_SECONDS, _current_cron_store, _elapsed_seconds, record_cron_missed)
  105. from cron.dispatch_hold import grace_starts_at
  106. from cron.jobs_oneshot_diagnostics import write_missed
  107. if _elapsed_seconds(d.scan.now, d.next_run_dt) <= ONESHOT_GRACE_SECONDS:
  108. return False
  109. # Due while a gateway drain held dispatch: its grace counts from the end of the hold. A claimed
  110. # one-shot already fired once, so it never gets the extension (that would re-dispatch it).
  111. claimed = d.job.get("run_claim") or d.job.get("fire_claim")
  112. grace_start = d.next_run_dt.timestamp() if claimed else grace_starts_at(
  113. _current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS)
  114. if d.scan.now.timestamp() - grace_start <= ONESHOT_GRACE_SECONDS:
  115. return False
  116. if job_catches_up(d.job):
  117. logger.info("Job '%s' missed its one-shot time %s; firing it once now (catch_up on).",
  118. d.label, d.next_run)
  119. return False
  120. # Due during an outage of its store: it was skipped, not missed; fire it once on recovery.
  121. if store_health.outage_covers(
  122. _current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS):
  123. return False
  124. claim = d.job.get("run_claim")
  125. # An outage-covered claim that never reached its fire claim never started: it stays covered.
  126. if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):
  127. return False
  128. if not (claim or d.job.get("fire_claim")):
  129. write_missed(d.job, d.next_run)
  130. d.scan.retire(d.job["id"])
  131. d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
  132. d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
  133. job, 1, first, "one-shot past its grace window; catch-up off", capped=False))
  134. return True