jobs_oneshot_diagnostics.py 3.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677
  1. """Operator-visible traces left in a job's output dir when the due scan or the dispatch-limit
  2. guard removes a one-shot that never completed (``cron/jobs.py`` late-imports these)."""
  3. import logging
  4. from typing import Any
  5. logger = logging.getLogger("cron.jobs")
  6. def _write(job: dict[str, Any], text: str, what: str) -> bool:
  7. """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""
  8. from cron.jobs import save_job_output
  9. try:
  10. save_job_output(job.get("id", ""), text)
  11. return True
  12. except Exception:
  13. logger.debug("Failed to write %s diagnostic for job %r", what, job.get("id"), exc_info=True)
  14. return False
  15. def write_wedged(job: dict[str, Any]) -> None:
  16. """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran
  17. (interrupted mid-run); removing it silently would leave no output, error, or record.
  18. A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never
  19. reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway
  20. kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing
  21. due, but a silent removal leaves the user with no output, no error, and no job record. Write a small
  22. diagnostic file into the job's output directory so the removal is observable and debuggable.
  23. """
  24. from cron.jobs import _hermes_now
  25. if job.get("last_run_at") is not None:
  26. return # a prior run was recorded — normal completion race, not a wedge
  27. repeat = job.get("repeat") or {}
  28. claim = job.get("run_claim") or {}
  29. written = _write(
  30. job,
  31. "# Cron job removed without producing output\n\n"
  32. f"- job id: {job.get('id')}\n"
  33. f"- name: {job.get('name')}\n"
  34. f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"
  35. f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"
  36. f"- removed at: {_hermes_now().isoformat()}\n\n"
  37. "This one-shot job's dispatch was claimed, but the run never "
  38. "completed (`last_run_at` was never written) — the scheduler "
  39. "process was most likely killed or restarted mid-execution. The "
  40. "job has been removed to stop it re-firing; recreate it to run "
  41. "again.\n",
  42. "wedged-oneshot")
  43. if written:
  44. logger.warning(
  45. "Job '%s': removed without a completed run — diagnostic written to "
  46. "its output directory",
  47. job.get("name", job.get("id", "?")))
  48. def write_missed(job: dict[str, Any], next_run: str) -> None:
  49. """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).
  50. """
  51. from cron.jobs import ONESHOT_GRACE_SECONDS, _hermes_now
  52. _write(
  53. job,
  54. "# Cron job removed before firing (run time outside grace window)\n\n"
  55. f"- job id: {job.get('id')}\n"
  56. f"- name: {job.get('name')}\n"
  57. f"- scheduled run time: {next_run}\n"
  58. f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"
  59. f"- removed at: {_hermes_now().isoformat()}\n\n"
  60. "This one-shot's run time is more than the grace window in the "
  61. "past (scheduler down past the window, host asleep, or jobs.json "
  62. "edited), which is outside the 'will never fire' contract "
  63. "enforced at create/update/resume time. The job was removed "
  64. "without running; recreate it (or use the Run button) to "
  65. "schedule it again.\n",
  66. "missed-oneshot")