| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677 |
- """Operator-visible traces left in a job's output dir when the due scan or the dispatch-limit
- guard removes a one-shot that never completed (``cron/jobs.py`` late-imports these)."""
- import logging
- from typing import Any
- logger = logging.getLogger("cron.jobs")
- def _write(job: dict[str, Any], text: str, what: str) -> bool:
- """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""
- from cron.jobs import save_job_output
- try:
- save_job_output(job.get("id", ""), text)
- return True
- except Exception:
- logger.debug("Failed to write %s diagnostic for job %r", what, job.get("id"), exc_info=True)
- return False
- def write_wedged(job: dict[str, Any]) -> None:
- """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran
- (interrupted mid-run); removing it silently would leave no output, error, or record.
- A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never
- reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway
- kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing
- due, but a silent removal leaves the user with no output, no error, and no job record. Write a small
- diagnostic file into the job's output directory so the removal is observable and debuggable.
- """
- from cron.jobs import _hermes_now
- if job.get("last_run_at") is not None:
- return # a prior run was recorded — normal completion race, not a wedge
- repeat = job.get("repeat") or {}
- claim = job.get("run_claim") or {}
- written = _write(
- job,
- "# Cron job removed without producing output\n\n"
- f"- job id: {job.get('id')}\n"
- f"- name: {job.get('name')}\n"
- f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"
- f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"
- f"- removed at: {_hermes_now().isoformat()}\n\n"
- "This one-shot job's dispatch was claimed, but the run never "
- "completed (`last_run_at` was never written) — the scheduler "
- "process was most likely killed or restarted mid-execution. The "
- "job has been removed to stop it re-firing; recreate it to run "
- "again.\n",
- "wedged-oneshot")
- if written:
- logger.warning(
- "Job '%s': removed without a completed run — diagnostic written to "
- "its output directory",
- job.get("name", job.get("id", "?")))
- def write_missed(job: dict[str, Any], next_run: str) -> None:
- """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).
- """
- from cron.jobs import ONESHOT_GRACE_SECONDS, _hermes_now
- _write(
- job,
- "# Cron job removed before firing (run time outside grace window)\n\n"
- f"- job id: {job.get('id')}\n"
- f"- name: {job.get('name')}\n"
- f"- scheduled run time: {next_run}\n"
- f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"
- f"- removed at: {_hermes_now().isoformat()}\n\n"
- "This one-shot's run time is more than the grace window in the "
- "past (scheduler down past the window, host asleep, or jobs.json "
- "edited), which is outside the 'will never fire' contract "
- "enforced at create/update/resume time. The job was removed "
- "without running; recreate it (or use the Run button) to "
- "schedule it again.\n",
- "missed-oneshot")
|