scheduler_worker_failure.py 2.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061
  1. """Job-level visibility for a recovered, acknowledged worker death."""
  2. def record_unknown_worker_outcome(
  3. job: dict, *, error=None, adapters=None, loop=None
  4. ) -> bool:
  5. """Project only this fire's uncertain outcome, without retrying its side effects.
  6. A terminal ledger row alone does not retire the job claim or notify its owner.
  7. Fence the notice and bookkeeping together so recovery cannot overwrite a newer
  8. fire or duplicate a failure the worker already recorded.
  9. """
  10. from cron import scheduler
  11. from cron.unreachable_retry import is_retry_run
  12. execution = scheduler.get_execution(str(job["execution_id"]))
  13. if not execution or execution.get("status") != "unknown":
  14. return False
  15. claim = job.get("fire_claim")
  16. owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
  17. if not owner:
  18. return False
  19. with scheduler.fire_claim_fence(job["id"], expected_owner=owner) as owns_claim:
  20. if not owns_claim:
  21. return True
  22. error = error or execution["error"]
  23. scheduler.logger.error("Job '%s': %s", job["id"], error)
  24. scope_tokens = scheduler._install_fire_secret_scope()
  25. delivery_error = None
  26. try:
  27. delivery_error, _ = scheduler._deliver_crash_failure(
  28. job, error, adapters=adapters, loop=loop
  29. )
  30. finally:
  31. scheduler._reset_fire_secret_scope(scope_tokens)
  32. scheduler.mark_job_run(
  33. job["id"],
  34. False,
  35. error,
  36. delivery_error=delivery_error,
  37. expected_fire_owner=owner,
  38. # A ladder re-run's occurrence already counted toward repeat.
  39. **({"ladder_rung": True} if is_retry_run(job) else {}),
  40. )
  41. return True
  42. def external_worker_exited_reason(returncode: int | None) -> str:
  43. """Cause for an attempt whose restart-safe worker this process watched exit.
  44. The generic dead-owner sweep can only say the owner vanished, so it names a
  45. scheduler restart — but this waiter never lost its scheduler: it handed the
  46. attempt to the external worker and then saw that worker exit with this status.
  47. The run is still ``unknown`` rather than ``failed`` (whether side effects ran is
  48. still unknown); only the cause becomes truthful (#128509).
  49. """
  50. return (
  51. f"Restart-safe cron worker exited with status {returncode} after adopting this "
  52. "execution, without writing a durable terminal state; whether side effects ran "
  53. "is unknown."
  54. )