scheduler_bookkeeping.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. """Terminal bookkeeping phases shared by scheduler execution paths."""
  2. from typing import Optional
  3. EMPTY_RESPONSE_ERROR = "Agent completed but produced empty response (model error, timeout, or misconfiguration)"
  4. def fail_empty_response(d, final_response):
  5. """A "successful" run with a blank answer is a soft failure (never ``ok``): the ordinary tail and
  6. receipt recovery both book it, so a recovered whitespace answer cannot turn the job green."""
  7. if d.success and not str(final_response or "").strip():
  8. d.success = False
  9. d.error = EMPTY_RESPONSE_ERROR
  10. def finish_interrupted_run(job, execution_id, delivery_error):
  11. """Shutdown already advanced this run; update a failed notice without spending another fire."""
  12. from cron import scheduler
  13. logger = scheduler.logger
  14. if delivery_error:
  15. try:
  16. from cron.jobs import update_job
  17. update_job(job['id'], {'last_delivery_error': delivery_error})
  18. except Exception as exc:
  19. logger.debug('Failed recording delivery_error for interrupted job %s: %s', job['id'], exc, exc_info=True)
  20. scheduler.finish_execution(execution_id, success=False,
  21. error='Interrupted by gateway shutdown before terminal completion.')
  22. def _classify_delivery_outcome(
  23. *, delivery_error, should_deliver: bool, unresolved_origin: bool,
  24. normalized_deliver: str, incident_acked: bool, success: bool,
  25. delivery_queued=None, notification_suppressed: bool = False,
  26. ) -> str:
  27. if delivery_error:
  28. return "failed"
  29. if should_deliver and delivery_queued:
  30. return "queued"
  31. if notification_suppressed:
  32. return "suppressed"
  33. if should_deliver and unresolved_origin:
  34. return "not_configured"
  35. if should_deliver and normalized_deliver != "local":
  36. return "delivered"
  37. if incident_acked and not success:
  38. # Failure ping withheld for a known signature: operator acked it, or it was already
  39. # alerted inside the reminder cooldown (vs. plain "suppressed").
  40. return "suppressed_acked"
  41. return "suppressed"
  42. def finish_completed_run(d, fire_owner: Optional[str], execution_id: str, *, recovered=False) -> bool:
  43. """mark_job_run (owner-fenced) + execution ledger row for a run that reached delivery.
  44. ``d`` is the scheduler's ``_RunDelivery``. Store and ledger writers are read off the
  45. ``cron.scheduler`` facade, which is where tests and the receipt-recovery path patch them."""
  46. from cron import scheduler
  47. job = d.job
  48. if not d.should_deliver and job.get("last_delivery_queued"):
  49. from cron.jobs import update_job
  50. update_job(job["id"], {"last_delivery_queued": None})
  51. job["last_delivery_queued"] = None
  52. mark_kwargs: dict = {"delivery_error": d.delivery_error}
  53. from cron.scheduler_authority import journal_path
  54. journal = journal_path(job['id'], execution_id)
  55. if not job.get('no_agent'):
  56. mark_kwargs['execution_id'] = execution_id
  57. if not d.success and job.pop("_model_unreachable", False):
  58. # Never-reached-the-model failure: schedule the Cowork-style bounded re-run
  59. # (cron/unreachable_retry.py) inside the same fenced store write.
  60. mark_kwargs["model_unreachable"] = True
  61. from cron.unreachable_retry import is_retry_run
  62. if is_retry_run(job):
  63. # A re-run of an occurrence that already counted: must not spend another repeat slot.
  64. mark_kwargs["ladder_rung"] = True
  65. _hold_s = job.pop("_quota_hold_seconds", None)
  66. if not d.success and _hold_s:
  67. # Provider window closed for a known duration: park past it (cron/quota_hold.py, #89376).
  68. mark_kwargs["quota_hold_seconds"] = _hold_s
  69. mark_kwargs["recover_consumed_fire"] = bool(job.get("_scheduled_instant"))
  70. if d.success and not d.delivery_error and d.should_deliver and job.get("last_delivery_queued"):
  71. mark_kwargs["status"] = "delivery_queued"
  72. if fire_owner is not None:
  73. mark_kwargs["expected_fire_owner"] = fire_owner
  74. if d.blocked_config:
  75. mark_kwargs["status"] = "blocked_config"
  76. # A run that removed its own record has nothing left to mark; the delivery above is its result.
  77. marked = scheduler.self_removal_delivery_allowed(job["id"]) or scheduler.mark_job_run(
  78. job["id"], d.success, d.error, **mark_kwargs)
  79. if fire_owner is not None and not marked:
  80. scheduler.finish_execution(
  81. execution_id, success=False,
  82. error="Fire claim ownership lost before terminal completion.")
  83. return True
  84. delivery_outcome = _classify_delivery_outcome(
  85. delivery_error=d.delivery_error,
  86. delivery_queued=job.get("last_delivery_queued"),
  87. notification_suppressed=bool(job.get("_notification_all_targets_suppressed")),
  88. should_deliver=d.should_deliver,
  89. unresolved_origin=d.unresolved_origin,
  90. # Read the lane the notice was actually routed through (failure_deliver on failure).
  91. normalized_deliver=scheduler._normalize_deliver_value(
  92. scheduler._delivery_lane_value(job, for_failure=not d.success)),
  93. incident_acked=d.incident_acked,
  94. success=d.success,
  95. )
  96. from cron.delivery_outcome import settle_quietly, settled_outcome
  97. if delivery_outcome == "queued":
  98. # A drain that already finished this send (cron/delivery_outcome.py) is the real outcome.
  99. delivery_outcome = settled_outcome(execution_id) or "queued"
  100. if delivery_outcome in ("delivered", "not_configured") and not d.success:
  101. # Failure ping left the process (or had a configured target): mark the incident alerted.
  102. scheduler._mark_incident_alerted(d.failure_incident_id)
  103. from functools import partial
  104. from cron.executions import FIRER_FINISHED_STATES, get_execution, recover_receipted_execution
  105. finish = partial(recover_receipted_execution, job_id=job['id']) if recovered else scheduler.finish_execution
  106. finished = finish(
  107. execution_id, success=d.success, error=d.error, delivery_outcome=delivery_outcome)
  108. if recovered and finished is None:
  109. current = get_execution(execution_id)
  110. # A live foreign firer still owns this row. Keep its only recovery link until settlement.
  111. # Legacy direct calls may have no execution row; an already-finished row is also safe.
  112. if current is not None and current['status'] not in FIRER_FINISHED_STATES:
  113. return False
  114. if job.get("last_delivery_queued"):
  115. # A drain that settled before this run's own bookkeeping landed found nothing to fence on.
  116. settle_quietly(job["id"], execution_id)
  117. journal.unlink(missing_ok=True)
  118. return True