| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129 |
- """Terminal bookkeeping phases shared by scheduler execution paths."""
- from typing import Optional
- EMPTY_RESPONSE_ERROR = "Agent completed but produced empty response (model error, timeout, or misconfiguration)"
- def fail_empty_response(d, final_response):
- """A "successful" run with a blank answer is a soft failure (never ``ok``): the ordinary tail and
- receipt recovery both book it, so a recovered whitespace answer cannot turn the job green."""
- if d.success and not str(final_response or "").strip():
- d.success = False
- d.error = EMPTY_RESPONSE_ERROR
- def finish_interrupted_run(job, execution_id, delivery_error):
- """Shutdown already advanced this run; update a failed notice without spending another fire."""
- from cron import scheduler
- logger = scheduler.logger
- if delivery_error:
- try:
- from cron.jobs import update_job
- update_job(job['id'], {'last_delivery_error': delivery_error})
- except Exception as exc:
- logger.debug('Failed recording delivery_error for interrupted job %s: %s', job['id'], exc, exc_info=True)
- scheduler.finish_execution(execution_id, success=False,
- error='Interrupted by gateway shutdown before terminal completion.')
- def _classify_delivery_outcome(
- *, delivery_error, should_deliver: bool, unresolved_origin: bool,
- normalized_deliver: str, incident_acked: bool, success: bool,
- delivery_queued=None, notification_suppressed: bool = False,
- ) -> str:
- if delivery_error:
- return "failed"
- if should_deliver and delivery_queued:
- return "queued"
- if notification_suppressed:
- return "suppressed"
- if should_deliver and unresolved_origin:
- return "not_configured"
- if should_deliver and normalized_deliver != "local":
- return "delivered"
- if incident_acked and not success:
- # Failure ping withheld for a known signature: operator acked it, or it was already
- # alerted inside the reminder cooldown (vs. plain "suppressed").
- return "suppressed_acked"
- return "suppressed"
- def finish_completed_run(d, fire_owner: Optional[str], execution_id: str, *, recovered=False) -> bool:
- """mark_job_run (owner-fenced) + execution ledger row for a run that reached delivery.
- ``d`` is the scheduler's ``_RunDelivery``. Store and ledger writers are read off the
- ``cron.scheduler`` facade, which is where tests and the receipt-recovery path patch them."""
- from cron import scheduler
- job = d.job
- if not d.should_deliver and job.get("last_delivery_queued"):
- from cron.jobs import update_job
- update_job(job["id"], {"last_delivery_queued": None})
- job["last_delivery_queued"] = None
- mark_kwargs: dict = {"delivery_error": d.delivery_error}
- from cron.scheduler_authority import journal_path
- journal = journal_path(job['id'], execution_id)
- if not job.get('no_agent'):
- mark_kwargs['execution_id'] = execution_id
- if not d.success and job.pop("_model_unreachable", False):
- # Never-reached-the-model failure: schedule the Cowork-style bounded re-run
- # (cron/unreachable_retry.py) inside the same fenced store write.
- mark_kwargs["model_unreachable"] = True
- from cron.unreachable_retry import is_retry_run
- if is_retry_run(job):
- # A re-run of an occurrence that already counted: must not spend another repeat slot.
- mark_kwargs["ladder_rung"] = True
- _hold_s = job.pop("_quota_hold_seconds", None)
- if not d.success and _hold_s:
- # Provider window closed for a known duration: park past it (cron/quota_hold.py, #89376).
- mark_kwargs["quota_hold_seconds"] = _hold_s
- mark_kwargs["recover_consumed_fire"] = bool(job.get("_scheduled_instant"))
- if d.success and not d.delivery_error and d.should_deliver and job.get("last_delivery_queued"):
- mark_kwargs["status"] = "delivery_queued"
- if fire_owner is not None:
- mark_kwargs["expected_fire_owner"] = fire_owner
- if d.blocked_config:
- mark_kwargs["status"] = "blocked_config"
- # A run that removed its own record has nothing left to mark; the delivery above is its result.
- marked = scheduler.self_removal_delivery_allowed(job["id"]) or scheduler.mark_job_run(
- job["id"], d.success, d.error, **mark_kwargs)
- if fire_owner is not None and not marked:
- scheduler.finish_execution(
- execution_id, success=False,
- error="Fire claim ownership lost before terminal completion.")
- return True
- delivery_outcome = _classify_delivery_outcome(
- delivery_error=d.delivery_error,
- delivery_queued=job.get("last_delivery_queued"),
- notification_suppressed=bool(job.get("_notification_all_targets_suppressed")),
- should_deliver=d.should_deliver,
- unresolved_origin=d.unresolved_origin,
- # Read the lane the notice was actually routed through (failure_deliver on failure).
- normalized_deliver=scheduler._normalize_deliver_value(
- scheduler._delivery_lane_value(job, for_failure=not d.success)),
- incident_acked=d.incident_acked,
- success=d.success,
- )
- from cron.delivery_outcome import settle_quietly, settled_outcome
- if delivery_outcome == "queued":
- # A drain that already finished this send (cron/delivery_outcome.py) is the real outcome.
- delivery_outcome = settled_outcome(execution_id) or "queued"
- if delivery_outcome in ("delivered", "not_configured") and not d.success:
- # Failure ping left the process (or had a configured target): mark the incident alerted.
- scheduler._mark_incident_alerted(d.failure_incident_id)
- from functools import partial
- from cron.executions import FIRER_FINISHED_STATES, get_execution, recover_receipted_execution
- finish = partial(recover_receipted_execution, job_id=job['id']) if recovered else scheduler.finish_execution
- finished = finish(
- execution_id, success=d.success, error=d.error, delivery_outcome=delivery_outcome)
- if recovered and finished is None:
- current = get_execution(execution_id)
- # A live foreign firer still owns this row. Keep its only recovery link until settlement.
- # Legacy direct calls may have no execution row; an already-finished row is also safe.
- if current is not None and current['status'] not in FIRER_FINISHED_STATES:
- return False
- if job.get("last_delivery_queued"):
- # A drain that settled before this run's own bookkeeping landed found nothing to fence on.
- settle_quietly(job["id"], execution_id)
- journal.unlink(missing_ok=True)
- return True
|