| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283 |
- """Liveness side of a running cron job: the inactivity watchdog loop and the execution-ledger
- progress stamps.
- Both read the same signal (the agent's idle time) and act on opposite edges of it. The
- watchdog interrupts the run once idle time crosses the configured limit. The stamper refreshes
- ``progress_at`` while idle time stays short, so the live-owner stale-claim sweep (#115692)
- measures silence since the last stamp rather than run length: a busy multi-hour job keeps
- stamping and is never reclaimed under a healthy owner, while a wedged worker reports growing
- idle time, stops stamping, and is still released once the derived bound passes.
- """
- from __future__ import annotations
- import logging
- import threading
- import time
- from typing import Callable, Optional
- from agent.session_activity import AwakeIdleMeter
- from cron.executions import touch_execution_progress
- logger = logging.getLogger(__name__)
- # execution id -> wall time of the stamper's last progress verdict. A canonical cron run executes in
- # the gateway owner, but its ledger row belongs to the firer (an external worker), so the owner's own
- # stamp is fenced off it; ``cron.status`` reports this and the firer stamps its row itself.
- owner_progress: dict[str, float] = {}
- def _inactivity_watchdog_loop(
- *, get_idle_seconds: Callable[[], float], limit_s: float, poll_s: float, stop: threading.Event,
- future_done: Callable[[], bool], meter: Optional[AwakeIdleMeter] = None,
- ) -> bool:
- """Poll idle time until limit (-> True), stop, or the future completes (-> False). Uses
- ``threading.Event.wait``, not asyncio, so a blocked event loop cannot disable the watchdog.
- Driven by ``threading.Event.wait`` (a kernel timeout), not asyncio, so a blocked event-loop /
- ``run_job`` thread cannot disable this watchdog the way ``asyncio.sleep`` / ``wait_for`` would (family A
- of #94285 — the 4118s-idle-on-a-600s-limit cron hang). Returns True when *limit_s* of inactivity was
- observed.
- """
- # A sleeping host freezes the job with it, so time asleep never counts as inactivity.
- if meter is None:
- meter = AwakeIdleMeter()
- while not stop.wait(poll_s):
- if future_done():
- return False
- try:
- idle = float(get_idle_seconds() or 0.0)
- except Exception:
- idle = 0.0
- if meter.measure(idle) >= limit_s:
- return True
- return False
- class ExecutionProgressStamper:
- def __init__(
- self, execution_id: str, job_name: str, *, idle_seconds: Callable[[], float],
- every_seconds: float,
- ):
- self._execution_id = execution_id
- self._job_name = job_name
- self._idle_seconds = idle_seconds
- self._every = every_seconds
- self._last = time.monotonic()
- def tick(self) -> None:
- if not self._execution_id:
- return
- now = time.monotonic()
- if now - self._last < self._every:
- return
- # Idle past the cadence means the agent itself stopped: no stamp, so the ledger goes
- # quiet exactly when the watchdog would.
- if self._idle_seconds() >= self._every:
- return
- self._last = now
- owner_progress[self._execution_id] = time.time()
- try:
- touch_execution_progress(self._execution_id)
- except Exception:
- logger.debug("Job '%s': execution progress stamp failed", self._job_name, exc_info=True)
|