scheduler_liveness.py 3.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. """Liveness side of a running cron job: the inactivity watchdog loop and the execution-ledger
  2. progress stamps.
  3. Both read the same signal (the agent's idle time) and act on opposite edges of it. The
  4. watchdog interrupts the run once idle time crosses the configured limit. The stamper refreshes
  5. ``progress_at`` while idle time stays short, so the live-owner stale-claim sweep (#115692)
  6. measures silence since the last stamp rather than run length: a busy multi-hour job keeps
  7. stamping and is never reclaimed under a healthy owner, while a wedged worker reports growing
  8. idle time, stops stamping, and is still released once the derived bound passes.
  9. """
  10. from __future__ import annotations
  11. import logging
  12. import threading
  13. import time
  14. from typing import Callable, Optional
  15. from agent.session_activity import AwakeIdleMeter
  16. from cron.executions import touch_execution_progress
  17. logger = logging.getLogger(__name__)
  18. # execution id -> wall time of the stamper's last progress verdict. A canonical cron run executes in
  19. # the gateway owner, but its ledger row belongs to the firer (an external worker), so the owner's own
  20. # stamp is fenced off it; ``cron.status`` reports this and the firer stamps its row itself.
  21. owner_progress: dict[str, float] = {}
  22. def _inactivity_watchdog_loop(
  23. *, get_idle_seconds: Callable[[], float], limit_s: float, poll_s: float, stop: threading.Event,
  24. future_done: Callable[[], bool], meter: Optional[AwakeIdleMeter] = None,
  25. ) -> bool:
  26. """Poll idle time until limit (-> True), stop, or the future completes (-> False). Uses
  27. ``threading.Event.wait``, not asyncio, so a blocked event loop cannot disable the watchdog.
  28. Driven by ``threading.Event.wait`` (a kernel timeout), not asyncio, so a blocked event-loop /
  29. ``run_job`` thread cannot disable this watchdog the way ``asyncio.sleep`` / ``wait_for`` would (family A
  30. of #94285 — the 4118s-idle-on-a-600s-limit cron hang). Returns True when *limit_s* of inactivity was
  31. observed.
  32. """
  33. # A sleeping host freezes the job with it, so time asleep never counts as inactivity.
  34. if meter is None:
  35. meter = AwakeIdleMeter()
  36. while not stop.wait(poll_s):
  37. if future_done():
  38. return False
  39. try:
  40. idle = float(get_idle_seconds() or 0.0)
  41. except Exception:
  42. idle = 0.0
  43. if meter.measure(idle) >= limit_s:
  44. return True
  45. return False
  46. class ExecutionProgressStamper:
  47. def __init__(
  48. self, execution_id: str, job_name: str, *, idle_seconds: Callable[[], float],
  49. every_seconds: float,
  50. ):
  51. self._execution_id = execution_id
  52. self._job_name = job_name
  53. self._idle_seconds = idle_seconds
  54. self._every = every_seconds
  55. self._last = time.monotonic()
  56. def tick(self) -> None:
  57. if not self._execution_id:
  58. return
  59. now = time.monotonic()
  60. if now - self._last < self._every:
  61. return
  62. # Idle past the cadence means the agent itself stopped: no stamp, so the ledger goes
  63. # quiet exactly when the watchdog would.
  64. if self._idle_seconds() >= self._every:
  65. return
  66. self._last = now
  67. owner_progress[self._execution_id] = time.time()
  68. try:
  69. touch_execution_progress(self._execution_id)
  70. except Exception:
  71. logger.debug("Job '%s': execution progress stamp failed", self._job_name, exc_info=True)