scheduler_thread.py 2.6 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364
  1. """Supervised daemon thread for the in-process cron ticker.
  2. The gateway (and the Desktop backend) run ``InProcessCronScheduler.start`` on a bare daemon thread.
  3. Every guard inside ``start`` keeps the loop alive on a per-tick failure, but nothing outside it could
  4. notice a thread that had already ended — the gateway kept serving while ``ticker_heartbeat`` froze and
  5. no job fired again until a restart (#111010). The supervisor is the missing outer layer: the
  6. housekeeping loop asks it once a cycle to respawn a ticker that died while shutdown was not requested.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import threading
  11. from typing import Any, Callable, Mapping
  12. logger = logging.getLogger(__name__)
  13. class SupervisedTickerThread:
  14. """``threading.Thread``-shaped handle whose ``restart_if_dead`` respawns a dead ticker."""
  15. def __init__(self, target: Callable[..., Any], *, args: tuple = (), kwargs: Mapping[str, Any] | None = None,
  16. stop_event: threading.Event, name: str = "cron-scheduler") -> None:
  17. self._target, self._args, self._kwargs = target, args, dict(kwargs or {})
  18. self._stop_event, self._name = stop_event, name
  19. # An external provider's start() (Chronos) arms remote one-shots and RETURNS by design;
  20. # only a target that escaped with an exception is a dead ticker worth respawning.
  21. self._crashed = False
  22. self._thread = self._spawn()
  23. self.restarts = 0
  24. def _run(self) -> None:
  25. try:
  26. self._target(*self._args, **self._kwargs)
  27. except BaseException:
  28. self._crashed = True
  29. raise
  30. def _spawn(self) -> threading.Thread:
  31. self._crashed = False
  32. return threading.Thread(target=self._run, daemon=True, name=self._name)
  33. def start(self) -> None:
  34. self._thread.start()
  35. def is_alive(self) -> bool:
  36. return self._thread.is_alive()
  37. def join(self, timeout: float | None = None) -> None:
  38. self._thread.join(timeout)
  39. def restart_if_dead(self) -> bool:
  40. """Respawn the ticker when it crashed without ``stop_event``; True when a restart happened."""
  41. if self._stop_event.is_set() or self._thread.is_alive() or not self._crashed:
  42. return False
  43. self.restarts += 1
  44. logger.error(
  45. "Cron ticker thread %r died without a stop request; restarting (restart #%d). Scheduled jobs "
  46. "did not fire while it was down — check errors.log for the escaping exception.",
  47. self._name, self.restarts,
  48. )
  49. self._thread = self._spawn()
  50. self._thread.start()
  51. return True