run_delivery_queue_watch.py 3.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778
  1. """Wake the restart-safe cron delivery drain when a worker writes the queue (#117307).
  2. A restart-safe worker runs its job outside the gateway, queues the final send in
  3. ``cron/deliveries.db`` and polls that row once a second; the gateway drained the
  4. queue only on its housekeeping tick, so a turn that ended in 8 s reached the user
  5. a minute later. Between ticks the housekeeping thread now stats the queue file
  6. (and its WAL) and drains as soon as either changed. The tick's own drain stays
  7. as the fallback for anything the stat cannot see.
  8. """
  9. from __future__ import annotations
  10. import os
  11. import time
  12. from pathlib import Path
  13. from typing import Callable, Iterable, Optional
  14. # How often the housekeeping thread looks at the queue file between its ticks.
  15. DELIVERY_QUEUE_WATCH_SECONDS = 1.0
  16. DRAIN_LABEL = "Cron durable delivery queue drain"
  17. class DeliveryQueueWatch:
  18. """Stamp-compare every served profile's queue file; ``drain`` when a stamp moved.
  19. The watch never resets itself after draining: the drain's own status writes
  20. move the stamp once more and cost one empty pass, after which the stamp is
  21. stable, while a worker that enqueued during the drain still moves it again.
  22. """
  23. def __init__(self, homes: Callable[[], Iterable[Optional[Path]]], drain: Callable[[], None]) -> None:
  24. self._homes = homes
  25. self.drain = drain
  26. self._served: list = list(homes())
  27. self._seen = self._stamp()
  28. def refresh_homes(self) -> None:
  29. """Re-resolve the served profiles once per tick (a directory read under multiplex),
  30. not on every poll; a profile that appears mid-interval is drained by the tick itself."""
  31. self._served = list(self._homes())
  32. def _stamp(self) -> tuple:
  33. from cron.delivery_queue import queue_path
  34. stamps = []
  35. for home in self._served:
  36. path = queue_path(home)
  37. for candidate in (path, path.with_name(path.name + "-wal")):
  38. try:
  39. info = os.stat(candidate)
  40. except OSError:
  41. continue
  42. stamps.append((str(candidate), info.st_mtime_ns, info.st_size))
  43. return tuple(stamps)
  44. def changed(self) -> bool:
  45. current = self._stamp()
  46. if current == self._seen:
  47. return False
  48. self._seen = current
  49. return True
  50. def wait_for_next_tick(stop_event, interval: float, watch: Optional[DeliveryQueueWatch], chore) -> None:
  51. """Sleep one housekeeping interval, draining the worker queue the moment it changes."""
  52. if watch is None:
  53. stop_event.wait(timeout=interval)
  54. return
  55. watch.refresh_homes()
  56. deadline = time.monotonic() + interval
  57. while True:
  58. remaining = max(0.0, deadline - time.monotonic())
  59. if stop_event.wait(timeout=min(remaining, DELIVERY_QUEUE_WATCH_SECONDS)):
  60. return
  61. if watch.changed():
  62. chore(DRAIN_LABEL, watch.drain)
  63. if time.monotonic() >= deadline:
  64. return