dispatch_hold.py 3.9 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485
  1. """A gateway drain holds cron dispatch; a one-shot due during the hold keeps its grace window.
  2. While the gateway drains (a restart or ``hermes update`` waiting on in-flight turns, up to
  3. ``agent.restart_after_turn_timeout``), the in-process ticker skips dispatch (``can_dispatch``).
  4. The one-shot grace (120s) kept counting from the due time, so a one-shot due mid-drain was retired
  5. as missed by the next gateway's first tick whenever the drain outlasted the grace: an update that
  6. waited three minutes on a Kanban turn dropped the reminder the user scheduled for that minute.
  7. Each held tick stamps ``<cron store>/dispatch_held.json`` (``since`` = first held tick, ``last`` =
  8. latest). It crosses the restart: the next gateway reads it from the same store. A one-shot that
  9. came due during the hold (and was not already expired when it began) counts its grace from the
  10. hold's last tick instead of its due time, so it fires once when dispatch resumes; a hold whose
  11. gateway never came back within the grace still retires it (#93526: no stale reminders after
  12. downtime).
  13. """
  14. from __future__ import annotations
  15. import contextlib
  16. import json
  17. import time
  18. from pathlib import Path
  19. from typing import Optional
  20. _FILE = "dispatch_held.json"
  21. # A held tick refreshes ``last`` once per ticker interval; the hold may have lasted this much longer.
  22. TICK_SLACK_SECONDS = 60.0
  23. # The longest a restart drain waits on in-flight turns by default (agent.restart_after_turn_timeout).
  24. MAX_EXTENSION_SECONDS = 1800.0
  25. # This process's ongoing holds: cron dir -> first held tick. Kept in memory so a failed write
  26. # mid-drain cannot reset ``since``; cleared when the ticker dispatches again (note_dispatching).
  27. _hold_since: dict[str, float] = {}
  28. def note_held(now: Optional[float] = None) -> None:
  29. """Record that this tick skipped dispatch for the active cron store (a drain is holding it)."""
  30. from cron.jobs import _current_cron_store
  31. from utils import atomic_json_write
  32. cron_dir = _current_cron_store().cron_dir
  33. now = time.time() if now is None else now
  34. if str(cron_dir) not in _hold_since:
  35. held = read(cron_dir)
  36. # Another process's hold still being stamped continues; anything older starts a new hold.
  37. since = now
  38. if held and now - held["last"] <= 2 * TICK_SLACK_SECONDS:
  39. since = held["since"]
  40. _hold_since[str(cron_dir)] = since
  41. with contextlib.suppress(OSError): # an unwritable store is reported by the tick itself
  42. atomic_json_write(Path(cron_dir) / _FILE, {"since": _hold_since[str(cron_dir)], "last": now})
  43. def note_dispatching() -> None:
  44. """The ticker dispatches the active cron store again: this process's hold on it has ended."""
  45. from cron.jobs import _current_cron_store
  46. _hold_since.pop(str(_current_cron_store().cron_dir), None)
  47. def note_held_profiles(profiles) -> None:
  48. """``note_held`` for each ``(name, home)`` a multiplex ticker serves, inside its cron scope."""
  49. from cron.scheduler_provider import _profile_cron_scope
  50. for _name, home in profiles:
  51. with _profile_cron_scope(home):
  52. note_held()
  53. def read(cron_dir: Path) -> Optional[dict]:
  54. try:
  55. data = json.loads((Path(cron_dir) / _FILE).read_text(encoding="utf-8-sig"))
  56. return {"since": float(data["since"]), "last": float(data["last"])}
  57. except (OSError, ValueError, TypeError, KeyError):
  58. return None
  59. def grace_starts_at(cron_dir: Path, due_at: float, grace: float) -> float:
  60. """When a one-shot due at *due_at* starts its *grace*: the end of a hold it came due in,
  61. else its due time. Only a one-shot due within MAX_EXTENSION_SECONDS of the hold's end is
  62. extended: an external drain has no time limit, and a reminder due hours into it is stale."""
  63. held = read(cron_dir)
  64. if (held and held["since"] - grace <= due_at <= held["last"] + TICK_SLACK_SECONDS
  65. and due_at >= held["last"] - MAX_EXTENSION_SECONDS):
  66. return max(due_at, held["last"] + TICK_SLACK_SECONDS)
  67. return due_at