| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485 |
- """A gateway drain holds cron dispatch; a one-shot due during the hold keeps its grace window.
- While the gateway drains (a restart or ``hermes update`` waiting on in-flight turns, up to
- ``agent.restart_after_turn_timeout``), the in-process ticker skips dispatch (``can_dispatch``).
- The one-shot grace (120s) kept counting from the due time, so a one-shot due mid-drain was retired
- as missed by the next gateway's first tick whenever the drain outlasted the grace: an update that
- waited three minutes on a Kanban turn dropped the reminder the user scheduled for that minute.
- Each held tick stamps ``<cron store>/dispatch_held.json`` (``since`` = first held tick, ``last`` =
- latest). It crosses the restart: the next gateway reads it from the same store. A one-shot that
- came due during the hold (and was not already expired when it began) counts its grace from the
- hold's last tick instead of its due time, so it fires once when dispatch resumes; a hold whose
- gateway never came back within the grace still retires it (#93526: no stale reminders after
- downtime).
- """
- from __future__ import annotations
- import contextlib
- import json
- import time
- from pathlib import Path
- from typing import Optional
- _FILE = "dispatch_held.json"
- # A held tick refreshes ``last`` once per ticker interval; the hold may have lasted this much longer.
- TICK_SLACK_SECONDS = 60.0
- # The longest a restart drain waits on in-flight turns by default (agent.restart_after_turn_timeout).
- MAX_EXTENSION_SECONDS = 1800.0
- # This process's ongoing holds: cron dir -> first held tick. Kept in memory so a failed write
- # mid-drain cannot reset ``since``; cleared when the ticker dispatches again (note_dispatching).
- _hold_since: dict[str, float] = {}
- def note_held(now: Optional[float] = None) -> None:
- """Record that this tick skipped dispatch for the active cron store (a drain is holding it)."""
- from cron.jobs import _current_cron_store
- from utils import atomic_json_write
- cron_dir = _current_cron_store().cron_dir
- now = time.time() if now is None else now
- if str(cron_dir) not in _hold_since:
- held = read(cron_dir)
- # Another process's hold still being stamped continues; anything older starts a new hold.
- since = now
- if held and now - held["last"] <= 2 * TICK_SLACK_SECONDS:
- since = held["since"]
- _hold_since[str(cron_dir)] = since
- with contextlib.suppress(OSError): # an unwritable store is reported by the tick itself
- atomic_json_write(Path(cron_dir) / _FILE, {"since": _hold_since[str(cron_dir)], "last": now})
- def note_dispatching() -> None:
- """The ticker dispatches the active cron store again: this process's hold on it has ended."""
- from cron.jobs import _current_cron_store
- _hold_since.pop(str(_current_cron_store().cron_dir), None)
- def note_held_profiles(profiles) -> None:
- """``note_held`` for each ``(name, home)`` a multiplex ticker serves, inside its cron scope."""
- from cron.scheduler_provider import _profile_cron_scope
- for _name, home in profiles:
- with _profile_cron_scope(home):
- note_held()
- def read(cron_dir: Path) -> Optional[dict]:
- try:
- data = json.loads((Path(cron_dir) / _FILE).read_text(encoding="utf-8-sig"))
- return {"since": float(data["since"]), "last": float(data["last"])}
- except (OSError, ValueError, TypeError, KeyError):
- return None
- def grace_starts_at(cron_dir: Path, due_at: float, grace: float) -> float:
- """When a one-shot due at *due_at* starts its *grace*: the end of a hold it came due in,
- else its due time. Only a one-shot due within MAX_EXTENSION_SECONDS of the hold's end is
- extended: an external drain has no time limit, and a reminder due hours into it is stale."""
- held = read(cron_dir)
- if (held and held["since"] - grace <= due_at <= held["last"] + TICK_SLACK_SECONDS
- and due_at >= held["last"] - MAX_EXTENSION_SECONDS):
- return max(due_at, held["last"] + TICK_SLACK_SECONDS)
- return due_at
|