quota_hold.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. """Hold a job's fires while a provider's usage window is known to be closed (#89376).
  2. A quota-exhausted provider answers with an explicit ``retry after <N>s`` (Codex 429: the
  3. ``AuthError`` from ``hermes_cli.auth_codex._codex_quota_exhausted_error``). When the whole
  4. fallback chain is unavailable, re-firing on cadence is guaranteed to fail identically until
  5. the window reopens — every fire is a usage probe plus a delivered failure alert. The failing
  6. run's alert says the job is held; ``mark_job_run`` then parks ``next_run_at`` at the recovery
  7. boundary (or the first legal occurrence after it, when several fall inside the window) and
  8. stamps ``quota_hold_until`` so the stale-error re-arm
  9. (``cron.jobs._job_is_stale_error_recurring``) does not pull the job back early.
  10. Complement to ``cron/unreachable_retry.py``: this one moves ``next_run_at`` out of a known
  11. closed provider window. Any run that reaches the model clears the marker.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. import re
  16. from datetime import datetime
  17. from typing import Any, Dict, Optional
  18. from cron import extra_fire
  19. from hermes_time import now as _hermes_now, safe_strftime
  20. logger = logging.getLogger("cron.scheduler")
  21. # Persisted while a hold is active: ISO instant the job was parked at.
  22. STATE_KEY = "quota_hold_until"
  23. SCHEDULE_EXPR_KEY = "quota_hold_cron_expr"
  24. # The provider's remaining seconds were measured when the probe ran; by the time the run is
  25. # recorded a little wall clock has passed, so land clearly past the boundary.
  26. HOLD_SLACK_SECONDS = 60
  27. _RETRY_AFTER_RE = re.compile(r"retry after (\d+)s", re.IGNORECASE)
  28. def hold_seconds_from_failure(exc: BaseException) -> Optional[float]:
  29. """Seconds the provider said it will stay closed, or None when *exc* (or anything in its
  30. cause chain) is not a rate-limited ``AuthError`` carrying a wait hint. Anchored on the
  31. AuthError itself, never on arbitrary text, so an unrelated "retry after" in an agent's
  32. output cannot park a job."""
  33. from hermes_cli.auth import AuthError, is_rate_limited_auth_error
  34. seen: set[int] = set()
  35. cur: Optional[BaseException] = exc
  36. while cur is not None and id(cur) not in seen:
  37. seen.add(id(cur))
  38. if isinstance(cur, AuthError) and is_rate_limited_auth_error(cur):
  39. hint = getattr(cur, "retry_after", None)
  40. if hint is None:
  41. m = _RETRY_AFTER_RE.search(str(cur))
  42. hint = float(m.group(1)) if m else None
  43. return float(hint) if hint is not None and float(hint) > 0 else None
  44. cur = cur.__cause__ or cur.__context__
  45. return None
  46. def hold_active(job: dict[str, Any], now: Optional[datetime] = None) -> bool:
  47. """True while the job is parked inside a provider window (an expired marker is inert)."""
  48. from cron.jobs import _instant_after, _parse_aware # late: jobs imports this module's helpers
  49. until = _parse_aware(job.get(STATE_KEY)) if job.get(STATE_KEY) else None
  50. return until is not None and _instant_after(until, now or _hermes_now())
  51. def clear_state(job: dict[str, Any]) -> None:
  52. job.pop(STATE_KEY, None)
  53. job.pop(SCHEDULE_EXPR_KEY, None)
  54. def is_recovery_fire(job: dict[str, Any], next_run: str) -> bool:
  55. """True for the exact off-lattice cron fire parked by ``plan_hold``.
  56. The expression fingerprint keeps a direct ``jobs.json`` schedule edit from inheriting the
  57. exception: edited schedules must still re-anchor without firing.
  58. """
  59. schedule = job.get("schedule") or {}
  60. return (
  61. schedule.get("kind") == "cron"
  62. and job.get(STATE_KEY) == next_run
  63. and job.get(SCHEDULE_EXPR_KEY) == schedule.get("expr")
  64. )
  65. def _window_end(hold_seconds: float) -> datetime:
  66. from cron.jobs import _seconds_after
  67. return _seconds_after(_hermes_now(), float(hold_seconds) + HOLD_SLACK_SECONDS)
  68. def _recovery_worthwhile(
  69. job: dict[str, Any], natural_next: datetime, window_end: datetime,
  70. ) -> bool:
  71. """One off-lattice recovery fire, and only for a sparse schedule.
  72. Bounded: a job already carrying ``quota_hold_until`` IS the recovery fire failing again, so
  73. it waits for the natural schedule instead of re-parking at every hold boundary (the
  74. probe-per-window cost the hold exists to prevent). Sparse: the natural occurrence must be at
  75. least half a cadence period past the boundary — the same half-period rule as
  76. ``cron.jobs._compute_grace_seconds`` — otherwise the recovery fire is a near-duplicate of the
  77. natural one (hourly job, hold ending at :58, would fire :58 AND :00).
  78. """
  79. from cron.jobs import _elapsed_seconds, _schedule_cadence_seconds
  80. if job.get(STATE_KEY):
  81. return False
  82. cadence = _schedule_cadence_seconds(job.get("schedule") or {})
  83. return bool(cadence) and _elapsed_seconds(natural_next, window_end) >= cadence / 2
  84. def plan_hold(
  85. job: dict[str, Any], hold_seconds: float, *, recover_consumed_fire: bool = False,
  86. ) -> bool:
  87. """Called under the jobs lock AFTER ``_advance_after_run`` computed the schedule's natural
  88. ``next_run_at`` for a failed run. A scheduled sparse cron may retry its consumed fire at the
  89. recovery boundary; manual runs keep the natural schedule. Otherwise coalesce fires through
  90. the closed window. Returns True when parked."""
  91. from cron.jobs import _instant_before, _parse_aware, compute_next_run
  92. schedule = job.get("schedule") or {}
  93. kind = schedule.get("kind")
  94. if kind not in {"cron", "interval"} or job.get("state") == "paused":
  95. clear_state(job)
  96. return False
  97. window_end = _window_end(hold_seconds)
  98. natural_next = _parse_aware(job.get("next_run_at"))
  99. blocked = natural_next is None or _instant_before(natural_next, window_end)
  100. recover = (kind == "cron" and not blocked and recover_consumed_fire
  101. and _recovery_worthwhile(job, natural_next, window_end))
  102. if not blocked and not recover:
  103. clear_state(job)
  104. return False
  105. if kind == "cron" and blocked and not extra_fire.queued(job):
  106. # Coalesce cron occurrences inside the closed window to the first legal instant after it.
  107. # A queued run-now is not an occurrence: it waits for the boundary, not the next slot.
  108. parked = compute_next_run(
  109. schedule, window_end.isoformat(), timezone=job.get("timezone")
  110. ) or window_end.isoformat()
  111. else:
  112. parked = window_end.isoformat()
  113. if recover:
  114. # Only the recovery fire is off-lattice; the coalesced instant is a legal occurrence.
  115. job[SCHEDULE_EXPR_KEY] = schedule.get("expr")
  116. else:
  117. job.pop(SCHEDULE_EXPR_KEY, None)
  118. extra_fire.reschedule(job, parked)
  119. job[STATE_KEY] = parked
  120. logger.warning(
  121. "Job '%s': provider usage window closed for %.0fs — holding fires until %s instead of "
  122. "failing on every cadence tick",
  123. job.get("name", job.get("id", "?")), float(hold_seconds), parked)
  124. return True
  125. def hold_notice(job: dict[str, Any], hold_seconds: Optional[float]) -> str:
  126. """Line appended to the ONE failure alert delivered on entering the hold, else ""."""
  127. if not hold_seconds or (job.get("schedule") or {}).get("kind") not in {"cron", "interval"}:
  128. return ""
  129. window_end = _window_end(hold_seconds)
  130. hours = float(hold_seconds) / 3600.0
  131. return (
  132. f"\nThe provider's usage window is closed for about {hours:.1f}h. This job is held "
  133. f"through {safe_strftime(window_end, '%Y-%m-%d %H:%M %Z')} and resumes at the first safe "
  134. "opportunity afterwards; no further alerts are sent while the provider is unavailable."
  135. )