unreachable_retry.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. """Automatic bounded re-runs for scheduled fires that never reached the model.
  2. Inspired by Claude Cowork (desktop changelog v1.46388.1, 2026-09-04): "automatic re-runs
  3. (after 5, 15, and 30 minutes) for a scheduled task that could not reach the model at all,
  4. for example right after the computer wakes behind a VPN."
  5. The class is deliberately narrow: the run must have FAILED with a transient network /
  6. DNS error (``cron.scheduler_preflight._is_transient_provider_resolve_error``) AND the
  7. agent must have completed zero API calls. Nothing was executed and nothing was spent, so
  8. re-running cannot double a side effect — unlike a generic failure retry (see PR #16512),
  9. which has to answer for one-shot dispatch accounting and mid-run side effects. Recurring
  10. jobs only: finite one-shots are pre-claimed by ``claim_dispatch`` (at-most-times, #38758)
  11. and must not regain a consumed dispatch here.
  12. While a retry is pending the failure notice is suppressed (Cowork re-runs silently); a
  13. run that reaches the model — success or not — resets the ladder. Disable with
  14. ``cron.retry_unreachable: false`` in config.yaml.
  15. """
  16. from __future__ import annotations
  17. import logging
  18. from datetime import datetime
  19. from typing import Any, Dict, Optional
  20. from cron.constants import is_recurring
  21. from hermes_time import now as _hermes_now
  22. logger = logging.getLogger("cron.scheduler")
  23. # Cowork's ladder: re-run after 5, 15, then 30 minutes; then give up until the
  24. # schedule's own next occurrence.
  25. RETRY_DELAYS_SECONDS: tuple[int, ...] = (300, 900, 1800)
  26. # Persisted on the job while a retry cycle is active: {"attempt": <1-based count of
  27. # retries already scheduled>, "at": <ISO instant of the pending retry>, "expr": <the cron
  28. # expression it was planned under>}. Cleared by any run
  29. # that reached the model.
  30. STATE_KEY = "unreachable_retry"
  31. def retry_enabled(cfg: Optional[dict] = None) -> bool:
  32. """``cron.retry_unreachable`` — default ON (spend-neutral: only fires when zero
  33. model calls were made)."""
  34. if cfg is None:
  35. try:
  36. from hermes_cli.config import load_config
  37. cfg = load_config() or {}
  38. except Exception: # config unreadable — keep the reliability default
  39. return True
  40. cron_cfg = (cfg or {}).get("cron")
  41. if not isinstance(cron_cfg, dict):
  42. return True
  43. return cron_cfg.get("retry_unreachable") is not False
  44. def is_model_unreachable_failure(exc: BaseException, agent: Any = None) -> bool:
  45. """True when *exc* is a transient network/DNS failure and *agent* (may be ``None``)
  46. never completed a model call — the run consumed nothing and executed nothing."""
  47. if int(getattr(agent, "session_api_calls", 0) or 0) > 0:
  48. return False
  49. from cron.scheduler_preflight import _is_transient_provider_resolve_error
  50. return _is_transient_provider_resolve_error(exc)
  51. def _ladder_applies(job: dict[str, Any]) -> bool:
  52. """The ladder's applicability gate: recurring, not paused, and enabled in config."""
  53. return is_recurring(job) and job.get("state") != "paused" and retry_enabled()
  54. def _ladder_instant(job: dict[str, Any], natural_next: Optional[datetime],
  55. now: datetime) -> Optional[datetime]:
  56. """The ladder instant ``plan_retry`` parks for this flagged failure, or None when it
  57. parks nothing: not recurring, paused, disabled, ladder exhausted, or the schedule's
  58. own *natural_next* fires at or before the rung. The single source of that decision
  59. for ``plan_retry`` and ``will_retry``."""
  60. attempt = int((job.get(STATE_KEY) or {}).get("attempt") or 0)
  61. if attempt >= len(RETRY_DELAYS_SECONDS) or not _ladder_applies(job):
  62. return None
  63. # late: jobs imports this module's helpers
  64. from cron.jobs import _instant_at_or_before, _seconds_after
  65. retry_dt = _seconds_after(now, RETRY_DELAYS_SECONDS[attempt])
  66. if natural_next is not None and _instant_at_or_before(natural_next, retry_dt):
  67. return None # the natural occurrence IS the retry
  68. return retry_dt
  69. def will_retry(job: dict[str, Any]) -> bool:
  70. """True iff ``plan_retry`` will park a re-run for this flagged failure, so the scheduler
  71. may hold the interim notice. Called before ``mark_job_run``."""
  72. repeat = job.get("repeat") or {}
  73. times = repeat.get("times")
  74. counted = 0 if is_retry_run(job) else 1 # a ladder re-run does not count (``ladder_rung``)
  75. if times is not None and times > 0 and int(repeat.get("completed") or 0) + counted >= times:
  76. return False # _advance_after_run completes the job; plan_retry never runs
  77. from cron.jobs import _parse_aware, compute_next_run
  78. now = _hermes_now()
  79. natural_next = _parse_aware(compute_next_run(
  80. job.get("schedule") or {}, now.isoformat(), timezone=job.get("timezone")))
  81. if natural_next is None:
  82. # The natural occurrence is uncomputable (e.g. croniter missing): _advance_after_run
  83. # leaves the record state=error — terminal — so plan_retry never runs either.
  84. return False
  85. return _ladder_instant(job, natural_next, now) is not None
  86. def clear_state(job: dict[str, Any]) -> None:
  87. """A run reached the model (any outcome): the ladder resets."""
  88. job.pop(STATE_KEY, None)
  89. def is_retry_fire(job: dict[str, Any], next_run: str) -> bool:
  90. """True for the exact ladder instant parked by ``plan_retry`` (off the cron lattice).
  91. The expression fingerprint keeps a direct ``jobs.json`` schedule edit from inheriting the
  92. exception, as in ``cron.quota_hold.is_recovery_fire``.
  93. """
  94. state = job.get(STATE_KEY) or {}
  95. return state.get("at") == next_run and state.get("expr") == (job.get("schedule") or {}).get("expr")
  96. def is_retry_run(job: dict[str, Any]) -> bool:
  97. """True when this dispatch snapshot fired the ladder instant parked by ``plan_retry``: a
  98. re-run of an occurrence that already counted toward ``repeat``, so it must not count again.
  99. Manual runs carry no ``_scheduled_instant`` and always count."""
  100. from cron.occurrences import scheduled_instant
  101. instant = job.get("_scheduled_instant")
  102. return instant is not None and scheduled_instant((job.get(STATE_KEY) or {}).get("at")) == instant
  103. def plan_retry(job: dict[str, Any]) -> bool:
  104. """Called under the jobs lock AFTER ``_advance_after_run`` computed the schedule's
  105. natural ``next_run_at`` for a failed, flagged run. Pulls ``next_run_at`` earlier to
  106. the ladder instant when that is sooner than the natural occurrence; exhausted or
  107. inapplicable cycles clear state and leave the schedule untouched. Returns True when
  108. a retry was scheduled."""
  109. from cron.jobs import _parse_aware # late: jobs imports this module's helpers
  110. retry_dt = _ladder_instant(job, _parse_aware(job.get("next_run_at")), _hermes_now())
  111. attempt = int((job.get(STATE_KEY) or {}).get("attempt") or 0)
  112. if retry_dt is None:
  113. if attempt >= len(RETRY_DELAYS_SECONDS) and _ladder_applies(job):
  114. # Ladder exhausted: fall back to the natural schedule and reset so the NEXT
  115. # occurrence gets a fresh ladder if the network is still down.
  116. logger.warning(
  117. "Job '%s': model unreachable after %d automatic re-runs — waiting for the "
  118. "scheduled occurrence at %s",
  119. job.get("name", job.get("id", "?")), attempt, job.get("next_run_at"))
  120. clear_state(job)
  121. return False
  122. delay = RETRY_DELAYS_SECONDS[attempt]
  123. retry_at = retry_dt.isoformat()
  124. job[STATE_KEY] = {"attempt": attempt + 1, "at": retry_at, "expr": job["schedule"].get("expr")}
  125. job["next_run_at"] = retry_at
  126. if job.get("state") != "paused":
  127. job["state"] = "scheduled"
  128. logger.info(
  129. "Job '%s': model unreachable with zero API calls — automatic re-run %d/%d in %ds "
  130. "(at %s)",
  131. job.get("name", job.get("id", "?")), attempt + 1, len(RETRY_DELAYS_SECONDS),
  132. delay, retry_at)
  133. return True