scheduler_tick.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204
  1. """In-process tick admission and dispatch; job execution stays in scheduler.py."""
  2. import concurrent.futures
  3. import contextlib
  4. import logging
  5. import time
  6. from cron import store_health
  7. from cron.constants import is_recurring
  8. logger = logging.getLogger("cron.scheduler")
  9. # Log the yield at most once per episode (reset when the skew changes) to avoid per-interval spam.
  10. _YIELD_LOG_INTERVAL_SECONDS = 3600.0
  11. _last_yield_log: dict[str, object] = {}
  12. def _log_tick_yield_once(reason: str) -> None:
  13. """Log the yield at error level once per episode (skew signature)."""
  14. global _last_yield_log
  15. now = time.monotonic()
  16. last_reason = _last_yield_log.get("reason")
  17. last_at = _last_yield_log.get("at", 0.0)
  18. if last_reason != reason or (now - float(last_at)) >= _YIELD_LOG_INTERVAL_SECONDS:
  19. logger.error(
  20. "Cron tick yielded: this process is running stale code (%s) and a "
  21. "fresher gateway owns the runtime lock — jobs will fire from that "
  22. "process. Restart this one to reclaim its ticks.",
  23. reason)
  24. _last_yield_log = {"reason": reason, "at": now}
  25. def tick(verbose=True, adapters=None, loop=None, sync=True, *, can_dispatch=None,
  26. headless: bool = False):
  27. from hermes_cli.backend_retirement import retirement
  28. # Hold admission through the entire scan/advance/submit handoff. A predicate alone races
  29. # prepare after the check but before a due job enters the running-job ledger.
  30. with retirement.work() as admitted:
  31. if not admitted:
  32. return 0
  33. return _tick_admitted(verbose, adapters, loop, sync, can_dispatch=can_dispatch,
  34. headless=headless)
  35. def _acquire_tick_lock_or_degrade(_sched):
  36. """The tick lock fd, or None when this tick must not run: another ticker holds the lock, or
  37. the store cannot be written (read-only/full/denied: the lock file cannot even be created), in
  38. which case the store is marked degraded and the MCP orphan sweep still runs."""
  39. lock_dir, lock_file = _sched._get_lock_paths()
  40. _sched._ensure_cron_dir(lock_dir)
  41. try:
  42. return _sched._acquire_tick_lock(lock_file)
  43. except OSError as exc:
  44. if exc.errno not in store_health.UNWRITABLE_ERRNOS:
  45. raise # EMFILE/ENFILE etc. stay a FAILED tick (#87644)
  46. store_health.note_unwritable(exc, "tick lock unavailable", "lock", cron_dir=lock_dir)
  47. _sched._sweep_mcp_orphans()
  48. return None
  49. def _advance_or_drop_recurring(_sched, due_jobs: list) -> list:
  50. """Persist the recurring jobs' advance before any runs; when the store refuses it, drop them
  51. (no durable advance -> a crash mid-run would re-fire them, so skipping is the at-most-once
  52. side). One-shots still go through their own fire claim."""
  53. try:
  54. _sched.advance_next_runs([job["id"] for job in due_jobs])
  55. return due_jobs
  56. except OSError as exc:
  57. skipped = [j for j in due_jobs if is_recurring(j)]
  58. store_health.note_unwritable(exc, f"skipped {len(skipped)} recurring job(s)", "advance", skipped)
  59. return [j for j in due_jobs if not is_recurring(j)]
  60. def _pre_dispatch_housekeeping(_sched, headless: bool) -> None:
  61. """Admission reconcile, legacy upgrade drain and dead-owner reap, run under the tick lock
  62. before due jobs are read."""
  63. from cron.scheduler_authority import reconcile_pending
  64. if headless:
  65. reconcile_pending(allow_connect=False)
  66. else:
  67. reconcile_pending()
  68. # Upgrade drain for the retired CLI lane's records: never let it stop due jobs below.
  69. try:
  70. from cron.bot_chat_legacy import drain_legacy_pending
  71. drain_legacy_pending()
  72. except Exception as _legacy_exc:
  73. _sched.logger.warning("Legacy Bot Chat pending drain failed: %s", _legacy_exc, exc_info=True)
  74. _sched._maybe_reap_dead_owners()
  75. def _tick_admitted(
  76. verbose: bool = True, adapters=None, loop=None, sync: bool = True, *, can_dispatch=None,
  77. headless: bool = False):
  78. """Check and run all due jobs. File-locked so only one tick runs at a time (gateway ticker vs
  79. standalone daemon / manual tick). ``can_dispatch``: optional gate; false leaves due jobs for the
  80. next allowed tick. ``headless``: the tick runs outside any gateway (system crontab, ``hermes
  81. cron tick``) and must refuse agent jobs rather than spawn one. Returns the number of jobs
  82. executed (0 if another tick holds the lock)."""
  83. from cron import scheduler as _sched
  84. # Stale-code yield gate — BEFORE the lock race. A process whose checkout was updated under it
  85. # serves mixed sys.modules (jobs die on ImportErrors); if a fresher gateway holds the runtime
  86. # lock, ITS ticker dispatches. With no fresh holder (desktop-standalone) the tick proceeds.
  87. _skew = _sched._should_yield_tick_to_fresh_gateway()
  88. if _skew is not None:
  89. _log_tick_yield_once(f"boot={_skew[0]} disk={_skew[1]}")
  90. raise _sched.CronTickYielded(_skew[0], _skew[1])
  91. lock_fd = _acquire_tick_lock_or_degrade(_sched)
  92. if lock_fd is None:
  93. return 0
  94. try:
  95. # `hermes pause` ESTOP: skip dispatch, never touch in-flight runs; check_paused logs once.
  96. with contextlib.suppress(ImportError):
  97. from agent.estop import check_paused as _estop_check_paused
  98. if _estop_check_paused("cron", _sched.logger):
  99. return 0
  100. if can_dispatch is not None and not can_dispatch():
  101. _sched.logger.debug("Cron dispatch paused while gateway drains existing work")
  102. return 0
  103. _pre_dispatch_housekeeping(_sched, headless)
  104. # Periodic worktree GC (6h, threaded) — the only sweep gateway-only boxes get.
  105. try:
  106. _sched._maybe_run_worktree_maintenance()
  107. except Exception as _wt_exc:
  108. _sched.logger.debug("Worktree maintenance dispatch failed: %s", _wt_exc)
  109. due_jobs = _sched.get_due_jobs()
  110. _sched._sweep_stale_inflight_for_tick(due_jobs)
  111. if headless:
  112. from cron.scheduler_gateway_gate import refuse_agent_jobs_without_gateway
  113. due_jobs = refuse_agent_jobs_without_gateway(due_jobs)
  114. if not due_jobs:
  115. # Idle tick: skip config load + pool setup, but still reap crashed jobs' MCP orphans.
  116. if verbose:
  117. # Idle tick: skip config load + pool partitioning entirely (#33612 — the gateway ticker
  118. # calls tick(verbose=False) every 60s, so idle ticks previously fell through to
  119. # load_config()). Still run the post-tick MCP orphan sweep: main intentionally sweeps on
  120. # idle ticks so orphaned stdio children from crashed jobs are reaped even when nothing is
  121. # due.
  122. _sched.logger.info("%s - No jobs due", _sched._hermes_now().strftime('%H:%M:%S'))
  123. store_health.recheck_idle()
  124. _sched._sweep_mcp_orphans()
  125. return 0
  126. if store_health.dispatch_blocked(due_jobs):
  127. # Known-unwritable store, re-probed at most once a minute: every advance/claim would
  128. # fail, so skip them (the skip is recorded) and keep the tick's housekeeping alive.
  129. _sched._sweep_mcp_orphans()
  130. return 0
  131. if verbose:
  132. _sched.logger.info("%s - %s job(s) due", _sched._hermes_now().strftime('%H:%M:%S'), len(due_jobs))
  133. # Advance next_run_at for recurring jobs FIRST, under the lock, before any execution
  134. # (at-most-once). Re-advancing running jobs keeps the grace window alive; mark_job_run
  135. # overwrites it on completion. Composes with the claim-time advance in claim_job_for_fire.
  136. due_jobs = _advance_or_drop_recurring(_sched, due_jobs)
  137. if not due_jobs:
  138. _sched._sweep_mcp_orphans()
  139. return 0
  140. _max_workers = _sched._resolve_max_parallel_workers()
  141. if verbose:
  142. _sched.logger.info(
  143. "Running %d job(s) in parallel (max_workers=%s)",
  144. len(due_jobs),
  145. _max_workers if _max_workers else "unbounded")
  146. def _process_job(job: dict) -> bool:
  147. return _sched._process_due_job(job, adapters, loop, verbose)
  148. # Persistent pool, non-blocking dispatch. Already-running jobs are skipped; mark_job_run
  149. # re-arms next_run_at on completion, so no catch-up queue is needed.
  150. _results: list = []
  151. _all_futures: list = []
  152. pool = _sched._get_parallel_pool(_max_workers)
  153. for job in due_jobs:
  154. fut = _sched._submit_with_guard(job, pool, _process_job)
  155. if fut is None:
  156. continue
  157. _all_futures.append(fut)
  158. if not sync:
  159. _results.append(True) # optimistically counted
  160. if sync:
  161. for f in concurrent.futures.as_completed(_all_futures):
  162. try:
  163. _results.append(f.result())
  164. except Exception as exc:
  165. _sched.logger.error("Cron job future failed: %s", exc)
  166. _results.append(False)
  167. _sched._sweep_mcp_orphans()
  168. return sum(_results)
  169. _sched._sweep_mcp_orphans_when_all_done(_all_futures)
  170. return sum(_results)
  171. finally:
  172. _sched._release_tick_lock(lock_fd)