scheduler_provider.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684
  1. """CronScheduler provider interface (Axis B — the trigger). EXPERIMENTAL: shape MAY change until a
  2. second provider validates it; growth MUST be additive (optional method with a default), never a
  3. changed start() signature or new abstractmethod. Providers decide only *when* a job fires —
  4. execution + delivery stay in cron.scheduler.run_job / _deliver_result; never reimplement them.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import inspect
  9. import logging
  10. import threading
  11. import time
  12. from abc import ABC, abstractmethod
  13. from pathlib import Path
  14. from typing import Any
  15. logger = logging.getLogger(__name__)
  16. # Cap for exponential tick backoff during fd exhaustion (interval doubled per failure).
  17. _EMFILE_BACKOFF_MAX_SECONDS = 15 * 60
  18. DEFAULT_MISFIRE_GRACE_MINUTES = 10
  19. # Cap for the exponential tick backoff applied while consecutive ticks fail with fd exhaustion
  20. # (EMFILE/ENFILE, #87644). Base is the tick interval (60s by default); each consecutive EMFILE failure
  21. # doubles the wait, capped here so a still-alive-but-exhausted gateway never sleeps longer than this between
  22. # recovery attempts.
  23. def _backoff_wait_seconds(interval: float, consecutive_failures: int) -> float:
  24. """Plain ``interval`` while healthy; doubles per fd-exhaustion failure, capped.
  25. Exponential tick backoff shared by both ticker loops (#87644).
  26. """
  27. if consecutive_failures <= 0:
  28. return interval
  29. return min(interval * (2 **(consecutive_failures - 1)), _EMFILE_BACKOFF_MAX_SECONDS)
  30. def _note_tick_failure(exc: BaseException, consecutive_failures: int) -> int:
  31. """On fd exhaustion: reclaim fds and bump the backoff counter; any other failure resets it —
  32. backoff is reserved for the EMFILE storm.
  33. Shared by both ticker loops (#87644): on fd exhaustion, attempt reclamation (gc.collect + raise the soft
  34. nofile limit) so the NEXT tick can succeed, and bump the counter so ``_backoff_wait_seconds`` backs off
  35. exponentially while the process has no chance of making progress.
  36. """
  37. from cron.scheduler import _is_fd_exhaustion, _reclaim_fds_best_effort
  38. if _is_fd_exhaustion(exc):
  39. _reclaim_fds_best_effort()
  40. return consecutive_failures + 1
  41. return 0
  42. def _guarded_store_write(action, description, *args, **kwargs):
  43. """Run a ticker status-marker write so a failing store never ends the ticker thread.
  44. The gateway runs the provider on an unsupervised daemon thread: one escaping exception
  45. there stops cron silently while the gateway keeps serving (#111010). Heartbeat/error
  46. markers are diagnostics for ``hermes cron status`` — losing one write to a broken store
  47. must degrade to a logged warning, not thread death.
  48. """
  49. try:
  50. action(*args, **kwargs)
  51. except BaseException as e:
  52. logger.warning("Cron %s write failed: %s", description, e, exc_info=True)
  53. def _profile_entry(entry) -> tuple:
  54. """Normalize a ``profile_homes`` entry (``(name, home)`` tuple or bare home) to ``(name,
  55. home)``."""
  56. return entry if isinstance(entry, tuple) else (None, entry)
  57. def _existing_profile_homes(profile_homes: list) -> list:
  58. """Drop homes no longer on disk: ticking/heartbeating a deleted home would recreate its
  59. ``cron/`` workspace and silently resurrect the profile.
  60. Ticking or heartbeating a deleted home recreates its ``cron/`` workspace (``record_ticker_heartbeat`` ->
  61. ``ensure_dirs`` -> ``mkdir(parents=True)``) on every 60s cycle, so the "deleted" profile silently comes
  62. back on disk and in ``hermes profile list`` (#47368). Filtering on directory existence leaves a deleted
  63. profile's home untouched, which is the correct invariant: a home that does not exist cannot hold jobs to
  64. fire.
  65. """
  66. if callable(profile_homes):
  67. # Live enumerator (multiplex gateway): a profile created after startup is ticked without a
  68. # restart; a raising enumerator keeps this cycle at zero homes rather than killing the ticker.
  69. try:
  70. profile_homes = list(profile_homes())
  71. except Exception:
  72. logger.warning("cron profile enumeration failed; skipping this cycle", exc_info=True)
  73. return []
  74. return [entry for entry in profile_homes if Path(_profile_entry(entry)[1]).is_dir()]
  75. def routed_profile_fire(home=None) -> bool:
  76. """True when a fire runs for a profile OTHER than the process's own.
  77. Derived from the fire's home itself (``home``, else the task's active ``get_hermes_home()``), never from
  78. a marker one entry point sets: the desktop ticker (``_profile_cron_scope``), the dashboard's
  79. manual Run now (``hermes_cli.web_server_cron._cron_store_scope``) and any other caller that
  80. binds a HERMES_HOME override for a sibling profile all reach ``run_one_job`` the same way, and a
  81. marker set only in the ticker left the manual path with the original cross-profile leak.
  82. The desktop backend fires every local profile from one process without setting the
  83. process-global multiplex flag, so every isolation keyed on ``is_multiplex_active()`` was inert
  84. for those fires: a sibling profile's ``.env`` landed in the shared ``os.environ`` with
  85. ``override=True`` and a scope miss read the launch profile's credentials (#107692).
  86. ``cron.scheduler._install_fire_secret_scope`` turns this into multiplex semantics for exactly
  87. the span the profile's secret scope covers, and the restart-safe handoff marks the worker
  88. payload with it. The launch identity is ``get_routing_process_hermes_home()`` (gateway/AGENTS.md
  89. "One launch-home identity")."""
  90. from hermes_constants import get_hermes_home, get_routing_process_hermes_home, hermes_home_key
  91. target = home if home is not None else get_hermes_home()
  92. return hermes_home_key(target) != hermes_home_key(get_routing_process_hermes_home())
  93. @contextlib.contextmanager
  94. def _profile_cron_scope(home):
  95. """Scope the calling thread to one profile's home + cron store for the block."""
  96. from cron.jobs import use_cron_store
  97. from hermes_constants import set_hermes_home_override, reset_hermes_home_override
  98. # Record per-profile heartbeat after each tick cycle. Distinguish a COMPLETED cycle (``_tick_error``
  99. # unset) — where each profile's beat reflects its own outcome, so a yielding profile does not darken
  100. # healthy siblings — from an aborted one (exception), where no profile completed and all beats are
  101. # unsuccessful (#32612).
  102. home_token = set_hermes_home_override(str(home))
  103. try:
  104. with use_cron_store(home):
  105. yield
  106. finally:
  107. reset_hermes_home_override(home_token)
  108. class CronScheduler(ABC):
  109. """Decides WHEN a due cron job fires. Only ``name`` + ``start`` are required; keep every other
  110. hook NON-abstract with a safe default (``test_abc_growth_stays_additive``)."""
  111. @property
  112. @abstractmethod
  113. def name(self) -> str:
  114. """Short identifier, e.g. 'builtin', 'chronos'."""
  115. def is_available(self) -> bool:
  116. """Whether this provider can run here. MUST NOT make network calls; False → built-in."""
  117. return True
  118. @abstractmethod
  119. def start(
  120. self, stop_event: threading.Event, *, adapters: Any = None, loop: Any = None,
  121. interval: int = 60,
  122. ) -> None:
  123. """Begin firing due jobs. Built-in BLOCKS until stop_event is set (run in a daemon thread);
  124. an external provider may return immediately but must still honor stop_event."""
  125. def stop(self) -> None:
  126. """Optional eager teardown; stop_event is the primary signal."""
  127. return
  128. # Optional hooks for external providers — default-safe; keep NON-abstract.
  129. def on_jobs_changed(self) -> None:
  130. """After a successful store mutation; external providers reconcile. Built-in: no-op."""
  131. return
  132. def register_job(self, job: dict[str, Any]) -> None:
  133. """Register the external trigger for a newly persisted job (must complete before callers
  134. report it as scheduled). Built-in: no-op."""
  135. return
  136. def recover_interrupted(self) -> int:
  137. """Run profile-local attempt recovery for every provider lifecycle."""
  138. from cron.executions import recover_interrupted_executions
  139. return recover_interrupted_executions()
  140. @property
  141. def supports_force_fire(self) -> bool:
  142. """Whether ``fire_due`` accepts ``force`` (signature-detected for older providers)."""
  143. return provider_supports_force_fire(self)
  144. def fire_due(
  145. self, job_id: str, *, adapters: Any = None, loop: Any = None, force: bool = False,
  146. manual: bool = False,
  147. ) -> bool:
  148. """Run one job NOW (inbound fire webhook entry). Store CAS claim (multi-machine
  149. at-most-once) then shared ``run_one_job``. True if THIS caller claimed and processed the
  150. attempt (even if the job failed); False if the claim was lost or the job is gone.
  151. ``manual`` marks an off-tick run-now (dashboard trigger): the claim must not stamp
  152. ``next_run_at`` as the occurrence, or that slot is skipped when it arrives. Webhook and
  153. misfire fires arriving at/after the due instant run that slot and keep the stamp; a fire
  154. arriving BEFORE the stored instant is off-tick like ``manual`` and stays occurrence-free
  155. (it cannot be the tick that owns a future slot)."""
  156. claimed_job = self.claim_fire(job_id, force=force, manual=manual)
  157. if claimed_job is None:
  158. return False
  159. return self.fire_claimed(claimed_job, adapters=adapters, loop=loop)
  160. def claim_fire(self, job_id: str, *, force: bool = False, manual: bool = False) -> dict | None:
  161. """Durably claim one fire + create its audit attempt. Transports call this synchronously
  162. before acknowledging, then pass the exact snapshot to ``fire_claimed`` off-thread."""
  163. from cron.executions import create_execution, finish_execution, set_execution_occurrence
  164. from cron.jobs import claim_job_for_fire
  165. execution = create_execution(job_id, source=self.name)
  166. claim_kwargs = {"return_job": True}
  167. if force:
  168. claim_kwargs["force"] = True
  169. if manual:
  170. claim_kwargs["manual"] = True
  171. try:
  172. claimed_job = claim_job_for_fire(job_id, **claim_kwargs)
  173. if isinstance(claimed_job, dict):
  174. set_execution_occurrence(execution["id"], claimed_job.get("_scheduled_instant"))
  175. except BaseException as exc:
  176. finish_execution(
  177. execution["id"], success=False,
  178. error=f"Fire claim failed before dispatch: {type(exc).__name__}: {exc}",
  179. )
  180. raise
  181. if not isinstance(claimed_job, dict):
  182. finish_execution(execution["id"], success=False, error="Fire claim was not acquired")
  183. return None
  184. claimed_job["execution_id"] = execution["id"]
  185. return claimed_job
  186. def fire_claimed(
  187. self, claimed_job: dict, *, adapters: Any = None, loop: Any = None,
  188. cancel_event: Any = None,
  189. ) -> bool:
  190. """Run an exact ``claim_fire`` snapshot; ``cancel_event`` lets the transport stop it
  191. cooperatively (e.g. dashboard lifespan drain)."""
  192. from cron.scheduler import run_one_job
  193. run_one_job(claimed_job, adapters=adapters, loop=loop, cancel_event=cancel_event)
  194. return True
  195. def reconcile(self) -> None:
  196. """Converge the external registry toward jobs.json (desired state). Built-in: no-op."""
  197. return
  198. def provider_supports_force_fire(provider: Any) -> bool:
  199. """Return whether a provider can safely receive ``fire_due(force=...)`` (signature-detected)."""
  200. return provider_fire_due_accepts(provider, "force")
  201. def provider_fire_due_accepts(provider: Any, name: str) -> bool:
  202. """Whether ``provider.fire_due`` takes keyword ``name`` (third-party providers may predate it)."""
  203. try:
  204. parameters = inspect.signature(provider.fire_due).parameters.values()
  205. except (TypeError, ValueError):
  206. return False
  207. return any(
  208. p.kind is inspect.Parameter.VAR_KEYWORD
  209. or (
  210. p.name == name
  211. and p.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
  212. )
  213. for p in parameters
  214. )
  215. def provider_supports_split_fire(provider: Any) -> bool:
  216. """Whether a provider implements the two-phase fire contract. A legacy provider overriding only
  217. ``fire_due`` must keep being driven through it — routing around the override would drop its
  218. custom claim/re-arm/telemetry behavior."""
  219. cls = type(provider)
  220. def overrides(name: str) -> bool:
  221. impl = getattr(cls, name, None)
  222. return impl is not None and impl is not getattr(CronScheduler, name)
  223. if overrides("claim_fire") or overrides("fire_claimed"):
  224. return True
  225. return not overrides("fire_due")
  226. def _misfire_grace_minutes() -> float:
  227. """``cron.misfire_grace_minutes`` from config; non-positive disables the catch-up sweep."""
  228. try:
  229. from hermes_cli.config import cfg_get, load_config
  230. config = load_config()
  231. return float(
  232. cfg_get(config, "cron", "misfire_grace_minutes", default=DEFAULT_MISFIRE_GRACE_MINUTES)
  233. )
  234. except Exception:
  235. return float(DEFAULT_MISFIRE_GRACE_MINUTES)
  236. def fire_overdue_jobs(
  237. provider: CronScheduler, *, adapters: Any = None, loop: Any = None, now: Any = None,
  238. ) -> int:
  239. """Misfire backstop (gateway housekeeping loop): fire jobs whose external HTTP fire never
  240. arrived, else ``next_run_at`` stays parked in the past forever. No-op for the built-in (its tick
  241. loop self-heals). Routes through the provider's own two-phase path so re-arm logic runs and a
  242. concurrent late external retry is de-duplicated by the store CAS; waits out
  243. ``cron.misfire_grace_minutes`` so the external retry gets first right. Returns jobs dispatched.
  244. """
  245. # `hermes pause` ESTOP: skip the sweep entirely. No state to unwind — the
  246. # next housekeeping pass after `hermes resume` catches overdue work up
  247. # through the existing claim_fire path. Distinct component name from the
  248. # ticker's "cron" so the log-once mechanism fires independently.
  249. with contextlib.suppress(ImportError):
  250. from agent.estop import check_paused as _estop_check_paused
  251. if _estop_check_paused("cron-misfire", logger):
  252. return 0
  253. from datetime import datetime
  254. if isinstance(provider, InProcessCronScheduler):
  255. return 0
  256. grace_minutes = _misfire_grace_minutes()
  257. if grace_minutes <= 0:
  258. return 0
  259. from cron.jobs import (
  260. ONESHOT_GRACE_SECONDS, _elapsed_seconds, _ensure_aware, _hermes_now,
  261. is_job_runnable, load_jobs,
  262. )
  263. if now is None:
  264. now = _hermes_now()
  265. fired = 0
  266. for job in load_jobs():
  267. if not is_job_runnable(job):
  268. continue
  269. next_run_at = job.get("next_run_at")
  270. if not next_run_at:
  271. continue
  272. try:
  273. due_dt = _ensure_aware(datetime.fromisoformat(next_run_at))
  274. except (ValueError, TypeError):
  275. continue
  276. overdue_seconds = _elapsed_seconds(now, due_dt)
  277. if overdue_seconds < grace_minutes * 60:
  278. continue
  279. job_id = str(job.get("id") or "")
  280. # One-shots past ONESHOT_GRACE_SECONDS "will never fire"; don't resurrect them hours late.
  281. # One-shot jobs share the module-wide policy: more than ONESHOT_GRACE_SECONDS past their run time
  282. # means "will never fire" (create/update/resume/recovery and, since #89571, the due-scan all enforce
  283. # it). The misfire backstop must not resurrect them hours late after downtime — that's #93526.
  284. schedule = job.get("schedule") or {}
  285. if str(schedule.get("kind") or "") == "once" and overdue_seconds > ONESHOT_GRACE_SECONDS:
  286. logger.warning(
  287. "Misfire catch-up: one-shot job %s (%s) was due %s "
  288. "(%.0f min overdue) — outside the %ss one-shot grace "
  289. "window, not firing.",
  290. job_id,
  291. job.get("name") or "unnamed",
  292. next_run_at,
  293. overdue_seconds / 60,
  294. ONESHOT_GRACE_SECONDS,
  295. )
  296. continue
  297. logger.warning(
  298. "Misfire catch-up: job %s (%s) was due %s (%.0f min overdue) and "
  299. "no external fire arrived — firing locally.",
  300. job_id,
  301. job.get("name") or "unnamed",
  302. next_run_at,
  303. overdue_seconds / 60,
  304. )
  305. try:
  306. # Claim synchronously (CAS loss = external retry beat us), run off-thread: never block.
  307. claimed = provider.claim_fire(job_id)
  308. if claimed is None:
  309. continue
  310. threading.Thread(
  311. target=provider.fire_claimed, args=(claimed,),
  312. kwargs={"adapters": adapters, "loop": loop}, daemon=True,
  313. name=f"cron-misfire-{job_id[:12]}",
  314. ).start()
  315. fired += 1
  316. except Exception as exc:
  317. logger.warning(
  318. "Misfire catch-up failed for job %s: %s: %s",
  319. job_id, type(exc).__name__, exc,
  320. )
  321. return fired
  322. def resolve_cron_scheduler() -> CronScheduler:
  323. """Resolve ``cron.provider``; missing/failing/unavailable providers fall back to the built-in
  324. with a warning — cron must never be left without a trigger."""
  325. name = ""
  326. try:
  327. from hermes_cli.config import cfg_get, load_config
  328. name = (cfg_get(load_config(), "cron", "provider", default="") or "").strip()
  329. except Exception:
  330. pass
  331. if not name or name in ("builtin", "in-process", "inprocess"):
  332. return InProcessCronScheduler()
  333. try:
  334. from plugins.cron_providers import load_cron_scheduler
  335. provider = load_cron_scheduler(name)
  336. if provider is None:
  337. logger.warning("cron.provider '%s' not found; using built-in ticker", name)
  338. return InProcessCronScheduler()
  339. if not provider.is_available():
  340. logger.warning("cron.provider '%s' not available; using built-in ticker", name)
  341. return InProcessCronScheduler()
  342. logger.info("Using cron scheduler provider: %s", provider.name)
  343. return provider
  344. except Exception as e:
  345. logger.warning("Failed to load cron.provider '%s' (%s); using built-in ticker", name, e)
  346. return InProcessCronScheduler()
  347. def scheduler_for_profile_mode(
  348. provider: CronScheduler, *, multiplex_profiles: bool
  349. ) -> CronScheduler:
  350. """External providers own one unscoped remote registry and cannot reconcile several profile
  351. stores: fail closed to the built-in multiplex ticker until the API carries profile identity."""
  352. if not multiplex_profiles or isinstance(provider, InProcessCronScheduler):
  353. return provider
  354. logger.warning(
  355. "cron.provider '%s' does not support multiplex_profiles; using built-in ticker",
  356. provider.name,
  357. )
  358. return InProcessCronScheduler()
  359. class InProcessCronScheduler(CronScheduler):
  360. """Default in-process 60s ticker; ``start()`` blocks until ``stop_event``. ``can_dispatch`` is
  361. an optional drain gate; skipped ticks leave due jobs intact for the next allowed tick."""
  362. @property
  363. def name(self) -> str:
  364. return "builtin"
  365. def start(
  366. self, stop_event, *, adapters=None, loop=None, interval=60, can_dispatch=None,
  367. profile_homes=None, profile_adapters=None, default_profile=None, profile_gate=None,
  368. profile_bot_platforms=None,
  369. ):
  370. from cron.scheduler import CronTickYielded
  371. from cron.scheduler import tick as cron_tick
  372. from cron.jobs import clear_ticker_error, record_ticker_error, record_ticker_heartbeat
  373. from cron.scheduler_ownership import register_ticked_homes
  374. from hermes_constants import get_process_hermes_home
  375. from cron.dispatch_hold import note_dispatching, note_held
  376. logger.info("In-process cron scheduler started (interval=%ds)", interval)
  377. # Multiplex: tick EACH profile's store every cycle, heartbeats/recovery scoped per profile.
  378. # ── Multiplex profiles ──────────────────────────────────────────── When profile_homes is set
  379. # (multiplex_profiles on), tick EACH profile's cron store on every tick cycle so secondary-profile
  380. # jobs actually fire instead of languishing in a store no ticker owns (#69377). Without this, only
  381. # the process-global HERMES_HOME (the default profile) is ticked. Heartbeats and recovery are also
  382. # scoped per profile so `hermes cron status` reflects liveness for every profile independently.
  383. if profile_homes is not None and (callable(profile_homes) or profile_homes):
  384. self._start_multiplex(
  385. stop_event, profile_homes=profile_homes, adapters=adapters, loop=loop,
  386. interval=interval, can_dispatch=can_dispatch, profile_adapters=profile_adapters,
  387. default_profile=default_profile, profile_gate=profile_gate,
  388. profile_bot_platforms=profile_bot_platforms,
  389. )
  390. return
  391. # Single-profile ticker: the launch home is the only home this process owns cron for.
  392. register_ticked_homes([get_process_hermes_home()])
  393. # Startup recovery and the initial heartbeat run before the guarded loop; a broken
  394. # store here must not take the whole ticker thread down (#111010) — the loop's own
  395. # per-tick handling logs, persists the reason and keeps the thread alive.
  396. try:
  397. recovered = self.recover_interrupted()
  398. if recovered:
  399. logger.warning(
  400. "Marked %d interrupted cron execution(s) unknown after restart", recovered
  401. )
  402. # Heartbeat before the first sleep so `hermes cron status` sees a live ticker
  403. # immediately.
  404. record_ticker_heartbeat()
  405. except BaseException as e:
  406. logger.error("Cron startup recovery error: %s", e, exc_info=True)
  407. _guarded_store_write(
  408. record_ticker_error, "startup error", f"{type(e).__name__}: {e}"
  409. )
  410. # EMFILE backoff: don't hammer the store while fds are exhausted; a clean tick resets it.
  411. consecutive_failures = 0
  412. next_tick = time.monotonic()
  413. while not stop_event.is_set():
  414. ok = False
  415. try:
  416. if can_dispatch is not None and not can_dispatch():
  417. logger.debug("Cron dispatch paused while gateway drains existing work")
  418. note_held()
  419. else:
  420. note_dispatching()
  421. cron_tick(
  422. verbose=False, adapters=adapters, loop=loop, sync=False,
  423. can_dispatch=can_dispatch,
  424. )
  425. ok = True
  426. except BaseException as e:
  427. # BaseException, not Exception: a SystemExit must not silently kill the ticker;
  428. # KeyboardInterrupt is caught on purpose — shutdown is driven by stop_event.
  429. # Catch BaseException (not just Exception) so a SystemExit from a misbehaving provider SDK /
  430. # agent retry path does not kill the ticker thread silently (#32612). KeyboardInterrupt is
  431. # intentionally caught here too — gateway shutdown is driven by stop_event (set by the main
  432. # thread's signal handler), not by an exception in this daemon thread, so swallowing it and
  433. # re-checking stop_event keeps shutdown clean.
  434. if isinstance(e, CronTickYielded):
  435. # Expected while a fresh gateway owns the lock; still recorded for status.
  436. logger.info("Cron tick yielded: %s", e)
  437. else:
  438. logger.error("Cron tick error: %s", e, exc_info=True)
  439. # Persist the reason so `hermes cron status` (separate process) shows WHY.
  440. _guarded_store_write(
  441. record_ticker_error, "tick error", f"{type(e).__name__}: {e}"
  442. )
  443. consecutive_failures = _note_tick_failure(e, consecutive_failures)
  444. # Liveness every iteration; success marker only on a clean tick.
  445. # EMFILE: reclaim fds + back off exponentially so the exhausted process stops hammering the
  446. # store while it has no chance of making progress (#87644).
  447. # Record liveness every iteration; bump the success marker only on a clean tick, so status can
  448. # tell "alive but failing every tick" from "actually firing jobs" (#32612, #32895).
  449. _guarded_store_write(record_ticker_heartbeat, "heartbeat", success=ok)
  450. if ok:
  451. _guarded_store_write(clear_ticker_error, "error clear")
  452. consecutive_failures = 0
  453. wait_for = _backoff_wait_seconds(interval, consecutive_failures)
  454. next_tick += wait_for
  455. now = time.monotonic()
  456. if next_tick < now:
  457. # Tick overran interval or host was suspended; re-anchor to avoid
  458. # burst-firing zero-length sleep cycles (#114467).
  459. next_tick = now + wait_for
  460. stop_event.wait(max(0.0, next_tick - now))
  461. def _start_multiplex(
  462. self, stop_event, *, profile_homes, adapters=None, loop=None, interval=60,
  463. can_dispatch=None, profile_adapters=None, default_profile=None, profile_gate=None,
  464. profile_bot_platforms=None,
  465. ):
  466. """Tick every profile's store, each scoped via ``_profile_cron_scope``. ``profile_gate(name,
  467. home)``, when given, is consulted every cycle; a rejected profile is neither ticked nor
  468. heartbeated."""
  469. from cron.scheduler import tick as cron_tick
  470. from cron.scheduler import CronTickYielded, _is_fd_exhaustion
  471. from cron.scheduler_preflight import (
  472. SharedRouteAdapters, _primary_profile_routes_for_current_home, has_credential_platform,
  473. )
  474. from cron.jobs import clear_ticker_error, record_ticker_error, record_ticker_heartbeat
  475. from cron.scheduler_ownership import register_ticked_homes
  476. from cron.dispatch_hold import note_dispatching, note_held_profiles
  477. initial_homes = _existing_profile_homes(profile_homes)
  478. register_ticked_homes([_profile_entry(entry)[1] for entry in initial_homes])
  479. logger.info(
  480. "Multiplex cron scheduler started for %d profile(s): %s%s",
  481. len(initial_homes),
  482. [p[0] if isinstance(p, tuple) else p for p in initial_homes],
  483. " (re-enumerated every cycle)" if callable(profile_homes) else "",
  484. )
  485. def tick_adapters_for(profile_name):
  486. # Deliver via the profile's OWN adapters; NEVER fall back to the default profile's
  487. # (wrong bot). A credentialless satellite may ride the PRIMARY adapter only for targets
  488. # an exact enabled route maps here; else fail closed (delivery skipped this tick).
  489. if profile_name is None or profile_name == default_profile:
  490. return adapters
  491. tick_adapters = (profile_adapters or {}).get(profile_name) or {}
  492. owned = (profile_bot_platforms or {}).get(profile_name)
  493. if not has_credential_platform(tick_adapters, owned) and adapters:
  494. return SharedRouteAdapters(adapters, _primary_profile_routes_for_current_home())
  495. return tick_adapters
  496. # Recovery + heartbeat per profile; one broken store must not abort startup for the others.
  497. # A profile may have been deleted since this snapshot was taken; never recreate a deleted home's
  498. # cron workspace via the heartbeat below (#47368).
  499. for entry in initial_homes:
  500. _, home = _profile_entry(entry)
  501. try:
  502. with _profile_cron_scope(home):
  503. recovered = self.recover_interrupted()
  504. if recovered:
  505. logger.warning(
  506. "Marked %d interrupted cron execution(s) for profile at %s",
  507. recovered, home,
  508. )
  509. record_ticker_heartbeat()
  510. except BaseException as e:
  511. logger.error(
  512. "Cron startup recovery error for profile at %s: %s", home, e, exc_info=True
  513. )
  514. consecutive_failures = 0
  515. next_tick = time.monotonic()
  516. while not stop_event.is_set():
  517. ok = False
  518. _tick_error = None
  519. _profile_errors: dict[str, str] = {}
  520. # Worst failure this cycle (fd exhaustion wins); backoff applied once per cycle.
  521. # See #87644.
  522. _cycle_exc: BaseException | None = None
  523. # Enumeration and gating run on the ticker thread; a raising gate callable must
  524. # fail THIS cycle (logged, no heartbeats, NO ticks), not end the thread (#111010).
  525. # Publish the list only once the gate has filtered it: a partial assignment would
  526. # tick the ungated set — the exact stand-down the Desktop gate exists for (#100489).
  527. cycle_homes: list = []
  528. try:
  529. enumerated = [_profile_entry(e) for e in _existing_profile_homes(profile_homes)]
  530. if profile_gate is not None:
  531. enumerated = [(name, home) for name, home in enumerated if profile_gate(name, home)]
  532. cycle_homes = enumerated
  533. # Republish the owned set BEFORE any tick: the per-profile yield gate asks
  534. # "do I own cron for this home?" and a profile added or gated out this cycle
  535. # must be reflected in that answer, not one cycle late.
  536. register_ticked_homes([home for _name, home in cycle_homes])
  537. except BaseException as e:
  538. logger.error("Cron profile enumeration error: %s", e, exc_info=True)
  539. _tick_error = f"{type(e).__name__}: {e}"
  540. consecutive_failures = _note_tick_failure(e, consecutive_failures)
  541. try:
  542. if can_dispatch is not None and not can_dispatch():
  543. logger.debug("Cron dispatch paused while gateway drains existing work")
  544. note_held_profiles(cycle_homes)
  545. else:
  546. for _pname, home in cycle_homes:
  547. try:
  548. with _profile_cron_scope(home):
  549. note_dispatching()
  550. cron_tick(
  551. verbose=False, adapters=tick_adapters_for(_pname), loop=loop,
  552. sync=False, can_dispatch=can_dispatch,
  553. )
  554. except CronTickYielded as e:
  555. # Yield for THIS profile only; one fresh gateway must not stop others.
  556. logger.info("Cron tick yielded for profile at %s: %s", home, e)
  557. _profile_errors[str(home)] = f"{type(e).__name__}: {e}"
  558. except BaseException as e:
  559. # THIS profile only; BaseException as in the single-profile loop.
  560. logger.error(
  561. "Cron tick error for profile at %s: %s", home, e, exc_info=True
  562. )
  563. _profile_errors[str(home)] = f"{type(e).__name__}: {e}"
  564. if _cycle_exc is None or _is_fd_exhaustion(e):
  565. _cycle_exc = e
  566. ok = not _profile_errors
  567. if _cycle_exc is not None:
  568. consecutive_failures = _note_tick_failure(_cycle_exc, consecutive_failures)
  569. except BaseException as e:
  570. logger.error("Cron tick error: %s", e, exc_info=True)
  571. _tick_error = f"{type(e).__name__}: {e}"
  572. # EMFILE: reclaim fds + exponential backoff (#87644).
  573. consecutive_failures = _note_tick_failure(e, consecutive_failures)
  574. # Completed cycle: each profile's own outcome; aborted cycle: all beats unsuccessful.
  575. for _, home in cycle_homes:
  576. with _profile_cron_scope(home):
  577. _home_ok = _tick_error is None and str(home) not in _profile_errors
  578. _guarded_store_write(
  579. record_ticker_heartbeat, "heartbeat", success=_home_ok
  580. )
  581. if _home_ok:
  582. _guarded_store_write(clear_ticker_error, "error clear")
  583. elif str(home) in _profile_errors:
  584. _guarded_store_write(
  585. record_ticker_error,
  586. "tick error",
  587. _profile_errors[str(home)],
  588. )
  589. elif _tick_error:
  590. _guarded_store_write(
  591. record_ticker_error, "tick error", _tick_error
  592. )
  593. if ok:
  594. consecutive_failures = 0
  595. wait_for = _backoff_wait_seconds(interval, consecutive_failures)
  596. next_tick += wait_for
  597. now = time.monotonic()
  598. if next_tick < now:
  599. # Tick overran interval or host was suspended; re-anchor to avoid
  600. # burst-firing zero-length sleep cycles (#114467).
  601. next_tick = now + wait_for
  602. stop_event.wait(max(0.0, next_tick - now))