__init__.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207
  1. """Chronos — NAS-mediated managed cron provider (scale-to-zero).
  2. Instead of a 60s ticker, asks NAS to arm one external one-shot per job at its next-fire time;
  3. NAS calls back ``/api/cron/fire`` and the job re-arms after running. start() never blocks or
  4. spawns a periodic wake; reconcile runs only on a warm process (start / on_jobs_changed / fire).
  5. Holds no scheduler credentials — speaks only to NAS ``agent-cron`` endpoints with the Nous token.
  6. Inert unless ``cron.provider: chronos``. Wire contract: ``website/docs/developer-guide/chronos-managed-cron-contract.md``.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import threading
  11. from typing import Any, Dict
  12. from cron.scheduler_provider import CronScheduler
  13. from ._nas_client import NasCronClientError
  14. logger = logging.getLogger("cron.chronos")
  15. def _cfg(*keys: str, default: Any = "") -> Any:
  16. """Read a cron.chronos.* config value (no network)."""
  17. try:
  18. from hermes_cli.config import cfg_get, load_config
  19. return cfg_get(load_config(), *keys, default=default)
  20. except Exception:
  21. return default
  22. class ChronosCronScheduler(CronScheduler):
  23. """NAS-mediated external cron provider."""
  24. def __init__(self) -> None:
  25. # Best-effort job_id -> fire_at cache; a cold process simply re-arms (idempotent).
  26. self._armed: dict[str, str] = {}
  27. self._lock = threading.Lock()
  28. self._client = None # lazily constructed (no network in is_available)
  29. # Set when NAS answered 403 invalid_client: the Nous token in auth.json is not this
  30. # instance's provisioned identity, so every arm would fail the same way for the life of
  31. # the process. Once set, NAS is left alone and the built-in ticker fires jobs (#97494).
  32. self._identity_rejected = False
  33. self._stop_event = None
  34. self._ticker_kwargs: dict[str, Any] = {}
  35. @property
  36. def name(self) -> str:
  37. return "chronos"
  38. def is_available(self) -> bool:
  39. """Config presence only — NO network: portal URL, publicly reachable callback URL and a
  40. Nous login; otherwise the resolver falls back to the built-in ticker."""
  41. if not (_cfg("cron", "chronos", "portal_url") and _cfg("cron", "chronos", "callback_url")):
  42. return False
  43. # Stored-token presence only (no refresh); refresh-aware token resolved at provision time.
  44. try:
  45. from hermes_cli.auth import get_provider_auth_state
  46. return bool((get_provider_auth_state("nous") or {}).get("access_token"))
  47. except Exception:
  48. return False
  49. def _get_client(self):
  50. if self._client is None:
  51. from ._nas_client import NasCronClient
  52. self._client = NasCronClient(_cfg("cron", "chronos", "portal_url"))
  53. return self._client
  54. def _reconcile_logged(self, log, what: str) -> None:
  55. try:
  56. self.reconcile()
  57. except Exception as e:
  58. log("Chronos %s reconcile failed: %s", what, e)
  59. def start(self, stop_event, *, adapters=None, loop=None, interval=60):
  60. """Arm all enabled jobs via NAS, then RETURN — no loop, no periodic wake (scale-to-zero)."""
  61. # Kept so a later identity rejection (boot or mid-life re-arm) can hand this process's
  62. # fires to the built-in ticker with the gateway's own adapters/loop.
  63. self._ticker_kwargs = {"adapters": adapters, "loop": loop, "interval": interval}
  64. self._stop_event = stop_event
  65. # A new lifecycle can't prove what an interrupted process did: classify unknown, never requeue.
  66. self.recover_interrupted()
  67. self._reconcile_logged(logger.warning, "start()")
  68. def stop(self) -> None:
  69. pass
  70. def on_jobs_changed(self) -> None:
  71. if not self._identity_rejected:
  72. self._reconcile_logged(logger.debug, "on_jobs_changed")
  73. def register_job(self, job: dict[str, Any]) -> None:
  74. """Arm the first one-shot for a new job; may raise so creation can report it."""
  75. try:
  76. self._arm_one_shot(job)
  77. except NasCronClientError as e:
  78. if not e.identity_rejected:
  79. raise
  80. self._note_identity_rejected() # the job is stored; the ticker fires it
  81. def _arm_one_shot(self, job: dict[str, Any]) -> None:
  82. """Arm one one-shot at next_run_at (agent computes the time; NAS executes).
  83. dedup_key=(job_id, fire_at) makes re-arming the same fire a no-op."""
  84. if self._identity_rejected:
  85. return # the built-in ticker owns this process's fires; NAS would 403 again
  86. job_id = job["id"]
  87. fire_at = job.get("next_run_at")
  88. if not fire_at:
  89. return
  90. self._get_client().provision(
  91. job_id=job_id, fire_at=fire_at, dedup_key=f"{job_id}:{fire_at}",
  92. agent_callback_url=str(_cfg("cron", "chronos", "callback_url") or ""))
  93. with self._lock:
  94. self._armed[job_id] = fire_at
  95. def _note_identity_rejected(self) -> None:
  96. """403 invalid_client is deterministic: NAS maps the bearer to a provisioned instance via an
  97. ``agent:*`` client or the hosted bootstrap session, and a plain ``hermes auth`` login is
  98. neither — re-logging in cannot fix it, which is what users try first (#97494). Without
  99. NAS the jobs have no trigger at all (the misfire sweep runs them ``misfire_grace_minutes``
  100. late), so the built-in ticker takes over this process's fires."""
  101. with self._lock:
  102. if self._identity_rejected:
  103. return
  104. self._identity_rejected = True
  105. logger.warning(
  106. "Chronos: NAS rejected this agent's Nous credential for agent-cron (403 invalid_client). "
  107. "The Nous token in auth.json is not this instance's provisioned identity (an agent:* client "
  108. "or the hosted bootstrap session), so no job can be armed. A normal `hermes auth` re-login "
  109. "cannot fix this; the hosted credential has to be restored from the Nous Portal. Falling back "
  110. "to the built-in cron ticker for this process so scheduled jobs keep firing on time.")
  111. if self._stop_event is None:
  112. return # start() never ran (e.g. a CLI `hermes cron add`); nothing to tick here
  113. from agent.memory_provider import spawn_context_thread
  114. from cron.scheduler_provider import InProcessCronScheduler
  115. spawn_context_thread(
  116. InProcessCronScheduler().start, name="cron-scheduler-chronos-fallback",
  117. args=(self._stop_event,), kwargs=self._ticker_kwargs).start()
  118. def _arm_logged(self, job: dict[str, Any], what: str) -> None:
  119. """Best-effort arm: log a warning instead of raising (reconcile/fire must not die)."""
  120. try:
  121. self._arm_one_shot(job)
  122. except Exception as e:
  123. if isinstance(e, NasCronClientError) and e.identity_rejected:
  124. self._note_identity_rejected()
  125. return
  126. logger.warning("Chronos failed to %s: %s", what, e)
  127. def _cancel(self, job_id: str) -> None:
  128. try:
  129. self._get_client().cancel(job_id=job_id)
  130. finally:
  131. with self._lock:
  132. self._armed.pop(job_id, None)
  133. def _list_armed(self) -> dict[str, str]:
  134. """Armed one-shots (job_id -> fire_at): in-memory map when warm, else ask NAS ({} on
  135. failure — reconcile then re-arms idempotently)."""
  136. with self._lock:
  137. if self._armed:
  138. return dict(self._armed)
  139. try:
  140. observed = {item["job_id"]: item.get("fire_at", "")
  141. for item in self._get_client().list_armed() if item.get("job_id")}
  142. with self._lock:
  143. self._armed.update(observed)
  144. return observed
  145. except Exception as e:
  146. logger.debug("Chronos _list_armed failed (will re-arm idempotently): %s", e)
  147. return {}
  148. def reconcile(self) -> None:
  149. """Converge NAS one-shots toward jobs.json: arm missing/changed, cancel orphans."""
  150. from cron.jobs import get_job, load_jobs
  151. desired: dict[str, str] = {
  152. j["id"]: j["next_run_at"] for j in load_jobs()
  153. if j.get("enabled") and j.get("next_run_at") and j.get("state") != "paused"}
  154. observed = self._list_armed()
  155. for job_id, fire_at in desired.items():
  156. if observed.get(job_id) != fire_at and (job := get_job(job_id)):
  157. self._arm_logged(job, f"arm job {job_id}")
  158. for job_id in observed.keys() - desired.keys():
  159. try:
  160. self._cancel(job_id)
  161. except Exception as e:
  162. logger.warning("Chronos failed to cancel orphan %s: %s", job_id, e)
  163. # No ``fire_due`` override on purpose: ``provider_supports_split_fire`` treats ANY override as
  164. # the legacy single-phase signal, opting Chronos out of claim admission and cancel-aware drain.
  165. def fire_claimed(
  166. self, claimed_job: dict, *, adapters: Any = None, loop: Any = None, cancel_event: Any = None
  167. ) -> bool:
  168. ran = super().fire_claimed(claimed_job, adapters=adapters, loop=loop, cancel_event=cancel_event)
  169. if ran:
  170. from cron.jobs import get_job
  171. job_id = claimed_job["id"]
  172. job = get_job(job_id)
  173. if job and job.get("enabled") and job.get("next_run_at"):
  174. self._arm_logged(job, f"re-arm job {job_id} after fire")
  175. return ran
  176. def register(ctx) -> None:
  177. """Plugin entrypoint — plugins/cron_providers discovery collects the provider here."""
  178. ctx.register_cron_scheduler(ChronosCronScheduler())