scheduler_gateway_gate.py 4.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. """Headless tick gate: agent jobs need a live gateway; a headless tick never starts one.
  2. ``run_canonical_job`` reaches the owner through ``connect_gateway`` → ``ensure_gateway_runtime``,
  3. which spawns an unmanaged daemon when nothing owns the profile. That is right for an interactive
  4. caller (``hermes cron run``) and wrong for ``hermes cron tick`` from a system crontab: every tick
  5. would leave a stray gateway behind. Headless surfaces refuse rather than spawn (same policy as
  6. approvals), so a headless tick skips its agent jobs while the gateway is down and lets them fire
  7. on the next tick once it is up — no run is recorded and the due instant does not move. The
  8. in-process ticker never reaches this gate: it runs inside the gateway by definition.
  9. "Down" is every outcome but ``ready``: a gateway still ``starting`` (or draining, or one whose
  10. control socket does not answer) would otherwise send the tick into ``ensure_gateway_runtime``'s
  11. 30 s wait and book the deadline as a FAILED run, consuming the slot — the crontab minute is the
  12. budget, and a slot lost to a restart window is a missed fire, not a job failure.
  13. """
  14. import logging
  15. import os
  16. logger = logging.getLogger(__name__)
  17. GATEWAY_DOWN_HINT = (
  18. "the gateway is not running, so agent jobs cannot fire; they run at the next tick once it is "
  19. "up. Start it with `hermes gateway start` (or `hermes gateway install` for a service)."
  20. )
  21. # A crontab tick has a minute; a gateway still starting answers within a few seconds or not at all.
  22. _HEADLESS_PROBE_TIMEOUT_S = 5.0
  23. def _gateway_unavailable(home) -> str | None:
  24. """Why a headless tick must hold its agent jobs, or None when a ready owner serves *home*.
  25. Everything short of ``ready`` — absent (``ensure`` would spawn), starting/draining (``ensure``
  26. would wait a whole deadline then fail the run), inaccessible/incompatible (no owner answers) —
  27. is held, never booked as a run."""
  28. from hermes_cli.gateway_runtime import discover_gateway_endpoint
  29. if os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip():
  30. return None # explicit remote gateway: connect_gateway never spawns for it
  31. observed = discover_gateway_endpoint(home, timeout=_HEADLESS_PROBE_TIMEOUT_S)
  32. if observed.state == "ready":
  33. return None
  34. if observed.state == "absent":
  35. return GATEWAY_DOWN_HINT
  36. reason = observed.state + (f" ({observed.reason_code})" if observed.reason_code else "")
  37. return (f"the gateway is {reason}, so agent jobs cannot fire; they run at the next tick once it "
  38. "is ready. Check it with `hermes gateway status`.")
  39. def _note_fire_refused(job_ids, detail: str) -> None:
  40. """Stamp ``last_fire_error`` (shown by ``hermes cron list``) and drop the dispatch marks the
  41. due scan just wrote: the slot was never handed over, so nothing must "restore" or "reclaim" it."""
  42. from cron.jobs import _hermes_now, _jobs_lock, load_jobs, save_jobs
  43. ids = set(job_ids)
  44. with _jobs_lock():
  45. jobs = load_jobs()
  46. for job in jobs:
  47. if job["id"] not in ids:
  48. continue
  49. job["last_fire_error"] = {"at": _hermes_now().isoformat(), "detail": detail[:500]}
  50. job.pop("pending_slot", None)
  51. if job.get("run_claim") is not None:
  52. job["run_claim"] = None
  53. save_jobs(jobs)
  54. def refuse_agent_jobs_without_gateway(due_jobs: list) -> list:
  55. """Return the due jobs a headless tick may dispatch; agent jobs are held back (skipped, not
  56. failed) when no ready gateway serves this home. Logs once per tick, not once per job."""
  57. from hermes_constants import get_hermes_home
  58. agent_jobs = [job for job in due_jobs if not job.get("no_agent")]
  59. if not agent_jobs:
  60. return due_jobs
  61. home = get_hermes_home().resolve()
  62. hint = _gateway_unavailable(home)
  63. if hint is None:
  64. return due_jobs
  65. ids = [str(job["id"]) for job in agent_jobs]
  66. _note_fire_refused(ids, f"Skipped: {hint}")
  67. logger.warning(
  68. "Cron tick for %s skipped %d agent job(s) (%s): %s",
  69. home, len(ids), ", ".join(ids), hint)
  70. return [job for job in due_jobs if job.get("no_agent")]