| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045 |
- """Cron subcommand for hermes CLI."""
- import contextlib
- import json
- import re
- import sys
- from datetime import timezone, UTC
- from pathlib import Path
- from typing import Any, Dict, Iterable, List, Optional
- PROJECT_ROOT = Path(__file__).parent.parent.resolve()
- sys.path.insert(0, str(PROJECT_ROOT))
- from hermes_cli.colors import Colors, color
- def _normalize_skills(single_skill=None, skills: Optional[Iterable[str]] = None) -> Optional[list[str]]:
- """Deduped, stripped skill names; None when neither argument was given."""
- if skills is None and single_skill is None:
- return None
- normalized: list[str] = []
- for item in list(skills) if skills is not None else [single_skill]:
- text = str(item or "").strip()
- if text and text not in normalized:
- normalized.append(text)
- return normalized
- def _cron_api(**kwargs):
- from tools.cronjob_tools import cronjob as cronjob_tool
- return json.loads(cronjob_tool(**kwargs))
- def _active_cron_provider_name() -> str:
- """Resolved cron scheduler provider name ('builtin', 'chronos', …); 'builtin' on failure."""
- try:
- from cron.scheduler_provider import resolve_cron_scheduler
- return resolve_cron_scheduler().name or "builtin"
- except Exception:
- return "builtin"
- def _builtin_gateway_liveness() -> Optional[bool]:
- """Tri-state readiness of a scheduler serving THIS home (None = probe failed).
- A gateway launched from this home proves itself by process liveness. Any other scheduler that
- ticks this home — the host multiplexer for a served profile, or the in-process ticker inside
- ``hermes serve`` / the Desktop backend (#121881) — proves itself only by the fresh heartbeat it
- writes into this home's store. External providers use their own machinery and are exempt.
- """
- try:
- if _active_cron_provider_name() != "builtin":
- return True
- # The runtime lock is held for exactly the gateway's lifetime — more reliable than PID
- # scanning (find_gateway_pids transiently misses the gateway right after a restart, and
- # inside the gateway it must never say "not running"). A crashing probe is "unknown".
- with contextlib.suppress(Exception):
- from gateway.status import is_gateway_runtime_lock_active
- if is_gateway_runtime_lock_active():
- return True
- from hermes_cli.gateway import find_gateway_pids, named_profile_served_by_running_multiplexer
- if find_gateway_pids():
- return True
- from cron.jobs import get_ticker_heartbeat_age, ticker_heartbeat_writer_alive
- if not _ticker_age_is_fresh(get_ticker_heartbeat_age()):
- return False
- # A served named profile's heartbeat is the host multiplexer's, whose process the served
- # record already proves. Any other writer (the in-process ticker, a shared gateway whose
- # record omits this profile) must itself still be running: a killed ticker's last stamp
- # reads fresh for ~3 minutes and would queue manual runs nobody picks up.
- return named_profile_served_by_running_multiplexer() or ticker_heartbeat_writer_alive()
- except Exception:
- return None
- def _warn_if_gateway_not_running() -> None:
- """Warn at create/list time when the scheduler is not ready; stay silent on an unknown probe result."""
- if _builtin_gateway_liveness() is not False:
- return
- from hermes_cli.profiles import get_active_profile_name
- # Name the inspected profile: the shared gateway may well be running and ticking ANOTHER
- # profile's jobs, so a bare "not running" reads as a system claim it cannot make (#99579).
- print(color(f" ⚠ Scheduler is not ready for profile '{get_active_profile_name()}': "
- "no gateway serves it and no fresh ticker heartbeat.", Colors.YELLOW))
- print(color(" If no gateway is running: hermes gateway install\n"
- " sudo hermes gateway install --system # Linux servers\n"
- " Check status: hermes cron status", Colors.DIM))
- def _format_lateness(seconds: float) -> str:
- """Render a lateness duration compactly: '31m', '2h 30m', '45s'."""
- try:
- seconds = max(0, int(seconds))
- except (TypeError, ValueError):
- return "?"
- if seconds < 60:
- return f"{seconds}s"
- minutes, _ = divmod(seconds, 60)
- hours, minutes = divmod(minutes, 60)
- days, hours = divmod(hours, 24)
- parts = [(days, "d"), (hours, "h"), (minutes if not days else 0, "m")]
- return " ".join(f"{n}{unit}" for n, unit in parts if n) or "0m"
- def _dispatch_kind_label(kind) -> Optional[str]:
- return {"catch_up": "catch-up after missed fire", "late": "late"}.get(kind)
- def _next_run_overdue_seconds(next_run_at: Any) -> Optional[float]:
- """Seconds the stored ``next_run_at`` is already in the past (negative while still
- upcoming); None when it is not a parseable ISO timestamp.
- Parses through the scheduler's own ``_parse_aware`` so the CLI and the ticker agree on
- the instant (mixed UTC offsets, DST folds, legacy naive stamps read as system-local).
- """
- from cron.jobs import _parse_aware
- from hermes_time import now
- dt = _parse_aware(next_run_at)
- if dt is None:
- return None
- # Same-tzinfo subtraction is wall-clock arithmetic in Python; compare instants.
- return (now().astimezone(UTC) - dt.astimezone(UTC)).total_seconds()
- def _next_run_row(job: dict[str, Any]) -> tuple[str, str]:
- """``("Next run" | "Overdue", value)`` for one job.
- A stamp parked past `cron doctor`'s grace on a job that is supposed to fire is the only
- user-visible trace of a dead scheduler; never present it as an upcoming run (#114309).
- """
- stamp = job.get("next_run_at", "?")
- overdue_s = _next_run_overdue_seconds(stamp)
- if (overdue_s is None or overdue_s <= _OVERDUE_GRACE_SECONDS
- or not job.get("enabled", True) or job.get("state") in {"paused", "completed"}):
- return ("Next run", stamp)
- return ("Overdue", color(f"{stamp} ({_format_lateness(overdue_s)} ago — the job has not fired; "
- "is the scheduler running?)", Colors.YELLOW))
- def _dispatch_display(dispatch: dict) -> Optional[str]:
- """One-line scheduled-vs-actual dispatch summary; None when the stamp is malformed.
- On-time dispatches render dim; late/catch-up dispatches render loudly so a run fired long
- after gateway downtime doesn't look like an ordinary success.
- See #99879.
- """
- if not isinstance(dispatch, dict):
- return None
- scheduled, actual, kind = (dispatch.get(k) for k in ("scheduled_at", "dispatched_at", "kind"))
- if not scheduled or not actual or not kind:
- return None
- lateness = _format_lateness(dispatch.get("lateness_seconds", 0))
- if kind == "on_time":
- return color(f"on time (scheduled {scheduled})", Colors.DIM)
- label = _dispatch_kind_label(kind) or "late"
- return (color(f"⚠ {label}: ", Colors.YELLOW) + f"scheduled {scheduled}, ran {actual} "
- + color(f"({lateness} late)", Colors.YELLOW))
- def _print_banner(title: str) -> None:
- """Boxed cyan section header shared by ``cron list`` and ``cron incidents``."""
- print()
- rule = "─" * 73
- for line in (f"┌{rule}┐", "│" + " " * 25 + title.ljust(48) + "│", f"└{rule}┘"):
- print(color(line, Colors.CYAN))
- print()
- def _unverified_targets(unverified) -> str:
- return ", ".join(map(str, unverified)) if isinstance(unverified, list) else str(unverified)
- _STATE_BADGES = {"paused": ("[paused]", Colors.YELLOW), "completed": ("[completed]", Colors.BLUE)}
- def cron_list(show_all: bool = False):
- """List all scheduled jobs."""
- from cron.jobs import effective_job_state
- _store_report, jobs = _probe_then_list_jobs(include_disabled=True, brief=True)
- if jobs is None:
- return
- if not show_all:
- jobs = [
- job for job in jobs
- if job.get("enabled", True) or effective_job_state(job) == "paused"
- ]
- from hermes_cli.profiles import get_active_profile_name
- if not jobs:
- print(color(f"No scheduled jobs in profile '{get_active_profile_name()}'.\n"
- "Create one with 'hermes cron create ...' or the /cron command in chat.", Colors.DIM))
- return
- _print_banner(f"Scheduled Jobs (profile: {get_active_profile_name()})")
- for job in jobs:
- # effective_job_state honours the scheduler flag — never [paused] when enabled=true.
- badge = _STATE_BADGES.get(effective_job_state(job)) or (
- ("[active]", Colors.GREEN) if job.get("enabled", True) else ("[disabled]", Colors.RED))
- print(f" {color(job.get('id', '?'), Colors.YELLOW)} {color(*badge)}")
- for label, value in _job_rows(job):
- print(f" {label + ':':<11}{value}")
- for line in _job_warnings(job):
- print(f" {line}")
- print()
- _warn_if_gateway_not_running()
- def _last_run_display(job: dict[str, Any]) -> str:
- last_status = job["last_status"]
- if last_status == "ok":
- return color("ok", Colors.GREEN)
- if last_status == "delivery_queued":
- return color("finished; delivery is still in progress", Colors.YELLOW)
- if last_status == "delivery_failed":
- # Agent succeeded but the result never reached the user — not green; last_error is None.
- return color(f"ran, but the result was not delivered ({_short_reason(job.get('last_delivery_error'))}). "
- f"{_delivery_fix_hint(job)}", Colors.YELLOW)
- display = color(f"{last_status}: {job.get('last_error', '?')}", Colors.RED)
- streak = int(job.get("failure_streak") or 0)
- if streak >= 2:
- display += color(f" ({streak} failures in a row)", Colors.RED)
- return display
- def _job_rows(job: dict[str, Any]) -> list[tuple[str, str]]:
- """``(label, value)`` detail rows for one job in ``cron list``."""
- # `repeat` / `deliver` may be present-but-null (dict-default only covers a missing key).
- repeat_info = job.get("repeat") or {}
- repeat_times = repeat_info.get("times")
- # `deliver` may be present-but-null in the job record (same pitfall as `repeat` above), so coalesce to
- # the default rather than relying on the dict-default, which only applies to a missing key. A null value
- # would otherwise reach `", ".join(None)` and crash the whole listing (#32896).
- deliver = job.get("deliver") or ["local"]
- skills = job.get("skills") or ([job["skill"]] if job.get("skill") else [])
- monitor_source = job.get("monitor_script") or job.get("monitor_url")
- mon_state = job.get("monitor_state") or {}
- latest_execution = job.get("latest_execution") or {}
- optional = [
- ("Skills", ", ".join(skills) if skills else ""),
- ("Script", job.get("script")),
- ("Monitor", f"{monitor_source} (agent runs only on output change)" if monitor_source
- else ""),
- ("Changed", mon_state.get("last_changed_at") if monitor_source else ""),
- ("Mode", color("no-agent", Colors.DIM) + " (script stdout delivered directly)"
- if job.get("no_agent") else ""),
- ("Timezone", job.get("timezone")),
- ("Grace", f"{job['grace_minutes']} min" if job.get("grace_minutes") is not None else ""),
- ("Workdir", job.get("workdir")),
- ("Python", job.get("interpreter")),
- ("Last run", f"{job.get('last_run_at', '?')} {_last_run_display(job)}"
- if job.get("last_status") else ""),
- ("Dispatch", _dispatch_display(job.get("last_dispatch"))),
- ("Execution", f"{latest_execution.get('status', '?')} {latest_execution.get('id', '?')}"
- if latest_execution else "")]
- return [
- ("Name", job.get("name", "(unnamed)")),
- ("Schedule", job.get("schedule_display", job.get("schedule", {}).get("value", "?"))),
- ("Repeat", f"{repeat_info.get('completed', 0)}/{repeat_times}" if repeat_times else "∞"),
- _next_run_row(job),
- ("Deliver", deliver if isinstance(deliver, str) else ", ".join(deliver)),
- ] + [(label, value) for label, value in optional if value]
- def _short_reason(text: Any, limit: int = 120) -> str:
- """First line of an adapter/error blob, whitespace-collapsed and capped, or 'no details'."""
- first = str(text or "").strip().splitlines()
- reason = " ".join(first[0].split()) if first else ""
- return (reason[: limit - 1] + "…") if len(reason) > limit else (reason or "no details")
- def _delivery_fix_hint(job: dict[str, Any]) -> str:
- return (f"Check the target with `hermes cron status` or change it with "
- f"`hermes cron edit {job.get('id', '<id>')} --deliver <target>`.")
- def _missed_fire_issue(job: dict[str, Any], fire_err: dict[str, Any]) -> str:
- return (f"missed scheduled fire at {fire_err.get('at', '?')}: {_short_reason(fire_err['detail'])}. "
- "The messaging gateway was unreachable. Run `hermes gateway restart`, then "
- f"`hermes cron run {job.get('id', '<id>')}` to run it now.")
- def _job_warnings(job: dict[str, Any]) -> list[str]:
- """Delivery / fire warning lines for one job in ``cron list``."""
- lines = []
- if queued := job.get("last_delivery_queued"):
- lines.append(f"Delivery still in progress (the result was handed off but not confirmed yet): {queued}")
- if job.get("last_delivery_error"):
- lines.append(f"{color('⚠ The result was not delivered:', Colors.YELLOW)} "
- f"{_short_reason(job['last_delivery_error'])}. {_delivery_fix_hint(job)}")
- # A live adapter acked the last send but returned no message_id / raw_response
- # (Slack/Matrix/Mattermost shape): accepted as delivered, but say so here.
- if unverified := job.get("last_delivery_unverified"):
- lines.append(f"{color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} adapter acked "
- f"{_unverified_targets(unverified)} without message_id/raw_response")
- fire_err = job.get("last_fire_error")
- if isinstance(fire_err, dict) and fire_err.get("detail"):
- lines.append(color(f"⚠ {_missed_fire_issue(job, fire_err)}", Colors.RED))
- # Sticky last-failure stamp (#118354): survives a later success, so a job that
- # self-healed still shows that a run failed recently. While last_status itself
- # reports the failure this is redundant-but-consistent; after recovery it is the
- # only job-level trace left.
- last_failure = job.get("last_failure")
- if isinstance(last_failure, dict) and last_failure.get("detail"):
- recovered = str(job.get("last_status") or "") in {"ok", "delivery_queued"}
- lines.append(color(
- f"⚠ Last failure at {last_failure.get('at', '?')}: "
- f"{_short_reason(last_failure['detail'])}" + (" — recovered since" if recovered else ""),
- Colors.YELLOW if recovered else Colors.RED))
- return lines
- def cron_tick():
- """Run due jobs once and exit (external-scheduler mode: a system crontab calls this).
- No gateway runs here by definition, and none is started: agent jobs are skipped with a logged
- reason until the gateway is up (headless surfaces refuse rather than spawn)."""
- from cron.scheduler import CronTickYielded, tick
- from hermes_cli.observability.shared_metrics_process import begin_process
- begin_process("cron")
- try:
- tick(verbose=True, headless=True)
- except CronTickYielded as exc:
- # Inert for a one-shot CLI (no boot fingerprint); report cleanly rather than traceback.
- print(color(f"✗ {exc}", Colors.YELLOW))
- print(" A fresher gateway process owns the runtime lock and will fire due jobs; this "
- "stale process yielded its tick.")
- return 1
- except OSError as exc:
- # Real lock-acquisition failures (EMFILE, EACCES) propagate; they are not contention.
- # For the one-shot CLI surface, report cleanly instead of dumping a traceback; the gateway ticker
- # loop handles its own retry. See #87644.
- print(color(f"✗ Cron tick failed: {exc}", Colors.RED))
- print(" Check `hermes cron status` and the gateway log for details.")
- return 1
- return 0
- def cron_runs(job_id: Optional[str] = None, limit: int = 20):
- """Show indexed durable cron execution history."""
- from cron.executions import list_executions
- records = list_executions(job_id=job_id, limit=limit)
- if not records:
- print("No cron execution attempts recorded.")
- return
- for record in records:
- print(f"{record.get('id', '?')} {record.get('status', '?'):<9} "
- f"job={record.get('job_id', '?')} source={record.get('source', '?')} "
- f"{record.get('claimed_at', '?')}")
- if record.get("error"):
- print(f" {record['error']}")
- _INCIDENT_STATE_COLORS = {"detected": Colors.RED, "alerted": Colors.YELLOW, "resolved": Colors.GREEN,
- "closed": Colors.DIM}
- def cron_incidents(args) -> int:
- """List (``[--state <s>]``) or ``ack <id>`` durable cron failure incidents.
- Acking closes an incident so its failure ping stays silent until the error signature changes.
- """
- from cron.incidents import ack_incident, list_incidents
- action = getattr(args, "incident_action", "list")
- if action == "ack":
- incident_id = getattr(args, "incident_id", None)
- if not incident_id:
- print(color("✗ Incident ID required: hermes cron incidents ack <incident_id>", Colors.RED))
- return 1
- if ack_incident(incident_id):
- print(color(f"✓ Incident {incident_id} acknowledged (closed).", Colors.GREEN))
- else:
- print(color(f"Incident {incident_id} not found or already closed.", Colors.YELLOW))
- return 0
- state = getattr(args, "state", None)
- incidents = list_incidents(state=state)
- if not incidents:
- print(color("No cron failure incidents recorded.", Colors.DIM))
- if state:
- print(color(f" (filtered by state '{state}')", Colors.DIM))
- return 0
- _print_banner("Cron Failure Incidents")
- for inc in incidents:
- state_display = color(inc["state"], _INCIDENT_STATE_COLORS.get(inc["state"], Colors.DIM))
- error_text = re.sub(r"\s+", " ", inc.get("error") or "").strip()
- if len(error_text) > 160:
- error_text = error_text[:157].rstrip() + "..."
- rows = [("Job", inc["job_id"]), ("Type", inc.get("failure_type", "unknown")),
- ("First seen", inc.get("first_seen_at", "?")),
- ("Last seen", inc.get("last_seen_at", "?")), ("Error", error_text),
- ("Output", inc.get("output_file"))]
- print(f" {color(inc['id'], Colors.YELLOW)} {state_display}")
- for label, value in rows:
- if label != "Output" or value:
- print(f" {label + ':':<12}{value}")
- print()
- print(color(f" {len(incidents)} incident(s) | ack one with: hermes cron incidents ack <id>",
- Colors.DIM))
- return 0
- _PERMISSION_HINT = (" Hint: jobs.json may be owned by another user (e.g. rewritten by a root "
- "`docker exec hermes hermes cron ...`). Fix ownership to match the gateway "
- "user, and prefer `docker exec -u <uid>:<gid>`.")
- _FD_EXHAUSTION_HINT = (" Hint: the ticker hit file-descriptor exhaustion (EMFILE). The scheduler "
- "now retries with backoff and attempts fd reclamation, but if the leak "
- "persists, restart the gateway to recover scheduling.")
- def _ticker_age_is_fresh(age: Optional[float]) -> bool:
- from cron.jobs import TICKER_INTERVAL_SECONDS
- return age is not None and age <= TICKER_INTERVAL_SECONDS * 3 + 20
- def _store_unwritable_report() -> Optional[dict]:
- """Probe the active cron store (never a marker inside it): ``None`` when it accepts writes.
- Runs before any job read: loading jobs re-secures the dir, which can mask a mode problem."""
- from cron.jobs import _current_cron_store
- from cron.store_health import probe_report
- return probe_report(_current_cron_store().cron_dir)
- def _probe_then_list_jobs(*, include_disabled: bool, brief: bool = False):
- """``(store report or None, jobs or None)``. Probes and prints the unwritable-store warning
- BEFORE reading jobs: loading re-secures the dir mode (masking a mode problem), and on an
- unwritable store the read itself can fail (it creates cron/output). Jobs is ``None`` only
- when that read failed on a store already reported unwritable."""
- from cron.jobs import list_jobs
- if (report := _store_unwritable_report()) is not None:
- _print_store_unwritable(report, brief=brief)
- try:
- return report, list_jobs(include_disabled=include_disabled)
- except OSError:
- if report is None:
- raise
- return report, None
- def _print_store_unwritable(report: dict, *, brief: bool = False) -> None:
- if brief: # `cron list`: one line
- print(color(f"⚠ Cron store {report['store']} is NOT writable ({report['error']}) — scheduled jobs "
- f"are being skipped; {report['fix']}. See `hermes cron status`.", Colors.RED))
- return
- print(color("⚠ Cron store is NOT writable — scheduled jobs are being skipped", Colors.RED))
- print(color(f" {report['store']}: {report['error']} (last successful write {report['since']})", Colors.RED))
- print(color(f" Fix: {report['fix']}. Each due job then fires once on the next tick.", Colors.YELLOW))
- def _print_ticker_health(pids: list, restart_command: str = "hermes gateway restart") -> None:
- """Report builtin-ticker liveness for a gateway process known to be alive.
- The ticker THREAD can die silently or stay alive while every tick fails, so check both
- the liveness heartbeat and the last-successful-tick marker before saying "will fire".
- """
- # See #32612, #32895.
- from cron.jobs import (
- get_ticker_heartbeat_age, get_ticker_last_error, get_ticker_success_age)
- from cron.scheduler import _is_fd_exhaustion_text as _cron_is_fd_exhaustion_text
- from cron.scheduler import stale_code_yield_labels
- hb_age = get_ticker_heartbeat_age()
- ok_age = get_ticker_success_age()
- last_error = get_ticker_last_error()
- pid_line = f" PID: {', '.join(map(str, pids))}" if pids else None
- def _warn(headline: str) -> None:
- print(color(headline, Colors.YELLOW))
- if pid_line:
- print(pid_line)
- if hb_age is None:
- # Ticker never started (non-cron profile, gateway just started, or a config issue).
- _warn("⚠ Gateway is running but the cron ticker has not reported a heartbeat.")
- print(" Cron jobs will NOT fire until the ticker writes its first heartbeat.\n"
- " If the gateway just started, wait ~60s and re-run `hermes cron status`.\n"
- f" If heartbeat never appears, restart: {restart_command}")
- elif not _ticker_age_is_fresh(hb_age): # ticker thread is gone
- _warn("⚠ Gateway is running but the cron ticker looks STALLED — "
- f"no heartbeat for {int(hb_age)}s (expected every ~60s).")
- print(f" Cron jobs may NOT be firing. Restart: {restart_command}")
- elif (skew := stale_code_yield_labels(last_error)) is not None:
- # `hermes update` moved the checkout under a running gateway: its ticker yields every
- # tick (heartbeat stays fresh, nothing dispatches) until the process is restarted (#117275).
- _warn("⚠ Gateway is running STALE code — its cron ticker yields every tick and "
- "fires NOTHING.")
- print(color(f" Booted on {skew[0]}, checkout is now at {skew[1]} "
- "(the code was updated under the running gateway).", Colors.RED))
- print(f" Restart it onto the new code: {restart_command}")
- elif (ok_age is not None and not _ticker_age_is_fresh(ok_age)) or (ok_age is None and last_error):
- # Loop alive but every tick fails (or has never succeeded since boot).
- _warn("⚠ Gateway and cron ticker are running, but no tick has "
- f"succeeded {'in ' + str(int(ok_age)) + 's' if ok_age is not None else 'yet'} "
- "— ticks may be failing.")
- if last_error:
- # WHY ticks fail: root-rewritten jobs.json (PermissionError) or fd exhaustion.
- # Show WHY ticks fail — e.g. a root-rewritten jobs.json (PermissionError) that silently locked
- # out the ticker's uid for ~14h in the field (#68483), or fd exhaustion (EMFILE) that used to
- # stall the scheduler invisibly (#87644).
- print(color(f" Last tick error: {last_error}", Colors.RED))
- if "Permission denied" in last_error:
- print(color(_PERMISSION_HINT, Colors.YELLOW))
- elif _cron_is_fd_exhaustion_text(last_error):
- print(color(_FD_EXHAUSTION_HINT, Colors.YELLOW))
- print(" Check the gateway log for 'Cron tick error'.")
- else:
- print(color("✓ Gateway is running — cron jobs will fire automatically", Colors.GREEN))
- if pid_line:
- print(pid_line)
- if hb_age is not None:
- print(f" Ticker heartbeat: {int(hb_age)}s ago")
- def cron_status():
- """Show cron execution status."""
- from hermes_cli.gateway import find_gateway_pids, named_profile_served_by_running_multiplexer
- from hermes_cli.profiles import get_active_profile_name
- print()
- store_report, active_jobs = _probe_then_list_jobs(include_disabled=False)
- if active_jobs is None:
- print(color(" Jobs could not be read from the unwritable store.", Colors.RED))
- elif store_report is not None:
- overdue = sum(1 for j in active_jobs if (_next_run_overdue_seconds(j.get("next_run_at")) or 0) > 0)
- print(color(f" {overdue} due run(s) not fired", Colors.RED))
- if store_report is not None:
- print()
- provider = _active_cron_provider_name()
- if provider != "builtin":
- # External providers fire via webhook: no ticker thread / heartbeat file by design, so
- # the liveness heuristics would always say "stalled".
- print(color(f"✓ Cron provider: {provider} — jobs fire via the managed scheduler, "
- "not the in-process ticker.", Colors.GREEN))
- print(color(" (No ticker heartbeat is expected for an external provider; "
- "due jobs are delivered by an authenticated webhook.)", Colors.DIM))
- else:
- from gateway.host_topology import host_gateway_serving
- active = get_active_profile_name()
- # FIRST question under multiplex-only: is the HOST gateway alive and does it tick THIS
- # profile? Starting from find_gateway_pids() (argv `-p <name>`) made every served profile
- # report "not running" and told the user to start a SECOND host process.
- host = None
- with contextlib.suppress(Exception):
- host = host_gateway_serving(active)
- pids = [] if host is not None else find_gateway_pids()
- gateway_alive_via_lock = False
- served_by_multiplexer = False
- in_process_ticker = False
- if host is None and not pids:
- # The pid scan transiently misses a live gateway right after a restart; the runtime
- # lock proves the process is alive. Declare "not running" only when both agree.
- with contextlib.suppress(Exception):
- # Same false-alarm class the cronjob tool fixed (#95947): the pid scan can transiently miss
- # a live gateway (just after a restart) while the runtime lock — held for exactly the
- # gateway's lifetime — proves the ticker's process is alive.
- from gateway.status import get_running_pid, is_gateway_runtime_lock_active
- gateway_alive_via_lock = is_gateway_runtime_lock_active()
- lock_pid = get_running_pid() if gateway_alive_via_lock else None
- pids = [lock_pid] if lock_pid else pids
- # Multiplexer identity does not establish the active profile's ticker health.
- if not gateway_alive_via_lock:
- served_by_multiplexer = named_profile_served_by_running_multiplexer()
- if not gateway_alive_via_lock and not served_by_multiplexer:
- # `hermes serve` / the Desktop backend ticks every local profile in-process: no
- # gateway argv, lock or host record — only the heartbeat it writes here (#121881).
- with contextlib.suppress(Exception):
- from cron.jobs import get_ticker_heartbeat_age, ticker_heartbeat_writer_alive
- in_process_ticker = (_ticker_age_is_fresh(get_ticker_heartbeat_age())
- and ticker_heartbeat_writer_alive())
- if host is not None:
- print(f" Scheduler host: {host.describe()}")
- # `hermes gateway restart` exits 78 for a served NAMED profile
- # (_guard_named_profile_under_multiplexer): name the profile that LAUNCHED the host
- # process. On a standalone fleet that is this profile itself, not default (#120871).
- owner = None
- with contextlib.suppress(Exception):
- from hermes_cli.gateway import host_multiplexer_serving
- owner = host_multiplexer_serving(active)
- host_profile = owner.profile_label if owner is not None else "default"
- _print_ticker_health([host.pid], restart_command=f"hermes --profile {host_profile} gateway restart")
- elif pids or gateway_alive_via_lock or served_by_multiplexer or in_process_ticker:
- if served_by_multiplexer:
- print(" Scheduler host: the host gateway (multiplexing this profile)")
- _print_ticker_health([], restart_command="hermes --profile default gateway restart")
- elif in_process_ticker:
- print(f" Scheduler host: an in-process ticker (hermes serve / Desktop backend) ticking profile '{active}'")
- _print_ticker_health([], restart_command="restart the Hermes Desktop app (or its serve backend)")
- else:
- _print_ticker_health(pids)
- else:
- # Only THIS profile's scheduler is knowable from here: a gateway may be alive on the
- # host ticking other profiles, so never assert a host-wide negative (#99579).
- print(color(f"✗ No scheduler is serving profile '{active}' — its cron jobs will NOT fire", Colors.RED))
- # When scheduling last worked before the host went away: without this, a
- # 7h-overdue job still reads as a normal upcoming "Next run" (#114309).
- with contextlib.suppress(Exception):
- from cron.jobs import TICKER_INTERVAL_SECONDS, get_ticker_heartbeat_age
- hb_age = get_ticker_heartbeat_age()
- if hb_age is not None and hb_age > TICKER_INTERVAL_SECONDS * 3 + 20:
- print(color(" Scheduler last ticked "
- f"{_format_lateness(hb_age)} ago — jobs that came due "
- "since then have not fired.", Colors.YELLOW))
- print("\n Start the ONE host gateway (it multiplexes every profile, this one included):\n"
- " hermes --profile default gateway install # user service\n"
- " sudo hermes --profile default gateway install --system # Linux servers: boot-time service\n"
- " hermes --profile default gateway run # Or run in foreground")
- if active not in ("default", "custom"):
- print("\n It serves this profile automatically. If a per-profile service or gateway\n"
- " from an older release is still installed, fold it in (preflight + dry run):\n"
- " hermes --profile default gateway migrate --multiplex --dry-run\n"
- " hermes --profile default gateway migrate --multiplex\n"
- " Check: hermes cron status from this profile should show its ticker heartbeat.\n")
- print()
- _print_active_jobs_summary(active_jobs)
- print()
- def _print_active_jobs_summary(jobs) -> None:
- """Print the '<N> active job(s)' + next-run line shared by every status path."""
- if not jobs:
- print(" No active jobs")
- return
- from cron.jobs import _parse_aware
- # Stored stamps carry mixed UTC offsets (an interval job keeps last_run_at's offset, a cron
- # job its configured zone), so order by instant, never by ISO text; display the stored stamp.
- # `_parse_aware` hands back one shared ZoneInfo, and Python compares same-tzinfo datetimes
- # by wall clock (wrong across a DST fold) — normalise to UTC before ordering.
- next_runs = [(parsed.astimezone(UTC), j["next_run_at"]) for j in jobs
- if (parsed := _parse_aware(j.get("next_run_at"))) is not None]
- print(f" {len(jobs)} active job(s)")
- if next_runs:
- earliest = min(next_runs, key=lambda run: run[0])[1]
- overdue_by = _next_run_overdue_seconds(earliest)
- if overdue_by is not None and overdue_by > _OVERDUE_GRACE_SECONDS:
- # #114309: a dead scheduler leaves next_run_at stranded in the past; presenting it
- # as an upcoming "Next run" hides the outage. Same 15m grace as `cron doctor`
- # (_OVERDUE_GRACE_SECONDS) so a job a few minutes behind the ticker's own
- # cadence doesn't flash OVERDUE here while doctor still calls it healthy.
- print(color(f" ⚠ Next run {earliest} is OVERDUE — passed "
- f"{_format_lateness(overdue_by)} ago but the job has not fired "
- "(is the scheduler running?)", Colors.YELLOW))
- else:
- print(f" Next run: {earliest}")
- # Post-downtime late fires show at status level, not just per-job in `cron list`.
- late = [j for j in jobs if isinstance(j.get("last_dispatch"), dict)
- and j["last_dispatch"].get("kind") in ("late", "catch_up")]
- if late:
- print()
- print(color(f" ⚠ {len(late)} job(s) last fired late:", Colors.YELLOW))
- for j in late:
- d = j["last_dispatch"]
- late_by = _format_lateness(d.get("lateness_seconds", 0))
- print(f" {j.get('id', '?')} {j.get('name', '(unnamed)')}: {_dispatch_kind_label(d.get('kind'))}, "
- f"scheduled {d.get('scheduled_at', '?')}, ran {d.get('dispatched_at', '?')} "
- + color(f"({late_by} late)", Colors.YELLOW))
- def _scripts_dir_for_cron() -> Path:
- """Scripts dir for cron jobs — via ``CRON_DIR`` so monkeypatched cron storage is honoured."""
- from cron.jobs import CRON_DIR
- return CRON_DIR.parent / "scripts"
- def _script_health_issue(script: str) -> Optional[str]:
- """Human-readable script issue, or ``None`` when the path is OK."""
- scripts_dir = _scripts_dir_for_cron().resolve()
- raw = Path(script).expanduser()
- path = raw.resolve() if raw.is_absolute() else (scripts_dir / raw).resolve()
- try:
- path.relative_to(scripts_dir)
- except ValueError:
- return f"script resolves outside {scripts_dir}: {script!r}"
- if not path.exists():
- return f"script not found: {path}"
- if not path.is_file():
- return f"script path is not a file: {path}"
- return None
- # A busy tick can push dispatch a few minutes late; only a next_run_at parked well in the past
- # means the job is silently not firing (ticker dead, gateway down, wedged fire-claim).
- # `cron status`'s OVERDUE line shares this grace so status, list, and doctor tell one
- # consistent story about when a job counts as overdue.
- _OVERDUE_GRACE_SECONDS = 15 * 60
- def _next_run_overdue_issue(next_run: str) -> Optional[str]:
- """Issue string when ``next_run_at`` is parked in the past."""
- overdue_s = _next_run_overdue_seconds(next_run)
- if overdue_s is None:
- return f"next_run_at is not a valid timestamp: {next_run!r}"
- if overdue_s <= _OVERDUE_GRACE_SECONDS:
- return None
- amount = f"{overdue_s / 3600:.1f}h" if overdue_s >= 3600 else f"{overdue_s / 60:.0f}m"
- return f"next_run_at is {amount} overdue — job is not firing (is the scheduler running?)"
- def _cron_doctor_issues_for_job(job: dict[str, Any]) -> list[str]:
- issues: list[str] = []
- last_status = str(job.get("last_status") or "").strip().lower()
- # "delivery_failed" = the agent run succeeded; the delivery issue below reports it.
- if last_status and last_status not in {"ok", "delivery_failed", "delivery_queued"}:
- issues.append(f"last run failed: {str(job.get('last_error') or 'unknown error').strip()}")
- if delivery_err := str(job.get("last_delivery_error") or "").strip():
- issues.append(f"last run finished but the result was not delivered ({_short_reason(delivery_err)}). "
- f"{_delivery_fix_hint(job)}")
- if unverified := job.get("last_delivery_unverified"):
- issues.append("last delivery unverified (adapter acked without evidence): "
- + _unverified_targets(unverified))
- # Dispatch records measure lateness, not whether the scheduler process was running.
- if isinstance(dispatch := job.get("last_dispatch"), dict):
- if label := _dispatch_kind_label(dispatch.get("kind")):
- issues.append(f"last fire was {label} (scheduled {dispatch.get('scheduled_at', '?')}, "
- f"{_format_lateness(dispatch.get('lateness_seconds', 0))} late). "
- "This warning clears at the next on-time fire.")
- if isinstance(fire_err := job.get("last_fire_error"), dict) and fire_err.get("detail"):
- # The handoff error survives next_run_at advancing beyond the failed dispatch.
- issues.append(_missed_fire_issue(job, fire_err))
- if job.get("enabled", True) and job.get("state") not in {"paused", "completed"}:
- next_run = str(job.get("next_run_at") or "").strip()
- issue = _next_run_overdue_issue(next_run) if next_run else "active job has no next_run_at"
- if issue:
- issues.append(issue)
- script = str(job.get("script") or "").strip()
- if job.get("no_agent") and not script:
- issues.append("no-agent job has no script")
- if script and (script_issue := _script_health_issue(script)):
- issues.append(script_issue)
- workdir = str(job.get("workdir") or "").strip()
- if workdir and not Path(workdir).expanduser().exists():
- issues.append(f"workdir not found: {workdir}")
- if job.get("timezone"):
- from cron.jobs_time import zone_for_job
- if zone_for_job(job) is None:
- issues.append(f"timezone {job['timezone']!r} is not a valid IANA zone; "
- "the job falls back to the profile timezone")
- if job.get("grace_minutes") is not None:
- from cron.jobs_time import normalize_grace_minutes
- try:
- normalize_grace_minutes(job.get("grace_minutes"))
- except ValueError as exc:
- issues.append(f"grace_minutes is unusable ({exc})")
- else:
- if (job.get("schedule") or {}).get("kind") != "cron":
- issues.append("grace_minutes has no effect on a non-cron schedule")
- return issues
- def cron_doctor() -> int:
- """Run read-only cron health checks and return a shell-friendly status."""
- from cron.jobs import list_jobs
- jobs = list_jobs(include_disabled=False)
- findings = [(job, issues) for job in jobs if (issues := _cron_doctor_issues_for_job(job))]
- if not findings:
- print(color("✓ Cron doctor found no issues", Colors.GREEN))
- note = f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured."
- print(color(note, Colors.DIM))
- return 0
- issue_count = sum(len(issues) for _, issues in findings)
- print(color(f"Cron doctor found {issue_count} issue(s) across {len(findings)} job(s):", Colors.YELLOW))
- print()
- for job, issues in findings:
- print(f" {color(job.get('id', '?'), Colors.YELLOW)} {job.get('name', '(unnamed)')}")
- for issue in issues:
- print(f" - {issue}")
- print()
- print(color("Review the findings above, then run `hermes cron doctor` again.", Colors.DIM))
- return 1
- _JOB_ARG_FIELDS = (("name", "name"), ("deliver", "deliver"), ("failure_deliver", "failure_deliver"),
- ("repeat", "repeat"), ("script", "script"), ("workdir", "workdir"),
- ("model", "model"), ("provider", "model_provider"), ("pinned", "pinned"),
- ("monitor_script", "monitor_script"), ("monitor_url", "monitor_url"),
- ("continuity", "continuity"), ("reasoning_effort", "reasoning_effort"),
- ("interpreter", "interpreter"), ("timezone", "timezone"),
- ("grace_minutes", "grace_minutes"), ("catch_up", "catch_up"))
- def _job_api_kwargs(args) -> dict[str, Any]:
- """Collect the create/update kwargs shared by ``cron create`` and ``cron edit``."""
- return {api_key: getattr(args, attr, None) for api_key, attr in _JOB_ARG_FIELDS}
- _JOB_DETAIL_LINES = (
- ("script", " Script: {}"),
- ("monitor_script", " Monitor: {} (agent runs only on output change)"),
- ("monitor_url", " Monitor: {} (agent runs only on output change)"),
- ("no_agent", " Mode: no-agent (script stdout delivered directly)"),
- ("continuity", " Continuity: on (each run sees the previous run's output)"),
- ("timezone", " Timezone: {} (schedule runs on this zone's wall clock)"),
- ("workdir", " Workdir: {}"),
- ("interpreter", " Python: {}"))
- def _print_job_details(job_data: dict[str, Any]) -> None:
- """Print the optional Script/Monitor/Mode/Continuity/Workdir lines of a job record."""
- for key, template in _JOB_DETAIL_LINES:
- if job_data.get(key):
- print(template.format(job_data[key]))
- def cron_create(args):
- # The gateway-lifecycle guard lives in cron.jobs.create_job (every creation path); a block
- # surfaces as result["error"].
- result = _cron_api(
- action="create", schedule=args.schedule, prompt=args.prompt,
- skill=getattr(args, "skill", None),
- skills=_normalize_skills(getattr(args, "skill", None), getattr(args, "skills", None)),
- no_agent=getattr(args, "no_agent", False) or None,
- **({"paused": args.paused, "paused_reason": getattr(args, "paused_reason", None)}
- if getattr(args, "paused", False) or getattr(args, "paused_reason", None) is not None else {}),
- **_job_api_kwargs(args))
- if not result.get("success"):
- print(color(f"Failed to create job: {result.get('error', 'unknown error')}", Colors.RED))
- return 1
- print(color(f"Created job: {result['job_id']}", Colors.GREEN))
- print(f" Name: {result['name']}\n Schedule: {result['schedule']}")
- if result.get("skills"):
- print(f" Skills: {', '.join(result['skills'])}")
- _print_job_details(result.get("job", {}))
- if not result.get("job", {}).get("enabled", True):
- print(" Created PAUSED — resume to schedule, or explicitly run now.")
- else:
- print(f" Next run: {result['next_run_at']}")
- _warn_if_gateway_not_running()
- return 0
- def cron_edit(args):
- from cron.jobs import AmbiguousJobReference, resolve_job_ref
- try:
- job = resolve_job_ref(args.job_id)
- except AmbiguousJobReference as exc:
- print(color(str(exc), Colors.RED))
- for m in exc.matches:
- print(f" {m['id']} (name: {m.get('name')!r})")
- return 1
- if not job:
- print(color(f"Job not found: {args.job_id}", Colors.RED))
- return 1
- existing_skills = list(job.get("skills") or ([job["skill"]] if job.get("skill") else []))
- replacement_skills = _normalize_skills(getattr(args, "skill", None), getattr(args, "skills", None))
- add_skills = _normalize_skills(None, getattr(args, "add_skills", None)) or []
- remove_skills = set(_normalize_skills(None, getattr(args, "remove_skills", None)) or [])
- final_skills = None
- if getattr(args, "clear_skills", False):
- final_skills = []
- elif replacement_skills is not None:
- final_skills = replacement_skills
- elif add_skills or remove_skills:
- final_skills = [skill for skill in existing_skills if skill not in remove_skills]
- final_skills += [skill for skill in add_skills if skill not in final_skills]
- result = _cron_api(action="update", job_id=args.job_id,
- schedule=getattr(args, "schedule", None),
- prompt=getattr(args, "prompt", None), skills=final_skills,
- no_agent=getattr(args, "no_agent", None), **_job_api_kwargs(args))
- if not result.get("success"):
- print(color(f"Failed to update job: {result.get('error', 'unknown error')}", Colors.RED))
- return 1
- updated = result["job"]
- print(color(f"Updated job: {updated['job_id']}", Colors.GREEN))
- print(f" Name: {updated['name']}\n Schedule: {updated['schedule']}")
- print(f" Skills: {', '.join(updated['skills'])}" if updated.get("skills") else
- " Skills: none")
- _print_job_details(updated)
- return 0
- def _job_action(action: str, job_id: str, success_verb: str) -> int:
- _stateless_token = None
- if action == "run":
- # One-shot CLI: a background-dispatched run (daemon thread, triggered when the CLI
- # inherits HERMES_SESSION_KEY) would be orphaned mid-LLM-call, leaving the execution row
- # stuck 'claimed'. Declaring the channel stateless forces a synchronous run; scoped to
- # this call so in-process callers (tests, embedding apps) are not tainted.
- with contextlib.suppress(Exception):
- # The background path in ``_try_dispatch_background_run`` triggers when the CLI inherits a
- # gateway/desktop session env (HERMES_SESSION_KEY); declare the channel stateless so
- # ``async_delivery_supported()`` gates it off and the run executes synchronously to completion
- # instead. See #86721.
- from gateway.session_context import _SESSION_ASYNC_DELIVERY
- _stateless_token = _SESSION_ASYNC_DELIVERY.set(False)
- try:
- result = _cron_api(action=action, job_id=job_id)
- finally:
- if _stateless_token is not None:
- _SESSION_ASYNC_DELIVERY.reset(_stateless_token)
- if not result.get("success"):
- print(color(f"Failed to {action} job: {result.get('error', 'unknown error')}", Colors.RED))
- return 1
- job = result.get("job") or result.get("removed_job") or {}
- print(color(f"{success_verb} job: {job.get('name', job_id)} ({job_id})", Colors.GREEN))
- if action in {"resume", "run"} and result.get("job", {}).get("next_run_at"):
- print(f" Next run: {result['job']['next_run_at']}")
- if action == "run":
- print(f" {_run_outcome(result.get('job', {}))}")
- if (job.get('executed') and job.get('execution_success') is False
- and job.get('last_status') != 'cancelled'
- and not job.get('delegation_id') and job.get('execution_mode') != 'background'):
- return 1
- return 0
- def _run_outcome(job: dict[str, Any]) -> str:
- """One-line verdict for a manual run.
- A background-dispatched run (execution_mode="background" / delegation_id) keeps running
- after this CLI exits, so report the dispatch rather than a success/failure verdict.
- """
- if job.get("delegation_id"):
- return f"Running in background (delegation {job['delegation_id']})."
- if job.get("execution_mode") == "background":
- return "Running in background."
- if job.get("executed"):
- if not job.get("execution_success") and job.get("last_status") == "cancelled":
- return "Ran now: cancelled."
- return f"Ran now: {'succeeded' if job.get('execution_success') else 'failed'}."
- return job.get("execution_skipped") or "It will run on the next scheduler tick."
- def cron_resume(args) -> int:
- """Resume a paused job or explicitly re-arm a completed one-shot."""
- run_at = getattr(args, "run_at", None)
- run_now = getattr(args, "run_now", False)
- if run_at and run_now:
- print(color("Use exactly one of --at or --run-now.", Colors.RED))
- return 1
- if not run_at and not run_now:
- return _job_action("resume", args.job_id, "Resumed")
- from cron.jobs import AmbiguousJobReference, _hermes_now, rearm_oneshot
- try:
- job = rearm_oneshot(args.job_id, _hermes_now().isoformat() if run_now else run_at)
- except (AmbiguousJobReference, ValueError) as exc:
- print(color(f"Failed to re-arm job: {exc}", Colors.RED))
- return 1
- if not job:
- print(color(f"Job not found: {args.job_id}", Colors.RED))
- return 1
- print(color(f"Re-armed job: {job.get('name', args.job_id)} ({args.job_id})", Colors.GREEN)
- + f"\n Next run: {job.get('next_run_at')}")
- return 0
- def cron_notepad(args) -> int:
- """Handle ``hermes cron notepad <job_id> [get|set|delete|list]`` (per-job durable KV).
- A running cron agent updates its own notepad via its terminal tool; the scheduler injects
- non-empty notepads into the job prompt on each run.
- """
- from cron import notepad
- job_id = str(getattr(args, "job_id", "") or "")
- action = getattr(args, "notepad_action", None) or "list"
- key = getattr(args, "key", None)
- value = getattr(args, "value", None)
- if not job_id:
- print(color("A job ID is required.", Colors.RED))
- return 1
- try:
- if action not in ("set", "get", "delete"): # list (default)
- notes = notepad.list_notes(job_id)
- if not notes:
- print(color(f"Notepad for job {job_id} is empty.", Colors.DIM))
- for note in notes:
- print(f" {color(note['key'], Colors.YELLOW)} = {note['value']}\n"
- f" {color('updated: ' + str(note['updated_at']), Colors.DIM)}")
- return 0
- usage_args = "set <key> <value>" if action == "set" else f"{action} <key>"
- if key is None or (action == "set" and value is None):
- print(color(f"Usage: hermes cron notepad <job_id> {usage_args}", Colors.RED))
- return 1
- if action == "set":
- notepad.set_note(job_id, key, value)
- print(color(f"Set notepad key '{key}' for job {job_id}.", Colors.GREEN))
- return 0
- if action == "get":
- stored = notepad.get_note(job_id, key)
- if stored is not None:
- print(stored)
- return 0
- elif notepad.delete_note(job_id, key):
- print(color(f"Deleted notepad key '{key}' for job {job_id}.", Colors.GREEN))
- return 0
- print(color(f"No notepad key '{key}' for job {job_id}.", Colors.YELLOW))
- return 1
- except ValueError as exc:
- print(color(f"Notepad error: {exc}", Colors.RED))
- return 1
- def cron_cancel(args) -> int:
- """Stop a job's live run. The schedule is untouched — the next fire still happens."""
- result = _cron_api(action="cancel", job_id=args.job_id,
- reason=getattr(args, "reason", None))
- if not result.get("success"):
- print(color(f"Failed to cancel run: {result.get('error', 'unknown error')}", Colors.RED))
- return 1
- job = result.get("job") or {}
- label = f"{job.get('name', args.job_id)} ({args.job_id})"
- if not result.get("cancelled"):
- print(color(f"Nothing to cancel for {label}", Colors.YELLOW))
- return 0
- print(color(f"Cancel requested for {label}", Colors.GREEN))
- print(f" Execution: {result['execution_id']} (pid {result.get('pid')})")
- print(f" {result.get('detail', '')}")
- return 0
- # Late-bound lambdas keep module-level monkeypatching working; list/status/runs return None -> 0.
- _CRON_SUBCOMMANDS = {
- "list": lambda a: cron_list(getattr(a, "all", False)) or 0,
- "status": lambda a: cron_status() or 0,
- "doctor": lambda a: cron_doctor(),
- "tick": lambda a: cron_tick(),
- "runs": lambda a: cron_runs(getattr(a, "job_id", None), getattr(a, "limit", 20)) or 0,
- "incidents": lambda a: cron_incidents(a),
- "notepad": lambda a: cron_notepad(a),
- "create": lambda a: cron_create(a),
- "edit": lambda a: cron_edit(a),
- "pause": lambda a: _job_action("pause", a.job_id, "Paused"),
- "resume": lambda a: cron_resume(a),
- "run": lambda a: _job_action("run", a.job_id, "Triggered"),
- "cancel": lambda a: cron_cancel(a),
- "remove": lambda a: _job_action("remove", a.job_id, "Removed"),
- }
- _CRON_SUBCOMMANDS["history"] = _CRON_SUBCOMMANDS["runs"]
- _CRON_SUBCOMMANDS["add"] = _CRON_SUBCOMMANDS["create"]
- _CRON_SUBCOMMANDS["rm"] = _CRON_SUBCOMMANDS["delete"] = _CRON_SUBCOMMANDS["remove"]
- def cron_command(args):
- """Handle cron subcommands."""
- subcmd = getattr(args, 'cron_command', None)
- handler = _CRON_SUBCOMMANDS.get("list" if subcmd is None else subcmd)
- if handler is not None:
- return handler(args)
- print(f"Unknown cron command: {subcmd}\n"
- "Usage: hermes cron [list|create|edit|pause|resume|run|cancel|remove|status|runs|doctor|tick]")
- sys.exit(1)
|