cronjob_tools.py 65 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271
  1. """Cron job management tool: one compressed action-oriented `cronjob_manage` tool
  2. (schema/context bloat avoided); `cronjob()` stays callable for direct Python callers."""
  3. import contextlib
  4. import json
  5. import logging
  6. import sys
  7. import threading
  8. import time
  9. from concurrent.futures import Future
  10. from pathlib import Path
  11. from typing import Any, Dict, List, Optional, Union
  12. import copy
  13. from hermes_constants import display_hermes_home
  14. logger = logging.getLogger(__name__)
  15. # Heartbeat cadence keeping the caller's inactivity watchdog at bay while a manual
  16. # `cronjob(action="run")` executes in-process (comfortably below HERMES_AGENT_TIMEOUT).
  17. # Mirrors the 10s cadence of tools/environments/base.py::touch_activity_if_due (delegate_task's heartbeat
  18. # uses 30s) — comfortably below the 1800s default HERMES_AGENT_TIMEOUT. See #76502.
  19. _CRON_RUN_HEARTBEAT_INTERVAL = 10.0
  20. # Hard ceiling: with HERMES_CRON_TIMEOUT=0 a truly hung run would otherwise mask the
  21. # gateway watchdog forever; past this the heartbeat stops and the watchdog regains authority.
  22. # The child cron run has its own inactivity watchdog (HERMES_CRON_TIMEOUT, default 600s) that bounds a
  23. # wedged job, but with HERMES_CRON_TIMEOUT=0 (explicit "unlimited") a truly hung run_one_job would otherwise
  24. # mask the gateway watchdog forever — pre-#76502 the parent was at least reaped at ~1800s.
  25. _CRON_RUN_HEARTBEAT_CEILING = 6 * 3600.0
  26. sys.path.insert(0, str(Path(__file__).parent.parent))
  27. from cron.jobs import (
  28. AmbiguousJobReference,
  29. claim_job_for_fire,
  30. get_job,
  31. is_job_runnable,
  32. list_jobs,
  33. mark_job_run,
  34. parse_schedule,
  35. pause_job,
  36. remove_job,
  37. resolve_job_ref,
  38. resume_job,
  39. trigger_job,
  40. update_job)
  41. from tools.cronjob_prompt_scan import _scan_cron_prompt
  42. from tools.cronjob_job_args import (
  43. _apply_continuity,
  44. _canonical_skills,
  45. _clean_str_list,
  46. _format_job,
  47. _gateway_liveness_notice,
  48. _local_delivery_notice,
  49. _mode_guidance_notes,
  50. _normalize_deliver_param,
  51. _normalize_optional_job_value,
  52. _origin_from_env,
  53. _repeat_display,
  54. _resolve_cron_context_deliver,
  55. _split_monitor_arg,
  56. _validate_bot_chat_deliver,
  57. _validate_context_from_refs,
  58. _validate_cron_base_url,
  59. _validate_cron_script_path)
  60. from tools.registry import registry, tool_error
  61. def _dumps(payload: dict[str, Any]) -> str:
  62. return json.dumps(payload, indent=2)
  63. def _notify_provider_jobs_changed_safe() -> None:
  64. """Tell the active scheduler provider the job set changed; best-effort, never raises."""
  65. try:
  66. from cron.scheduler import _notify_provider_jobs_changed
  67. _notify_provider_jobs_changed()
  68. except Exception:
  69. pass
  70. # ---------------------------------------------------------------------------
  71. # Manual run execution (claim -> run_one_job -> report)
  72. # ---------------------------------------------------------------------------
  73. def _relay_fronted_delivery_platforms(job: dict[str, Any]) -> set:
  74. """Delivery-platform names for this job that the relay connector fronts."""
  75. try:
  76. from gateway.relay import relay_fronted_platforms
  77. except Exception:
  78. return set()
  79. fronted = relay_fronted_platforms()
  80. if not fronted:
  81. return set()
  82. try:
  83. from cron.scheduler import _resolve_delivery_targets
  84. targets = _resolve_delivery_targets(job) or []
  85. except Exception:
  86. return set()
  87. return {t.get("platform") for t in targets if t.get("platform")} & fronted
  88. def _api_server_base_url() -> str:
  89. """``http://host:port`` of the local api_server, mirroring its bind resolution
  90. (extra.host -> API_SERVER_HOST -> 127.0.0.1); a wildcard bind listens on loopback too."""
  91. import os
  92. port_raw = os.getenv("API_SERVER_PORT", "").strip()
  93. try:
  94. port = int(port_raw) if port_raw else 8642
  95. except ValueError:
  96. port = 8642
  97. try:
  98. from hermes_cli.config import cfg_get, load_config_readonly
  99. host = str(cfg_get(load_config_readonly(), "platforms", "api_server", "extra", "host", default="") or "").strip()
  100. except Exception:
  101. host = ""
  102. host = host or os.getenv("API_SERVER_HOST", "").strip()
  103. if not host or host in ("0.0.0.0", "::", "*"):
  104. host = "127.0.0.1"
  105. if ":" in host and not host.startswith("["):
  106. host = f"[{host}]" # bare IPv6 literal
  107. return f"http://{host}:{port}"
  108. def _forward_relay_fronted_run(job: dict[str, Any], extra_prompt: Optional[str] = None) -> Optional[str]:
  109. """Forward a manual run to the gateway when it targets a relay-fronted platform: such delivery
  110. has no standalone sender — the gateway's live relay adapter is the only path, reached via
  111. ``POST /api/jobs/{id}/run`` (marks the job due; ``extra_prompt`` rides in the body). Returns a
  112. JSON result string when forwarding engages, else None (normal in-process run)."""
  113. if not _relay_fronted_delivery_platforms(job):
  114. return None
  115. from agent.secret_scope import get_secret
  116. key = get_secret("API_SERVER_KEY", "") or ""
  117. try:
  118. import httpx
  119. resp = httpx.post(
  120. f"{_api_server_base_url()}/api/jobs/{job['id']}/run", headers={"Authorization": f"Bearer {key}"},
  121. json=({"prompt": extra_prompt} if extra_prompt else {}), timeout=10.0)
  122. except Exception:
  123. resp = None
  124. if resp is not None and resp.status_code < 300:
  125. return _dumps({
  126. "success": True,
  127. "forwarded_to_gateway": True,
  128. "note": (
  129. "This job targets a relay-fronted platform; it was dispatched "
  130. "to the running gateway, whose live relay adapter owns that "
  131. "delivery."),
  132. })
  133. return _dumps({
  134. "success": False,
  135. "error": (
  136. "This job targets a relay-fronted platform, which has no "
  137. "standalone sender. Start the gateway — its ticker will "
  138. "deliver the job on schedule via the live relay adapter."),
  139. })
  140. def _primary_routed_delivery_platforms(job: dict[str, Any]) -> set:
  141. """Delivery-platform names this satellite profile reaches only through the primary gateway's
  142. ``profile_routes``: routed here, with no credential of its own to send standalone."""
  143. try:
  144. from cron.scheduler import _resolve_delivery_targets
  145. from cron.scheduler_preflight import _delivery_platform_routed_from_primary_gateway
  146. routed = {t["platform"] for t in _resolve_delivery_targets(job) or []
  147. if t.get("platform") and _delivery_platform_routed_from_primary_gateway(t["platform"])}
  148. if not routed:
  149. return set()
  150. from gateway.config import load_gateway_config
  151. return routed - {p.value for p in load_gateway_config().get_connected_platforms()}
  152. except Exception:
  153. return set()
  154. def _hand_off_primary_routed_run(job: dict[str, Any], extra_prompt: Optional[str] = None) -> Optional[str]:
  155. """Queue a manual run for the gateway ticker when the job delivers through the primary gateway's
  156. profile route: only the gateway process holding the primary's bot can send it, so an in-process
  157. run would spend the whole turn and then record ``delivery_failed`` (#120330). Returns a JSON
  158. result string when the hand-off engages, else None (normal in-process run)."""
  159. runner_ref = getattr(sys.modules.get("gateway.run"), "_gateway_runner_ref", None)
  160. if callable(runner_ref) and runner_ref() is not None:
  161. return None # inside the gateway: its live adapter delivers (#89302)
  162. if not is_job_runnable(job):
  163. return None # keep the normal paused refusal; trigger_job would resume the job
  164. routed = _primary_routed_delivery_platforms(job)
  165. if not routed:
  166. return None
  167. from hermes_cli.cron import _builtin_gateway_liveness
  168. alive = _builtin_gateway_liveness()
  169. if alive is not True:
  170. # None = the probe could not tell; a run queued for a ticker that may not exist is worse
  171. # than an honest refusal (nothing would ever pick it up).
  172. cause = ("Start the gateway — its ticker will deliver the job on schedule." if alive is False
  173. else "Could not determine whether a gateway serves this profile; check "
  174. "`hermes cron status` and re-run.")
  175. return _dumps({
  176. "success": False,
  177. "error": (
  178. f"This job delivers to {', '.join(sorted(routed))} through the primary "
  179. f"gateway's profile route, which has no standalone sender. {cause}"),
  180. })
  181. updated = trigger_job(job["id"], extra_prompt=extra_prompt)
  182. _notify_provider_jobs_changed_safe()
  183. return _dumps({
  184. "success": True,
  185. "job": _format_job(updated),
  186. "note": (
  187. "This job delivers through the primary gateway's profile route; it was "
  188. "queued for that gateway's next scheduler tick, which runs and delivers it."),
  189. })
  190. def _manual_run_delivery_note(deliver: str, refreshed: dict[str, Any]) -> str:
  191. """Parenthetical delivery note for a manual run's summary; follows the refreshed record's
  192. ``last_delivery_error`` so the summary never claims success over a failed delivery.
  193. Follows the refreshed job record (#83993): ``run_one_job`` writes ``last_delivery_error`` via
  194. ``mark_job_run`` when the post-run delivery (telegram/discord/…) failed, and the summary must not claim
  195. success over that record — the calling agent relays this line to the user. Local jobs never deliver; an
  196. empty/missing error keeps the legacy wording byte-for-byte.
  197. """
  198. # Falsy deliver ("", stored JSON null) is normalized to "local" at fire time -> saved
  199. # locally. Whitespace-only values fall through so the fire-time "no target" error surfaces.
  200. if not deliver or deliver == "local":
  201. return " (output saved locally only)"
  202. err = str(refreshed.get("last_delivery_error") or "").strip()
  203. if not err:
  204. if refreshed.get("last_delivery_queued"):
  205. return " (output queued for Bot Chat; completion unverified, do not resend)"
  206. return " (output was delivered there by the job itself)"
  207. return f" (⚠ delivery FAILED: {err[:200]})"
  208. _ALREADY_RUNNING_ERROR = (
  209. "Job is already running (a scheduler tick or another "
  210. "manual run is executing it); not started again.")
  211. def _claim_for_manual_run(job_id: str, log_label: str):
  212. """At-most-once claim shared by the sync and background run paths: ``(claimed_job, None)`` or
  213. ``(None, error_dict)`` in the ``_execute_job_now`` shape. A lost claim is labelled precisely —
  214. claim_job_for_fire also returns False for paused/disabled/missing jobs, not just in-flight ones."""
  215. try:
  216. claimed_job = claim_job_for_fire(job_id, manual=True, return_job=True)
  217. if isinstance(claimed_job, dict):
  218. return claimed_job, None
  219. refreshed = get_job(job_id)
  220. if refreshed is None:
  221. reason = "Job no longer exists; nothing to run."
  222. elif not is_job_runnable(refreshed):
  223. reason = "Job is paused/disabled; resume it before running."
  224. else:
  225. reason = "Job is already being fired by the scheduler; not run again."
  226. return None, {"claimed": False, "success": False, "error": reason}
  227. except Exception as e:
  228. logger.error("Failed to claim cron job %s for %s: %s", job_id, log_label, e)
  229. with contextlib.suppress(Exception):
  230. mark_job_run(job_id, False, str(e))
  231. return None, {"claimed": True, "success": False, "error": str(e)}
  232. def _execute_job_now(job: dict[str, Any], extra_prompt: Optional[str] = None) -> dict[str, Any]:
  233. """Run a job now, outside the scheduler tick: claim via ``claim_job_for_fire`` (the ticker's
  234. CAS, so a concurrent tick cannot double-fire and next_run_at advances), then fire through
  235. the shared ``run_one_job`` body. Returns {"claimed", "success", "error"}."""
  236. claimed_job, err = _claim_for_manual_run(job["id"], "immediate run")
  237. return err if err is not None else _run_claimed_job(claimed_job, extra_prompt=extra_prompt)
  238. @contextlib.contextmanager
  239. def _run_heartbeat(job_name: str):
  240. """Heartbeat into the caller's activity tracker while a manual run executes (minutes,
  241. synchronously on the caller's thread — without tool activity the gateway inactivity
  242. watchdog would kill the parent turn). Best-effort: no callback -> no thread."""
  243. stop = threading.Event()
  244. thread = None
  245. try:
  246. # run_one_job records last_run_at/last_status via mark_job_run (which also clears the fire claim)
  247. # and returns True iff it processed the job. ``job`` here is the exact claimed snapshot
  248. # (owner-bearing), so the shared body fences every terminal write by that owner. A manual `run`
  249. # executes the job synchronously on the caller's thread, and a cron job is itself a full agent run
  250. # that routinely takes minutes. The calling turn emits no tool activity for that entire window, so
  251. # the gateway inactivity watchdog concludes the agent is hung and kills the parent turn (#76502).
  252. # Fire a heartbeat into the caller's activity tracker (the same signal tool progress uses) while the
  253. # job runs, so the watchdog sees a working tool instead of a silent one — mirrors the delegate_task
  254. # heartbeat pattern. Best-effort: if no activity callback is registered (direct Python callers,
  255. # tests), behavior is unchanged.
  256. from tools.environments.base import get_activity_callback
  257. # Capture on THIS thread: the callback is thread-local (installed by the tool
  258. # executor), so a freshly spawned thread cannot read it.
  259. activity_cb = get_activity_callback()
  260. except Exception:
  261. activity_cb = None
  262. def _heartbeat_loop() -> None:
  263. started = time.monotonic()
  264. while not stop.wait(_CRON_RUN_HEARTBEAT_INTERVAL):
  265. elapsed = time.monotonic() - started
  266. if elapsed > _CRON_RUN_HEARTBEAT_CEILING:
  267. # A run this long with an unlimited child watchdog is likely wedged.
  268. logger.warning(
  269. "cronjob run heartbeat ceiling reached for job "
  270. "'%s' (%.0fs) — stopping heartbeat; gateway watchdog regains authority",
  271. job_name, elapsed)
  272. return
  273. try:
  274. activity_cb(f"cronjob: running job '{job_name}' ({int(elapsed)}s elapsed)")
  275. except Exception:
  276. continue # one transient callback error must not drop protection
  277. if activity_cb is not None:
  278. thread = threading.Thread(target=_heartbeat_loop, daemon=True, name="cronjob-run-heartbeat")
  279. thread.start()
  280. try:
  281. yield
  282. finally:
  283. stop.set()
  284. if thread is not None:
  285. thread.join(timeout=_CRON_RUN_HEARTBEAT_INTERVAL + 1)
  286. def _run_claimed_job(job: dict[str, Any], extra_prompt: Optional[str] = None) -> dict[str, Any]:
  287. """Fire an already-claimed job through the shared ``run_one_job`` body (split from
  288. ``_execute_job_now`` so the background path can claim synchronously and hand the run
  289. to a worker). Returns {"claimed": True, "success": bool, "error": ...}."""
  290. job_id = job["id"]
  291. _registered = False
  292. fire_owner = None
  293. registration_owner = object()
  294. running_future = Future()
  295. running_future.set_running_or_notify_cancel()
  296. try:
  297. from cron.scheduler import release_running_job, run_one_job, try_register_running_job
  298. # In-flight dedupe: the fire claim's TTL is routinely outlived by real jobs, so
  299. # register in the scheduler's shared running set (same guard the ticker uses;
  300. # also visible to the gateway shutdown drain).
  301. # In-flight dedupe (idea from #53395 by @izumi0uu): the fire claim's TTL (300s) is routinely
  302. # outlived by real jobs, so it alone cannot stop a manual run from double-firing a job the ticker
  303. # (or another manual run) is still executing.
  304. if not try_register_running_job(
  305. job_id, owner=registration_owner, future=running_future,
  306. ):
  307. return {"claimed": True, "success": False, "error": _ALREADY_RUNNING_ERROR}
  308. _registered = True
  309. claim = job.get("fire_claim")
  310. fire_owner = str(claim.get("by") or "") if isinstance(claim, dict) else None
  311. # Inside the gateway process deliver on the loop that owns clients such as
  312. # Matrix/aiohttp (a standalone asyncio.run() loop breaks them).
  313. runner_ref = getattr(sys.modules.get("gateway.run"), "_gateway_runner_ref", None)
  314. # Manual runs invoked from a gateway agent execute outside the scheduler ticker, but they still
  315. # share the process with the live platform adapters. Calling those clients from run_one_job's
  316. # standalone asyncio.run() loop raises errors like "Timeout context manager should be used inside a
  317. # task" and can break encrypted Matrix delivery (#61495 — salvaged from #63586 by @Fly-onlyone).
  318. runner = runner_ref() if callable(runner_ref) else None
  319. adapters = getattr(runner, "adapters", None) if runner is not None else None
  320. # ``runner.adapters`` is the LAUNCH profile's map; under multiplex the run executes with
  321. # HERMES_HOME bound to the owning profile, so resolve that profile's adapters the way the
  322. # ticker (``tick_adapters_for``) does — fail closed, never the default bot (#124248). A
  323. # resolution error propagates to the ``except`` below and marks the run failed.
  324. if runner is not None and hasattr(runner, "_adapters_for_profile"):
  325. from hermes_constants import get_hermes_home, profile_name_for_home
  326. profile = profile_name_for_home(get_hermes_home())
  327. adapters = runner._adapters_for_profile(profile)
  328. # A credentialless shared-bot satellite borrows the primary's bot for ROUTED targets
  329. # only — the same grant the ticker's ``tick_adapters_for`` makes, never the full map.
  330. if getattr(runner, "_is_shared_bot_satellite", lambda _p: False)(profile):
  331. from cron.scheduler_preflight import (
  332. SharedRouteAdapters, _primary_profile_routes_for_current_home)
  333. adapters = SharedRouteAdapters(adapters, _primary_profile_routes_for_current_home())
  334. gateway_loop = getattr(runner, "_gateway_loop", None) if runner is not None else None
  335. try:
  336. # run_one_job records last_run_at/last_status via mark_job_run; `job` is the
  337. # owner-bearing claimed snapshot, so terminal writes stay fenced by that owner.
  338. with _run_heartbeat(str(job.get("name") or job_id)):
  339. processed = run_one_job(job, adapters=adapters, loop=gateway_loop, extra_prompt=extra_prompt)
  340. finally:
  341. _registered = False
  342. release_running_job(job_id, owner=registration_owner)
  343. refreshed = get_job(job_id) or {}
  344. execution = None
  345. execution_id = job.get("execution_id")
  346. if execution_id:
  347. from cron.executions import get_execution
  348. execution = get_execution(str(execution_id))
  349. last_status = refreshed.get("last_status")
  350. # "delivery_failed": the run succeeded but output never reached the user — not a
  351. # success for the caller; surface last_delivery_error.
  352. run_error = refreshed.get("last_error")
  353. if last_status == "delivery_failed" and not run_error:
  354. run_error = refreshed.get("last_delivery_error")
  355. # That is NOT a success for the caller — the calling agent relays this result — so report it as
  356. # failed and surface the delivery error, which lives in last_delivery_error (last_error is None for
  357. # these runs, and a bare success=False with error=None reads as an unexplained failure). See #83993.
  358. ok = last_status in {"ok", "delivery_queued"}
  359. if execution is not None and execution.get("status") != "completed":
  360. ok = False
  361. run_error = execution.get("error") or f"execution ended in {execution.get('status') or 'unknown'} state"
  362. return {"claimed": True, "success": bool(processed and ok), "error": run_error}
  363. except Exception as e:
  364. logger.error("Failed to execute cron job %s immediately: %s", job_id, e)
  365. with contextlib.suppress(Exception):
  366. mark_job_run(job_id, False, str(e), expected_fire_owner=fire_owner)
  367. return {"claimed": True, "success": False, "error": str(e)}
  368. finally:
  369. # Setup failures and BaseException must release only our registration.
  370. if _registered:
  371. release_running_job(job_id, owner=registration_owner)
  372. running_future.set_result(None)
  373. def execute_job_for_event(
  374. job_ref: str, extra_prompt: Optional[str] = None
  375. ) -> dict[str, Any]:
  376. """Fire an existing cron job in response to an external event.
  377. Public entry point for event-driven triggers (the webhook adapter's
  378. ``cron_job`` routes). Resolves ``job_ref`` (ID or name) and
  379. fires it through the exact same claimed-run body a manual
  380. ``cronjob(action='run')`` uses, so at-most-once claiming, in-flight
  381. dedupe, delivery, and ``[SILENT]`` handling stay identical across the
  382. scheduler / manual / event paths.
  383. ``extra_prompt`` is injected as transient per-run context (the job's
  384. stored prompt is never mutated), exactly like ``action='run'`` with a
  385. ``prompt`` argument.
  386. Returns the ``_execute_job_now`` result shape:
  387. ``{"claimed": bool, "success": bool, "error": str|None}``.
  388. """
  389. try:
  390. job = resolve_job_ref(job_ref)
  391. except AmbiguousJobReference as e:
  392. return {"claimed": False, "success": False, "error": str(e)}
  393. if job is None:
  394. return {
  395. "claimed": False,
  396. "success": False,
  397. "error": f"Cron job '{job_ref}' not found.",
  398. }
  399. return _execute_job_now(job, extra_prompt=extra_prompt)
  400. def _latest_job_output_excerpt(job_id: str, max_chars: int = 2000) -> Optional[str]:
  401. """Excerpt of the job's most recent saved output file for the background completion
  402. block (parent sees what the job produced). Never raises."""
  403. try:
  404. from cron.jobs import get_cron_output_dir
  405. out_dir = get_cron_output_dir() / job_id
  406. files = sorted(out_dir.glob("*.md"))
  407. if not files:
  408. return None
  409. text = files[-1].read_text(encoding="utf-8-sig", errors="replace").strip()
  410. if not text:
  411. return None
  412. if len(text) > max_chars:
  413. text = text[:max_chars] + f"\n… (truncated; full output: {files[-1]})"
  414. return text
  415. except Exception:
  416. return None
  417. def _reap_stale_executions(job_name: str) -> None:
  418. """Reap execution rows left 'claimed'/'running' by a provably-dead owner (e.g. a prior
  419. one-shot `hermes cron run` that died mid-run). The ticker does this at startup; one-shot
  420. invocations have no such moment, so a stale claim would block every later manual run.
  421. Best-effort self-heal: must not block dispatch."""
  422. try:
  423. # Reap any execution row this job (or any job) left stranded 'claimed'/ 'running' by a dead owner
  424. # process -- e.g. a PRIOR one-shot `hermes cron run` invocation whose dispatched runner died with
  425. # the exiting process before writing a terminal status (issue #86721). Safe and cheap: provably-dead
  426. # owners (PID gone, or PID reused by a different process per its start time) are reaped, as is a
  427. # live owner whose claim is older than the derived stale bound (the process itself is not killed).
  428. from cron.executions import recover_interrupted_executions
  429. _reclaimed = recover_interrupted_executions()
  430. if _reclaimed:
  431. logger.warning(
  432. "Reclaimed %d stale cron execution(s) from dead owner(s) before dispatching job '%s'",
  433. _reclaimed, job_name)
  434. except Exception as _reap_exc:
  435. logger.debug("Stale execution reclaim failed: %s", _reap_exc)
  436. def _background_session_key(session_id: Optional[str]) -> str:
  437. """Routing key for a detached completion, captured on THIS thread (contextvars don't
  438. cross the pool). Empty string = no durable consumer."""
  439. try:
  440. from tools.approval_context import get_current_session_key
  441. session_key = get_current_session_key(default="")
  442. except Exception:
  443. session_key = ""
  444. # CLI path: the approval contextvar is only bound during gateway/TUI turns; the CLI
  445. # drain filters completions by the durable session id, and an empty key would fail
  446. # closed (completion never claimable).
  447. return session_key or (str(session_id) if session_id else "")
  448. def _run_session_usage(job_id: str, execution_id: Optional[str]) -> dict[str, Any]:
  449. """``{"model", "api_calls"}`` the fire's run session recorded in state.db (summed over its
  450. compression chain), or ``{}`` when no run session exists (no_agent job, refused fire)."""
  451. if not execution_id:
  452. return {}
  453. import sqlite3
  454. from hermes_constants import get_hermes_home
  455. from hermes_state import SessionDB
  456. # A missing or locked state.db costs the notice its usage line, never the completion.
  457. try:
  458. db = SessionDB(db_path=get_hermes_home() / "state.db", read_only=True)
  459. except (sqlite3.Error, OSError):
  460. return {}
  461. try:
  462. run = next((r for r in db.list_cron_job_runs(job_id, limit=50)
  463. if r.get("cron_execution_id") == execution_id), None)
  464. if run is None:
  465. return {}
  466. rows = [db.get_session(sid) or {} for sid in db.get_compression_chain(run["id"])]
  467. except sqlite3.Error as exc:
  468. logger.debug("cronjob run: run-session usage lookup failed for %s: %s", job_id, exc)
  469. return {}
  470. finally:
  471. db.close()
  472. model = next((r["model"] for r in reversed(rows) if r.get("model")), None)
  473. return {"api_calls": sum(int(r.get("api_call_count") or 0) for r in rows), **({"model": model} if model else {})}
  474. def _manual_run_completion(
  475. res: dict[str, Any], job_id: str, job_name: str, deliver: str, started_at: float,
  476. execution_id: Optional[str] = None) -> dict[str, Any]:
  477. """Async-delegation completion block for a finished background manual run. Model and API
  478. calls come from the run's own session; without one they are omitted, never guessed."""
  479. duration = round(time.time() - started_at, 2)
  480. refreshed = get_job(job_id) or {}
  481. lines = [
  482. f"Cron job '{job_name}' ({job_id}) finished its manual run.",
  483. f"Result: {'ok' if res.get('success') else 'FAILED'}"
  484. + (f" — {res.get('error')}" if res.get("error") else ""),
  485. f"Delivery target: {deliver}" + _manual_run_delivery_note(deliver, refreshed),
  486. ]
  487. if refreshed.get("next_run_at"):
  488. lines.append(f"Next scheduled run: {refreshed['next_run_at']}")
  489. excerpt = _latest_job_output_excerpt(job_id)
  490. if excerpt:
  491. lines += ["--- JOB OUTPUT ---", excerpt]
  492. return {
  493. "status": "completed" if res.get("success") else "error", "summary": "\n".join(lines),
  494. "error": res.get("error"), "duration_seconds": duration, **_run_session_usage(job_id, execution_id),
  495. }
  496. def _try_dispatch_background_run(
  497. job: dict[str, Any], session_id: Optional[str] = None, extra_prompt: Optional[str] = None,
  498. ) -> Optional[dict[str, Any]]:
  499. """Claim ``job`` now (SYNCHRONOUSLY, so unrunnable jobs report immediately), then fire it
  500. on the async-delegation executor like ``delegate_task``'s background mode: the tool returns
  501. a handle and a ``type="async_delegation"`` completion re-enters as a fresh turn (role
  502. alternation legal, prompt cache intact) instead of blocking the parent turn for minutes.
  503. Returns None when background delivery is unavailable (caller runs sync); ``{"claimed":
  504. False}`` on a lost claim; ``{"claimed": True, "dispatched": True, "delegation_id"}``; or
  505. ``{"claimed": True, "dispatched": False, ...}`` when the pool was full and it ran inline."""
  506. job_id = job["id"]
  507. job_name = str(job.get("name") or job_id)
  508. # Reap BEFORE the async/sync branch: the one-shot `hermes cron run` path returns early
  509. # below, and this is the only moment it heals a stale claim left by a killed prior run (#113923).
  510. _reap_stale_executions(job_name)
  511. # Finite sessions cannot route a detached result back after the turn ends (delegate_task's gate).
  512. try:
  513. from gateway.session_context import async_delivery_supported
  514. if not async_delivery_supported():
  515. return None
  516. except Exception:
  517. pass
  518. # Routing capture BEFORE the claim: no routable session = no durable consumer for a detached
  519. # completion, so don't claim-and-dispatch (direct callers like `hermes cron run` exit right after).
  520. session_key = _background_session_key(session_id)
  521. # CLI path: the approval contextvar is only bound during gateway/TUI turns. The CLI drain filters
  522. # completions by the durable agent session id (#64240), so stamp it as the key — an empty key would fail
  523. # closed and the completion could never be claimed.
  524. if not session_key:
  525. return None
  526. # Early dedupe so a mid-run job reports in THIS response, not as a delayed error completion
  527. # (authoritative check: try_register_running_job). Home-scoped: one process ticks every
  528. # profile, so the bare-id union would report another profile's same-named job as running.
  529. try:
  530. from cron.scheduler import is_job_running
  531. if is_job_running(job_id):
  532. return {"claimed": False, "success": False, "error": _ALREADY_RUNNING_ERROR}
  533. except Exception:
  534. pass
  535. claimed_job, err = _claim_for_manual_run(job_id, "background run")
  536. if err is not None:
  537. if err["claimed"]:
  538. err["dispatched"] = False
  539. return err
  540. origin_ui_session_id = ""
  541. try:
  542. from gateway.session_context import get_session_env
  543. origin_ui_session_id = get_session_env("HERMES_UI_SESSION_ID", "") or ""
  544. except Exception:
  545. pass
  546. try:
  547. from tools.async_delegation import _current_origin_session_id, dispatch_async_delegation
  548. origin_session_id = _current_origin_session_id()
  549. except Exception as e:
  550. logger.warning(
  551. "cronjob run: async delegation registry unavailable (%s); running job '%s' inline.", e, job_name)
  552. result = _run_claimed_job(claimed_job, extra_prompt=extra_prompt)
  553. result["dispatched"] = False
  554. return result
  555. try:
  556. from tools.delegate_tool import _get_max_async_children
  557. max_async = _get_max_async_children()
  558. except Exception:
  559. max_async = 3
  560. started_at = time.time()
  561. # Scheduler's own normalizer (falsy -> "local", list -> comma string) on the claimed snapshot.
  562. from cron.scheduler import _normalize_deliver_value
  563. deliver = _normalize_deliver_value(claimed_job.get("deliver", "local"))
  564. def _runner() -> dict[str, Any]:
  565. res = _run_claimed_job(claimed_job, extra_prompt=extra_prompt)
  566. # run_one_job stamps the fire's ledger id onto this claimed snapshot.
  567. return _manual_run_completion(res, job_id, job_name, deliver, started_at,
  568. (claimed_job or {}).get("execution_id"))
  569. dispatch = dispatch_async_delegation(
  570. goal=f"Manual run of cron job '{job_name}' ({job_id})",
  571. context=("Triggered via cronjob(action='run'). The job executed in its own "
  572. "fresh cron session; this block reports its outcome."),
  573. # model=None: the run session (via _runner) is the only source; a pin a refused fire never ran is not.
  574. toolsets=None, role="cron_run", model=None, session_key=session_key,
  575. parent_session_id=str(session_id) if session_id else None, runner=_runner,
  576. origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id,
  577. max_async_children=max_async)
  578. if dispatch.get("status") == "dispatched":
  579. return {"claimed": True, "dispatched": True, "delegation_id": dispatch.get("delegation_id")}
  580. # Pool at capacity (or submit failure): the claim is already taken and must not be stranded.
  581. logger.info(
  582. "cronjob run: background pool unavailable (%s); running job '%s' inline.",
  583. dispatch.get("error", "rejected"), job_name)
  584. result = _run_claimed_job(job, extra_prompt=extra_prompt)
  585. result["dispatched"] = False
  586. return result
  587. # ---------------------------------------------------------------------------
  588. # Tool actions. Each takes the cronjob() argument dict `a` (and the resolved
  589. # job record for job-bound actions) and returns the JSON result string.
  590. # ---------------------------------------------------------------------------
  591. def _with_guidance(result: dict[str, Any], job: dict[str, Any], deliver: Optional[str]) -> dict[str, Any]:
  592. """Attach mode/delivery guidance (create and update echo the same notes)."""
  593. _notes = _mode_guidance_notes(job, deliver)
  594. if _notes:
  595. result["guidance"] = _notes
  596. return result
  597. def _action_create(a: dict[str, Any]) -> str:
  598. prompt, script = a["prompt"], a["script"]
  599. deliver = _normalize_deliver_param(a["deliver"])
  600. if not a["schedule"]:
  601. return tool_error("schedule is required for create", success=False)
  602. canonical_skills = _canonical_skills(a["skill"], a["skills"])
  603. _no_agent = bool(a["no_agent"])
  604. # no_agent=True -> the script IS the job (prompt/skills optional); else prompt or skills.
  605. if _no_agent:
  606. if not script:
  607. return tool_error(
  608. "create with no_agent=True requires a script — "
  609. "the script is the job. In no_agent mode the LLM is "
  610. "skipped entirely: prompt and skills are ignored, "
  611. "non-empty stdout is delivered verbatim, empty stdout "
  612. "sends nothing (watchdog pattern), and a non-zero exit or timeout sends an error alert.",
  613. success=False)
  614. elif not prompt and not canonical_skills:
  615. return tool_error("create requires either prompt or at least one skill", success=False)
  616. error = (
  617. (prompt and _scan_cron_prompt(prompt))
  618. or (script and _validate_cron_script_path(script))
  619. or (a["monitor_script"] and _validate_cron_script_path(a["monitor_script"]))
  620. # A model-supplied base_url must not route a named provider's stored credential
  621. # to an attacker endpoint.
  622. or _validate_cron_base_url(a["provider"], a["base_url"])
  623. # bot-chat targets are machine-local: fail the CREATE, not the run.
  624. or _validate_bot_chat_deliver(deliver)
  625. # failure_deliver shares deliver's grammar and validators.
  626. or _validate_bot_chat_deliver(_normalize_deliver_param(a["failure_deliver"]))
  627. or (a["context_from"] and _validate_context_from_refs(
  628. [a["context_from"]] if isinstance(a["context_from"], str) else a["context_from"])))
  629. if error:
  630. return tool_error(error, success=False)
  631. context_from = a["context_from"]
  632. if a["continuity"] is not None:
  633. context_from = _apply_continuity(context_from, a["continuity"])
  634. from cron.scheduler import CronSchedulerRegistrationError, create_job_with_scheduler_registration
  635. try:
  636. job = create_job_with_scheduler_registration(
  637. prompt=prompt or "", schedule=a["schedule"], name=a["name"], repeat=a["repeat"],
  638. deliver=_resolve_cron_context_deliver(deliver),
  639. origin=_origin_from_env(a["schedule"]),
  640. skills=canonical_skills,
  641. model=_normalize_optional_job_value(a["model"]), provider=_normalize_optional_job_value(a["provider"]),
  642. base_url=_normalize_optional_job_value(a["base_url"], strip_trailing_slash=True),
  643. script=_normalize_optional_job_value(script), context_from=context_from,
  644. # [] is an explicit zero-tool allowlist, not a clear back to the unrestricted default (#82010).
  645. enabled_toolsets=a["enabled_toolsets"] if a["enabled_toolsets"] is not None else None,
  646. workdir=_normalize_optional_job_value(a["workdir"]),
  647. no_agent=_no_agent, attach_to_session=a["attach_to_session"],
  648. monitor_script=_normalize_optional_job_value(a["monitor_script"]),
  649. monitor_url=_normalize_optional_job_value(a["monitor_url"]),
  650. # CLI-only lane: absent from CRONJOB_SCHEMA and the model dispatch (models don't pick models).
  651. reasoning_effort=a["reasoning_effort"], interpreter=a["interpreter"],
  652. timezone=_normalize_optional_job_value(a["timezone"]),
  653. grace_minutes=a["grace_minutes"],
  654. pinned=bool(a["pinned"]),
  655. failure_deliver=_resolve_cron_context_deliver(_normalize_deliver_param(a["failure_deliver"])),
  656. catch_up=a["catch_up"],
  657. **({"paused": a["paused"], "paused_reason": a["paused_reason"]}
  658. if a["paused"] is not False or a["paused_reason"] is not None else {}))
  659. except CronSchedulerRegistrationError as exc:
  660. _partial = exc.to_dict()
  661. return tool_error(_partial.pop("error"), success=False, **_partial)
  662. _create_message = " ".join(filter(None, (f"Cron job '{job['name']}' created.",
  663. "Created PAUSED — resume to schedule, or explicitly run now." if not job.get("enabled", True) else None,
  664. _local_delivery_notice(job, deliver))))
  665. # The builtin ticker lives in the gateway process: with no gateway running the job is stored
  666. # but never fires — tell the model (the CLI already warns).
  667. _result = {
  668. "success": True, "job_id": job["id"], "name": job["name"], "skill": job.get("skill"),
  669. "skills": job.get("skills", []), "schedule": job["schedule_display"], "repeat": _repeat_display(job),
  670. "deliver": job.get("deliver", "local"), "next_run_at": job["next_run_at"], "job": _format_job(job),
  671. "message": _create_message, **_gateway_liveness_notice(),
  672. }
  673. return _dumps(_with_guidance(_result, job, deliver))
  674. def _action_list(a: dict[str, Any]) -> str:
  675. jobs = [_format_job(job) for job in list_jobs(include_disabled=a["include_disabled"])]
  676. _result = {"success": True, "count": len(jobs), "jobs": jobs}
  677. # Same inert-job class as create; an empty list has nothing inert.
  678. if jobs:
  679. # Same silent-inert-job class as create (#87033): an agent inspecting existing jobs in a
  680. # gateway-less environment must learn they are not firing, not just see a clean list.
  681. _result.update(_gateway_liveness_notice(plural=True))
  682. return _dumps(_result)
  683. def _action_remove(job: dict[str, Any], a: dict[str, Any]) -> str:
  684. job_id = job["id"]
  685. if not remove_job(job_id):
  686. return tool_error(f"Failed to remove job '{job_id}'", success=False)
  687. _notify_provider_jobs_changed_safe()
  688. return _dumps({
  689. "success": True,
  690. "message": f"Cron job '{job['name']}' removed.",
  691. "removed_job": {"id": job_id, "name": job["name"], "schedule": job.get("schedule_display")},
  692. })
  693. def _job_state_result(updated: dict[str, Any]) -> str:
  694. _notify_provider_jobs_changed_safe()
  695. return _dumps({"success": True, "job": _format_job(updated)})
  696. def _refreshed_job_view(job_id: str) -> dict[str, Any]:
  697. """Re-read so the response reflects the post-run last_run_at/last_status."""
  698. return _format_job(get_job(job_id) or {"id": job_id})
  699. def _action_run(job: dict[str, Any], a: dict[str, Any]) -> str:
  700. job_id = job["id"]
  701. # `prompt` on run is transient per-fire context appended to the stored prompt, never
  702. # persisted; same strict scan as stored prompts.
  703. extra_prompt = a["prompt"] or None
  704. # See #57331, #57342, #57360.
  705. if extra_prompt:
  706. scan_error = _scan_cron_prompt(extra_prompt)
  707. if scan_error:
  708. return tool_error(scan_error, success=False)
  709. # Primary-routed satellite delivery has no sender outside the gateway: hand the run to its ticker.
  710. handed_off = _hand_off_primary_routed_run(job, extra_prompt=extra_prompt)
  711. if handed_off is not None:
  712. return handed_off
  713. # A manual run must actually run even with no ticker active. Preferred: background
  714. # dispatch (handle now, outcome as a completion event); inline fallback otherwise.
  715. bg = _try_dispatch_background_run(job, session_id=a["session_id"], extra_prompt=extra_prompt)
  716. if bg is not None and bg.get("dispatched"):
  717. _notify_provider_jobs_changed_safe()
  718. result = _refreshed_job_view(job_id)
  719. result["executed"] = True
  720. result["execution_mode"] = "background"
  721. result["delegation_id"] = bg.get("delegation_id")
  722. return _dumps({
  723. "success": True,
  724. "job": result,
  725. "note": (
  726. "The job is running in the background. You and the "
  727. "user can keep working; its outcome re-enters the "
  728. "conversation as a new message when it finishes. "
  729. "Do not wait or poll — just continue."),
  730. })
  731. if bg is not None:
  732. exec_result = bg # terminal result: claim lost or inline fallback
  733. else:
  734. # Relay-fronted manual run: no live adapter here — forward to the running gateway.
  735. forwarded = _forward_relay_fronted_run(job, extra_prompt=extra_prompt)
  736. if forwarded is not None:
  737. return forwarded
  738. exec_result = _execute_job_now(job, extra_prompt=extra_prompt)
  739. # A claimed direct run advances next_run_at and may race an external provider's
  740. # one-shot for the same occurrence; a lost consumed fire cannot re-arm itself, so
  741. # reconcile after the run has persisted its final state.
  742. claimed = exec_result.get("claimed", False)
  743. if claimed:
  744. _notify_provider_jobs_changed_safe()
  745. result = _refreshed_job_view(job_id)
  746. result["executed"] = claimed
  747. result["execution_success"] = exec_result.get("success", False)
  748. if not claimed:
  749. result["execution_skipped"] = exec_result.get("error") or (
  750. "Already being fired by the scheduler; not run again.")
  751. elif exec_result.get("error"):
  752. result["execution_error"] = exec_result["error"]
  753. return _dumps({"success": True, "job": result})
  754. def _pick(updates: dict[str, Any], job: dict[str, Any], key: str) -> Any:
  755. """Effective value of ``key`` after this update: pending update wins over the stored job."""
  756. return updates[key] if key in updates else job.get(key)
  757. def _update_core_fields(job: dict[str, Any], a: dict[str, Any], updates: dict[str, Any]) -> Optional[str]:
  758. """prompt / name / deliver / skills / model pins; returns an error string or None."""
  759. prompt, deliver, skill, skills = a["prompt"], a["deliver"], a["skill"], a["skills"]
  760. if prompt is not None:
  761. scan_error = _scan_cron_prompt(prompt)
  762. if scan_error:
  763. return scan_error
  764. updates["prompt"] = prompt
  765. if a["name"] is not None and a["name"].strip():
  766. # Blank name is a no-op, not a clear: a model re-sending the whole schema with
  767. # type-default empties must not wipe untouched fields.
  768. updates["name"] = a["name"]
  769. if deliver is not None:
  770. bot_chat_error = _validate_bot_chat_deliver(_normalize_deliver_param(deliver))
  771. if bot_chat_error:
  772. return bot_chat_error
  773. updates["deliver"] = _resolve_cron_context_deliver(_normalize_deliver_param(deliver))
  774. if a["failure_deliver"] is not None:
  775. # '' clears the override (failures fall back to deliver); non-empty values share
  776. # deliver's validation AND its cron-context origin resolution (a job created from
  777. # inside a cron run must never store literal 'origin').
  778. _norm_fd = _normalize_deliver_param(a["failure_deliver"])
  779. if _norm_fd:
  780. bot_chat_error = _validate_bot_chat_deliver(_norm_fd)
  781. if bot_chat_error:
  782. return bot_chat_error
  783. _norm_fd = _resolve_cron_context_deliver(_norm_fd)
  784. updates["failure_deliver"] = _norm_fd
  785. if a["catch_up"] is not None:
  786. updates["catch_up"] = a["catch_up"]
  787. if skills is not None or skill is not None:
  788. canonical_skills = _canonical_skills(skill, skills)
  789. updates["skills"] = canonical_skills
  790. updates["skill"] = canonical_skills[0] if canonical_skills else None
  791. if a["model"] is not None:
  792. updates["model"] = _normalize_optional_job_value(a["model"])
  793. if a["provider"] is not None:
  794. updates["provider"] = _normalize_optional_job_value(a["provider"])
  795. if a["pinned"] is not None:
  796. updates["pinned"] = bool(a["pinned"])
  797. if a["base_url"] is not None:
  798. updates["base_url"] = _normalize_optional_job_value(a["base_url"], strip_trailing_slash=True)
  799. if a["reasoning_effort"] is not None:
  800. # CLI-only lane; update_job validates, empty string clears the pin.
  801. updates["reasoning_effort"] = a["reasoning_effort"]
  802. if a["interpreter"] is not None:
  803. # CLI-only lane like reasoning_effort; update_job trims, empty string clears.
  804. updates["interpreter"] = a["interpreter"]
  805. # Re-validate the EFFECTIVE provider/base_url on EVERY update: a job persisted before
  806. # this guard may hold an unsafe pair, and editing an unrelated field must not leave it
  807. # schedulable. Merging this update over the stored job lets an operator remediate.
  808. return _validate_cron_base_url(_pick(updates, job, "provider"), _pick(updates, job, "base_url"))
  809. def _update_script_fields(job: dict[str, Any], a: dict[str, Any], updates: dict[str, Any]) -> Optional[str]:
  810. """script / monitor_script / monitor_url (empty string clears); returns an error string or None."""
  811. monitor_script, monitor_url = a["monitor_script"], a["monitor_url"]
  812. for field, value in (("script", a["script"]), ("monitor_script", monitor_script)):
  813. if value is not None:
  814. if value:
  815. path_error = _validate_cron_script_path(value)
  816. if path_error:
  817. return path_error
  818. updates[field] = _normalize_optional_job_value(value) if value else None
  819. if monitor_url is not None:
  820. updates["monitor_url"] = _normalize_optional_job_value(monitor_url) if monitor_url else None
  821. if (monitor_script is not None or monitor_url is not None) and (
  822. _pick(updates, job, "monitor_script") and _pick(updates, job, "monitor_url")):
  823. return("monitor_script and monitor_url are mutually exclusive — clear one before setting the other.")
  824. return None
  825. def _update_context_from(job: dict[str, Any], a: dict[str, Any], updates: dict[str, Any]) -> Optional[str]:
  826. """context_from / continuity: empty string / list clears; otherwise every ref must
  827. exist. Stored as a list (or None) to match create_job()."""
  828. context_from, continuity = a["context_from"], a["continuity"]
  829. if context_from is None and continuity is None:
  830. return None
  831. if context_from is None:
  832. context_from = list(job.get("context_from") or []) # continuity-only update
  833. refs = _clean_str_list(context_from)
  834. if continuity is not None:
  835. refs = _apply_continuity(refs, continuity) or []
  836. if refs:
  837. ref_error = _validate_context_from_refs(refs)
  838. if ref_error:
  839. return ref_error
  840. updates["context_from"] = refs or None
  841. return None
  842. def _update_run_fields(job: dict[str, Any], a: dict[str, Any], updates: dict[str, Any]) -> Optional[str]:
  843. """enabled_toolsets / attach_to_session / workdir / no_agent / repeat / schedule."""
  844. if a["enabled_toolsets"] is not None:
  845. # [] is an explicit zero-tool allowlist, not a clear back to the unrestricted default (#82010).
  846. updates["enabled_toolsets"] = a["enabled_toolsets"]
  847. if a["attach_to_session"] is not None:
  848. updates["attach_to_session"] = bool(a["attach_to_session"])
  849. if a["workdir"] is not None:
  850. # Empty string clears; otherwise update_job() validates/normalizes.
  851. updates["workdir"] = _normalize_optional_job_value(a["workdir"]) or None
  852. if a["no_agent"] is not None:
  853. # Flipping to True needs a script on the job or in this same update.
  854. target_no_agent = bool(a["no_agent"])
  855. if target_no_agent and not _pick(updates, job, "script"):
  856. return (
  857. "Cannot set no_agent=True on a job without a script. "
  858. "Set `script` in the same update, or on the job first.")
  859. updates["no_agent"] = target_no_agent
  860. if a["repeat"] is not None:
  861. # Shared chokepoint coerces string forms ('forever'/'once'/'3') and 0/negative.
  862. from cron.jobs import normalize_repeat_value
  863. repeat_state = dict(job.get("repeat") or {})
  864. repeat_state["times"] = normalize_repeat_value(a["repeat"])
  865. updates["repeat"] = repeat_state
  866. if a["timezone"] is not None:
  867. # '' clears the pin (the job inherits the profile timezone); update_job validates the key.
  868. updates["timezone"] = _normalize_optional_job_value(a["timezone"]) or ""
  869. if a["grace_minutes"] is not None:
  870. # '' / null clears the override; 0 is a real value ("never catch up") and passes through.
  871. updates["grace_minutes"] = (
  872. None if isinstance(a["grace_minutes"], str) and not a["grace_minutes"].strip()
  873. else a["grace_minutes"])
  874. if _pick(updates, job, "grace_minutes") is not None:
  875. kind = (_pick(updates, job, "schedule") or {}).get("kind")
  876. if kind != "cron":
  877. return "grace_minutes only applies to cron schedules."
  878. if a["schedule"] is not None:
  879. parsed_schedule = parse_schedule(a["schedule"])
  880. updates["schedule"] = parsed_schedule
  881. updates["schedule_display"] = parsed_schedule.get("display", a["schedule"])
  882. if job.get("state") != "paused":
  883. updates["state"] = "scheduled"
  884. updates["enabled"] = True
  885. return None
  886. # Validation order is behavior (first failing field wins): keep this sequence.
  887. _UPDATE_STEPS = (_update_core_fields, _update_script_fields, _update_context_from, _update_run_fields)
  888. def _action_update(job: dict[str, Any], a: dict[str, Any]) -> str:
  889. updates: dict[str, Any] = {}
  890. for step in _UPDATE_STEPS:
  891. error = step(job, a, updates)
  892. if error:
  893. return tool_error(error, success=False)
  894. if not updates:
  895. return tool_error("No updates provided.", success=False)
  896. updated = update_job(job["id"], updates)
  897. _notify_provider_jobs_changed_safe()
  898. # An update can switch modes or delivery — echo the same guidance as create.
  899. return _dumps(_with_guidance(
  900. {"success": True, "job": _format_job(updated)}, updated, _normalize_deliver_param(a["deliver"])))
  901. _JOBLESS_ACTIONS = {"create": _action_create, "list": _action_list}
  902. def _action_cancel(job: dict[str, Any], a: dict[str, Any]) -> str:
  903. """Stop the job's LIVE run. Per-run, not per-schedule: the job keeps its next fire."""
  904. from cron.cancellation import cancel_run
  905. # The schedule is untouched, so no provider notification — nothing about future fires moved.
  906. return _dumps({"job": _format_job(job), **cancel_run(job["id"], reason=a.get("reason"))})
  907. _JOB_ACTIONS = {
  908. "remove": _action_remove, "update": _action_update,
  909. "run": _action_run, "run_now": _action_run, "trigger": _action_run,
  910. "pause": lambda job, a: _job_state_result(pause_job(job["id"], reason=a["reason"])),
  911. "resume": lambda job, a: _job_state_result(resume_job(job["id"])),
  912. "cancel": _action_cancel,
  913. }
  914. def _resolve_job_or_error(job_id: str):
  915. """``(job, None)`` or ``(None, json_error)`` for a job_id/name reference."""
  916. try:
  917. job = resolve_job_ref(job_id)
  918. except AmbiguousJobReference as exc:
  919. return None, _dumps({
  920. "success": False,
  921. "error": str(exc),
  922. "matches": [
  923. {"id": m["id"], "name": m.get("name"), "schedule": m.get("schedule_display"), "next_run_at": m.get("next_run_at")}
  924. for m in exc.matches
  925. ],
  926. })
  927. if not job:
  928. return None, _dumps(
  929. {"success": False, "error": f"Job with ID or name '{job_id}' not found. Use cronjob(action='list') to inspect jobs."},
  930. )
  931. return job, None
  932. def cronjob(
  933. action: str,
  934. job_id: Optional[str] = None,
  935. prompt: Optional[str] = None,
  936. schedule: Optional[str] = None,
  937. name: Optional[str] = None,
  938. repeat: Optional[int] = None,
  939. deliver: Optional[str] = None,
  940. include_disabled: bool = False,
  941. skill: Optional[str] = None,
  942. skills: Optional[list[str]] = None,
  943. model: Optional[str] = None,
  944. provider: Optional[str] = None,
  945. base_url: Optional[str] = None,
  946. reason: Optional[str] = None,
  947. script: Optional[str] = None,
  948. context_from: Optional[str | list[str]] = None,
  949. continuity: Optional[bool] = None,
  950. enabled_toolsets: Optional[list[str]] = None,
  951. workdir: Optional[str] = None,
  952. no_agent: Optional[bool] = None,
  953. attach_to_session: Optional[bool] = None,
  954. monitor_script: Optional[str] = None,
  955. monitor_url: Optional[str] = None,
  956. reasoning_effort: Optional[str] = None,
  957. failure_deliver: Optional[str | list[str]] = None,
  958. catch_up: Optional[bool] = None,
  959. task_id: str | None = None,
  960. session_id: Optional[str] = None,
  961. paused: bool = False,
  962. paused_reason: Optional[str] = None,
  963. pinned: Optional[bool] = None,
  964. interpreter: Optional[str] = None,
  965. timezone: Optional[str] = None,
  966. grace_minutes: Optional[int | str] = None) -> str:
  967. """Unified cron job management tool."""
  968. a = dict(locals())
  969. del a["task_id"] # unused but kept for handler signature compatibility
  970. try:
  971. normalized = (action or "").strip().lower()
  972. handler = _JOBLESS_ACTIONS.get(normalized)
  973. if handler is not None:
  974. return handler(a)
  975. if not job_id:
  976. return tool_error(f"job_id is required for action '{normalized}'", success=False)
  977. # Job resolution precedes the action check (an unknown action on a missing job
  978. # reports the missing job) — preserved ordering.
  979. job, error = _resolve_job_or_error(job_id)
  980. if error is not None:
  981. return error
  982. handler = _JOB_ACTIONS.get(normalized)
  983. if handler is None:
  984. return tool_error(f"Unknown cron action '{action}'", success=False)
  985. return handler(job, a)
  986. except Exception as e:
  987. return tool_error(str(e), success=False)
  988. def _script_description(home: str) -> str:
  989. return (f"Optional script run each tick; stdout is injected into the agent's prompt as context (with no_agent=True "
  990. f"the script IS the job). Relative paths resolve under {home}/scripts/; .sh/.bash via bash, else Python. "
  991. "On update, '' clears.")
  992. def _cronjob_schema_overrides() -> dict:
  993. """Rebuild the ``script`` path hint from the ACTIVE profile at every get_definitions(): the
  994. static schema is built once per process, but the multiplexed gateway serves every profile from
  995. that process, so a path baked in at import would name the launch profile's home (#95685)."""
  996. params = copy.deepcopy(CRONJOB_SCHEMA["parameters"])
  997. params["properties"]["script"]["description"] = _script_description(display_hermes_home())
  998. return {"parameters": params}
  999. CRONJOB_SCHEMA = {
  1000. "name": "cronjob_manage",
  1001. "description": """Schedule recurring or one-off agent tasks (cron jobs): create, list, edit, run. action='create' schedules a job from a prompt and/or skills; 'list' inspects jobs; 'update'/'pause'/'resume'/'remove' manage one by job_id (always list first — never guess job IDs); 'run' fires a job immediately in the BACKGROUND (returns a handle at once, outcome re-enters the conversation when done — do not wait or poll; optional 'prompt' adds transient context for that fire only); 'cancel' stops a job's LIVE run without touching its schedule (a no-op when nothing is in flight).
  1002. Jobs run in a fresh session with no current-chat context, so prompts must be self-contained, and the agent's FINAL RESPONSE is what gets delivered — cron runs are autonomous and cannot ask questions. Jobs run on the main agent model (whatever `hermes model` is set to when they fire) unless the user pins one. Prefer updating an existing job over creating near-duplicates.""",
  1003. "parameters": {
  1004. "type": "object",
  1005. "properties": {
  1006. "paused": {"type": "boolean", "description": "Create only: persist disabled atomically. Resume to schedule; explicit run remains available. Default false."},
  1007. "paused_reason": {"type": "string", "description": "Create only: auditable reason; requires paused=true."},
  1008. "action": {
  1009. "type": "string",
  1010. "description": "One of: create, list, update, pause, resume, remove, run, cancel. When action=create, the 'schedule' and 'prompt' fields are REQUIRED."
  1011. },
  1012. "job_id": {
  1013. "type": "string",
  1014. "description": "Required for update/pause/resume/remove/run/cancel."
  1015. },
  1016. "pinned": {
  1017. "type": "boolean",
  1018. "description": "For create/update. ONLY set when the user explicitly asks to pin (or unpin) a job's model. pinned=true locks the CURRENT main agent model (and its provider) onto the job so later `hermes model` / `/model` changes never touch it; pinned=false releases the lock so the job follows the main agent model again. Never set it on your own initiative: by default jobs follow the main model."
  1019. },
  1020. "prompt": {
  1021. "type": "string",
  1022. "description": "For create/update: the full self-contained prompt (when it runs goes in 'schedule'; data windows like 'since yesterday' stay here; paired with any skills as the task instruction). For run: optional transient context for that single fire (never persisted)."
  1023. },
  1024. "schedule": {
  1025. "type": "string",
  1026. "type": "string",
  1027. "description": "REQUIRED for create. Schedule forms: (1) recurring interval — '30m', 'every 2h', 'every hour' (EVERY 30 minutes / 2 hours / hour, forever by default); (2) explicit one-shot by duration — 'in 30m', 'in 2h' (fires ONCE that far from now; use this for 'remind me in N minutes' — do NOT hand-compute an absolute timestamp); (3) natural day/time — 'every monday 9am', 'weekdays at 9am', 'every day at 9am' (recurring weekly/daily); (4) cron syntax — '0 9 * * *' (daily 9am); (5) absolute one-shot — ISO timestamp '2026-06-01T09:00:00'."
  1028. },
  1029. "name": {
  1030. "type": "string",
  1031. "description": "Optional human-friendly name"
  1032. },
  1033. "repeat": {
  1034. "type": "integer",
  1035. "description": "Optional repeat count. Omit for defaults (once for one-shot, forever for recurring)."
  1036. },
  1037. "deliver": {
  1038. "type": "string",
  1039. "description": "Where the job's output is POSTED as a one-way message (the job itself always runs in a fresh session with no chat context). Omit to address the chat/topic this job was created from. Otherwise: 'local' (save only, no delivery), 'all' (every connected home channel, resolved at fire time), 'bot-chat' or 'bot-chat:<profile>' (inject into a Bot Chat as a real message), 'room:<room_id>' (post into a Bot Mode Group Chat; it starts a round only for members the output @mentions), or platform:chat_id:thread_id (e.g. 'telegram:-1001234567890:17585'). Comma-combine like 'origin,all'. From a local chat (CLI, TUI or Desktop) the default writes the result into that chat's history as a note, seen when the chat is reopened and by its next turn, not as a live message."
  1040. },
  1041. "failure_deliver": {
  1042. "type": "string",
  1043. "description": "Optional override target for FAILURE notices only (same grammar as deliver). When set, engine failure/interruption notices go here instead of the deliver target; 'local' suppresses them entirely (state still recorded in cron list/run history). Use for jobs delivering into shared channels where failure noise is unwanted. Omit = failures follow deliver (default). On update, '' clears."
  1044. },
  1045. "skills": {
  1046. "type": "array",
  1047. "items": {"type": "string"},
  1048. "description": "Optional ordered skill names loaded before the cron prompt. On update, [] clears."
  1049. },
  1050. "catch_up": {
  1051. "type": "boolean",
  1052. "description": "Runs missed while the machine was off: true = run once on return, false = skip. Omit for the default (recurring run once, one-shots skip)."
  1053. },
  1054. "script": {
  1055. "type": "string",
  1056. "description": _script_description("the profile HERMES_HOME")
  1057. },
  1058. "monitor": {
  1059. "type": "string",
  1060. "description": "Optional change-detector that gates the agent: an http(s) URL (fetched each tick) or a script path (same rules as `script`, run each tick) — cheap, no LLM. Output identical to the previous tick skips the agent run entirely; changed output wakes the agent with a diff injected into the prompt. First tick always runs (baseline). Output must be deterministic (no timestamps) or every tick looks changed. Incompatible with no_agent. On update, '' clears."
  1061. },
  1062. "no_agent": {
  1063. "type": "boolean",
  1064. "default": False,
  1065. "description": "True = no LLM: the scheduler runs `script` (required) on schedule and delivers its stdout verbatim; empty stdout sends nothing (watchdog pattern). Use for script-only pings with fixed output; keep False for anything needing reasoning."
  1066. },
  1067. "context_from": {
  1068. "type": "array",
  1069. "items": {"type": "string"},
  1070. "description": "Optional job ID(s) whose most recent completed output is injected as context each run — chains jobs (A collects, B processes). For a job's OWN previous output prefer `continuity`. On update, [] clears."
  1071. },
  1072. "continuity": {
  1073. "type": "boolean",
  1074. "description": "True = each run sees the job's own previous output, so it can dedupe and continue where it left off (scouts, monitors, incremental digests). Default false. On update, false turns it off."
  1075. },
  1076. "enabled_toolsets": {
  1077. "type": "array",
  1078. "items": {"type": "string"},
  1079. "description": "Optional toolset names to restrict the job's agent to (e.g. [\"web\", \"terminal\"]) — cuts token overhead. Infer from the prompt. Omit for all default tools. [] = an explicit zero-tool allowlist (the job's agent gets no toolsets, MCP servers included unless named); it is preserved as [], never widened back to the default."
  1080. },
  1081. "workdir": {
  1082. "type": "string",
  1083. "description": "Optional absolute existing path to run the job from: injects that directory's AGENTS.md/context files and anchors terminal/file tools there. On update, '' clears."
  1084. },
  1085. "timezone": {
  1086. "type": "string",
  1087. "description": "Optional IANA zone (e.g. 'America/New_York') the cron expression is read in, instead of the profile's. On update, '' clears."
  1088. },
  1089. "attach_to_session": {
  1090. "type": "boolean",
  1091. "description": "True = the job's delivery is CONTINUABLE — the user can reply and the agent has the brief in context (threads on thread-capable platforms, mirrored into the DM elsewhere). Use for conversational recurring jobs (briefings); leave unset for fire-and-forget alerts. Scope: the job's own conversation only — the origin chat, the home-channel fallback when deliver='origin' captured no origin (script-created jobs), a user-written bare platform target (deliver='slack' — that platform's home channel), or the job's single explicit platform:chat target (this flag is the only way to attach an explicit target). Broadcast targets are never attached; no effect when deliver='local'."
  1092. },
  1093. },
  1094. "required": ["action"]
  1095. }
  1096. }
  1097. def check_cronjob_requirements() -> bool:
  1098. """Available in interactive CLI mode, gateway/messaging platforms, and cron runs (the
  1099. scheduler is internal; no crontab needed). Flags must be explicitly truthy via
  1100. ``env_var_enabled``. An external cron worker has the presence vars stripped from its env, so
  1101. the cron session marker keeps ``cron.allow_agent_scheduling`` meaningful there."""
  1102. from gateway.session_context import get_session_env
  1103. from utils import env_var_enabled, is_truthy_value
  1104. return (
  1105. env_var_enabled("HERMES_INTERACTIVE")
  1106. or env_var_enabled("HERMES_GATEWAY_SESSION")
  1107. or env_var_enabled("HERMES_EXEC_ASK")
  1108. or is_truthy_value(get_session_env("HERMES_CRON_SESSION", ""))
  1109. )
  1110. # Agent-facing arguments forwarded verbatim to cronjob(). model / provider / base_url are
  1111. # intentionally NOT here: per-job inference pins are user-owned (dashboard, `hermes cron
  1112. # create/edit --model`, hand-edited jobs) — the agent must not point unattended spend at a
  1113. # different model. Programmatic callers of cronjob() itself retain the parameters.
  1114. _HANDLER_FORWARDED_ARGS = (
  1115. "job_id", "prompt", "schedule", "name", "repeat", "deliver", "failure_deliver", "skill", "skills", "reason",
  1116. "script", "context_from", "continuity", "enabled_toolsets", "workdir", "no_agent", "attach_to_session",
  1117. "paused_reason", "pinned", "timezone", "grace_minutes", "catch_up")
  1118. def _cronjob_handler(args, **kw):
  1119. """Model-tool dispatch: resolves the one model-facing ``monitor`` field into the stored
  1120. ``monitor_script``/``monitor_url`` pair (legacy field names still accepted)."""
  1121. _mon_script, _mon_url = _split_monitor_arg(args.get("monitor"), args.get("monitor_script"), args.get("monitor_url"))
  1122. return cronjob(
  1123. action=args.get("action", ""),
  1124. include_disabled=args.get("include_disabled", True),
  1125. monitor_script=_mon_script,
  1126. monitor_url=_mon_url,
  1127. task_id=kw.get("task_id"),
  1128. session_id=kw.get("session_id"),
  1129. paused=args.get("paused", False),
  1130. **{key: args.get(key) for key in _HANDLER_FORWARDED_ARGS},
  1131. )
  1132. registry.register(
  1133. name="cronjob_manage",
  1134. toolset="cronjob",
  1135. schema=CRONJOB_SCHEMA,
  1136. handler=_cronjob_handler,
  1137. check_fn=check_cronjob_requirements,
  1138. emoji="⏰",
  1139. dynamic_schema_overrides=_cronjob_schema_overrides,
  1140. )