cron.py 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976
  1. """Cron dashboard routes.
  2. The ``*_sync`` workers, profile resolution and the threadpool wrapper
  3. (``_run_cron_dashboard_io``) live in web_server_cron and are reached through the
  4. late-binding seam so ``monkeypatch.setattr(web_server_cron, ...)`` keeps working.
  5. """
  6. import asyncio
  7. import functools
  8. import math
  9. import re
  10. import time
  11. from contextlib import contextmanager
  12. from datetime import datetime
  13. from pathlib import Path
  14. from typing import Any, Dict, List, Optional
  15. from fastapi import APIRouter, HTTPException, Request
  16. from fastapi.responses import JSONResponse
  17. from hermes_cli.web_deps import late
  18. from hermes_cli.config import cfg_get
  19. from hermes_cli.web_server_cron import (
  20. _create_cron_job_sync, _cron_optional_text, _cron_string_list, _mutate_cron_for_profile, _normalize_dashboard_cron_script, _raise_if_cron_registration_error, _run_cron_dashboard_io, _validate_dashboard_cron_context_from, _validate_dashboard_cron_effective_job,
  21. )
  22. from hermes_cli.web_models import AutomationBlueprintInstantiate, CronJobCreate, CronJobUpdate
  23. from hermes_cli.web_routers._common import log as _log
  24. from hermes_time import get_timezone as _get_timezone
  25. from hermes_state_keys import legacy_cron_run_job
  26. from hermes_constants import (
  27. get_hermes_home as _get_hermes_home,
  28. reset_hermes_home_override as _reset_hermes_home_override,
  29. set_hermes_home_override as _set_hermes_home_override,
  30. )
  31. router = APIRouter()
  32. _find_cron_job_profile = late("_find_cron_job_profile", "hermes_cli.web_server_cron")
  33. _fire_cron_job_for_profile = late("_fire_cron_job_for_profile", "hermes_cli.web_server_cron")
  34. _forward_cron_fire_to_gateway = late("_forward_cron_fire_to_gateway", "hermes_cli.web_server_cron")
  35. _gateway_intentionally_stopped = late("_gateway_intentionally_stopped", "hermes_cli.web_server_cron")
  36. _notify_cron_provider_for_profile = late("_notify_cron_provider_for_profile", "hermes_cli.web_server_cron")
  37. _call_cron_for_profile = late("_call_cron_for_profile", "hermes_cli.web_server_cron")
  38. load_config = late("load_config", "hermes_cli.config")
  39. _cron_profile_dicts = late("_cron_profile_dicts", "hermes_cli.web_server_cron")
  40. _cron_profile_home = late("_cron_profile_home", "hermes_cli.web_server_cron")
  41. _open_session_db_for_profile = late("_open_session_db_for_profile", "hermes_cli.web_server_sessions")
  42. _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
  43. def _job_not_found() -> HTTPException:
  44. return HTTPException(status_code=404, detail="Job not found")
  45. def _normalize_dashboard_cron_updates(updates: dict[str, Any], profile_home: Path) -> dict[str, Any]:
  46. """Normalize dashboard JSON into cron.jobs.update_job's storage shape.
  47. Stays in the dashboard adapter layer on purpose: cron/jobs.py is the source
  48. of truth for scheduling; this only translates form payloads into shapes the
  49. core functions already accept.
  50. """
  51. normalized = dict(updates or {})
  52. for key in ("model", "provider", "workdir"):
  53. if key in normalized:
  54. normalized[key] = _cron_optional_text(normalized[key])
  55. if "script" in normalized:
  56. normalized["script"] = _normalize_dashboard_cron_script(normalized["script"], profile_home)
  57. if "base_url" in normalized:
  58. normalized["base_url"] = _cron_optional_text(normalized["base_url"], strip_trailing_slash=True)
  59. if "deliver" in normalized:
  60. normalized["deliver"] = _cron_optional_text(normalized["deliver"]) or "local"
  61. if "failure_deliver" in normalized:
  62. # Same normalization as deliver, but empty CLEARS the override (failures
  63. # fall back to deliver) rather than coalescing — the field is optional.
  64. normalized["failure_deliver"] = _cron_optional_text(normalized["failure_deliver"])
  65. for key in ("context_from", "enabled_toolsets"):
  66. if key in normalized:
  67. normalized[key] = _cron_string_list(normalized[key])
  68. return normalized
  69. def _job_owner_profile(job_id: str, profile: Optional[str]) -> Optional[str]:
  70. """Profile that holds ``job_id`` (its jobs.json and the state.db with its run sessions).
  71. ``profile`` is a caller *hint*, not proof of ownership: the Desktop lists
  72. jobs cross-profile (``?profile=all``) while its per-item calls carry the
  73. ambient active profile. Trusting that hint opened a profile that does not
  74. hold the job, so the job's run-session lookup matched nothing and
  75. answered ``200 {"runs": []}`` — the UI rendered "No runs yet" for a job
  76. that had run many times (#115345). A hint that does hold the job still
  77. wins, so deliberately scoped lookups (the same job id in two profiles,
  78. e.g. a copied jobs.json) keep reading the named profile.
  79. """
  80. if profile:
  81. jobs = _call_cron_for_profile(profile, "list_jobs", True)
  82. if any(j.get("id") == job_id or j.get("name") == job_id for j in jobs):
  83. return profile
  84. return _find_cron_job_profile(job_id)
  85. def _job_profile(job_id: str, profile: Optional[str]) -> str:
  86. """Owning profile for the get/update/pause/resume/trigger/delete family; 404 when no profile
  87. holds the job. Same hint validation as the run lookup, so a wrong-profile hint from the
  88. cross-profile list cannot 404 (or act on the wrong store for) a job the server can locate."""
  89. selected = _job_owner_profile(job_id, profile)
  90. if not selected:
  91. raise _job_not_found()
  92. return selected
  93. def _found(job):
  94. if not job:
  95. raise _job_not_found()
  96. return job
  97. def _list_cron_jobs_sync(profile: str = "all"):
  98. requested = (profile or "all").strip()
  99. if requested.lower() != "all":
  100. return _call_cron_for_profile(requested, "list_jobs", True)
  101. # Aggregating across profiles can surface the SAME job id more than once —
  102. # e.g. a job copied into a second profile's cron/jobs.json during profile
  103. # creation. Deduplicate by id, deterministically preferring the default
  104. # profile's copy over per-iteration order (#51721): collect all jobs first,
  105. # then resolve duplicates by id with default-profile priority, rather than
  106. # keeping whichever copy happened to be seen first during the profile loop.
  107. all_jobs: list[dict[str, Any]] = []
  108. for item in _cron_profile_dicts():
  109. name = str(item.get("name") or "")
  110. if not name:
  111. continue
  112. try:
  113. all_jobs.extend(_call_cron_for_profile(name, "list_jobs", True))
  114. except Exception:
  115. _log.exception("Failed to list cron jobs for profile %s", name)
  116. by_id: dict[str, dict[str, Any]] = {}
  117. unkeyed: list[dict[str, Any]] = []
  118. for job in all_jobs:
  119. if not isinstance(job, dict):
  120. continue
  121. jid = job.get("id") or job.get("job_id")
  122. # cron.jobs._normalize_job_record() fills a missing id with the literal
  123. # sentinel string "unknown" — which is truthy, so a plain `if not jid`
  124. # check would NOT catch it and two genuinely different id-less jobs from
  125. # different profiles would collapse into one under this shared sentinel
  126. # key. Treat the sentinel the same as no id: never deduplicated.
  127. if not jid or jid == "unknown":
  128. unkeyed.append(job)
  129. continue
  130. existing = by_id.get(jid)
  131. if existing is None or (
  132. job.get("is_default_profile") and not existing.get("is_default_profile")
  133. ):
  134. by_id[jid] = job
  135. return list(by_id.values()) + unkeyed
  136. def _get_cron_job_sync(job_id: str, profile: Optional[str] = None):
  137. return _found(_call_cron_for_profile(_job_profile(job_id, profile), "get_job", job_id))
  138. _CRON_OUTPUT_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S"
  139. def _cron_output_runs_dir(profile: Optional[str], job_id: str) -> Path:
  140. """Output docs live under the job's home — resolve it even without a hint."""
  141. if profile:
  142. try:
  143. _, profile_home = _cron_profile_home(profile)
  144. except Exception:
  145. profile_home = _get_hermes_home()
  146. else:
  147. profile_home = _get_hermes_home()
  148. return Path(profile_home) / "cron" / "output" / job_id
  149. @contextmanager
  150. def _owner_home_scope(profile: Optional[str]):
  151. """Keep reads in the owner's profile context for the whole run-history build.
  152. Filename stems, the execution ledger and ``hermes_time``'s configured zone all
  153. resolve through the *current* ``get_hermes_home()``; without this scope a
  154. cross-profile listing decodes another profile's docs with the dashboard's
  155. zone (an hours-off timestamp) and reads the dashboard's executions.db. The
  156. dashboard's own profile needs no override (its home is already current).
  157. """
  158. if not profile:
  159. yield None
  160. return
  161. try:
  162. _, profile_home = _cron_profile_home(profile)
  163. except Exception:
  164. yield None
  165. return
  166. if Path(profile_home).resolve() == _get_hermes_home().resolve():
  167. yield None
  168. return
  169. token = _set_hermes_home_override(str(profile_home))
  170. try:
  171. yield profile_home
  172. finally:
  173. _reset_hermes_home_override(token)
  174. def _cron_output_run_timestamp(path: Path) -> Optional[float]:
  175. """Epoch seconds for an output filename's wall time.
  176. save_job_output writes the stem with hermes_time.now() — the configured
  177. IANA timezone (HERMES_TIMEZONE / config), falling back to server-local.
  178. Read it back with that same ZONE, not a fixed offset snapshot of today's
  179. local offset: a snapshot is wrong when the configured zone differs from
  180. the server's, and off by an hour for a historical file across a DST
  181. transition. With no configured zone, astimezone() on the naive datetime
  182. resolves the offset in effect on the filename's own date.
  183. """
  184. try:
  185. naive = datetime.strptime(path.stem, _CRON_OUTPUT_FILENAME_FORMAT)
  186. except ValueError:
  187. return None
  188. tz = _get_timezone()
  189. if tz is not None:
  190. return naive.replace(tzinfo=tz).timestamp()
  191. return naive.astimezone().timestamp()
  192. def _cron_output_run_preview(path: Path, max_chars: int = 180) -> str:
  193. try:
  194. raw = path.read_text(encoding="utf-8-sig", errors="replace")
  195. except OSError:
  196. return ""
  197. preview = re.sub(r"\s+", " ", raw).strip()
  198. if len(preview) <= max_chars:
  199. return preview
  200. return preview[: max_chars - 1].rstrip() + "…"
  201. def _cron_job_last_run_timestamp(job: Optional[dict[str, Any]]) -> Optional[float]:
  202. if not isinstance(job, dict):
  203. return None
  204. raw = job.get("last_run_at")
  205. if isinstance(raw, (int, float)):
  206. return float(raw)
  207. if isinstance(raw, str):
  208. text = raw.strip()
  209. if not text:
  210. return None
  211. try:
  212. return datetime.fromisoformat(text).timestamp()
  213. except ValueError:
  214. return None
  215. return None
  216. def _cron_output_status_label(job: Optional[dict[str, Any]]) -> str:
  217. if not isinstance(job, dict):
  218. return ""
  219. status = str(job.get("last_status") or "").strip()
  220. if not status:
  221. return ""
  222. return status.replace("_", " ").upper()
  223. def _cron_output_run_row(started_at: float, title: str, preview: Optional[str]) -> dict[str, Any]:
  224. return {
  225. "title": title,
  226. "preview": preview or None,
  227. "source": "cron_output",
  228. "started_at": started_at,
  229. "last_active": started_at,
  230. "ended_at": started_at,
  231. "input_tokens": 0,
  232. "output_tokens": 0,
  233. "message_count": 0,
  234. "tool_call_count": 0,
  235. "model": None,
  236. "cwd": None,
  237. "archived": False,
  238. "is_active": False,
  239. }
  240. def _iso_to_epoch(text: Any) -> Optional[float]:
  241. if not isinstance(text, str) or not text.strip():
  242. return None
  243. try:
  244. return datetime.fromisoformat(text.strip()).timestamp()
  245. except ValueError:
  246. return None
  247. def _owner_profile_executions(canonical_job_id: str) -> list[dict[str, Any]]:
  248. """Terminal execution-ledger rows for the job, newest first.
  249. Each script-only fire creates exactly one ledger row (claimed → completed /
  250. failed), so the ledger — not filename proximity to ``last_run_at`` — is the
  251. per-attempt record that pairs an output doc with its real status. Read inside
  252. the owner-home scope so it opens the OWNER's cron/executions.db.
  253. """
  254. try:
  255. from cron.executions import _TERMINAL_STATES, list_executions
  256. rows = list_executions(job_id=canonical_job_id, limit=100)
  257. except Exception:
  258. return []
  259. terminal: list[dict[str, Any]] = []
  260. for row in rows:
  261. if str(row.get("status") or "") not in _TERMINAL_STATES:
  262. continue
  263. terminal.append({
  264. "id": str(row.get("id") or ""),
  265. "claimed_at": _iso_to_epoch(row.get("claimed_at")) or 0.0,
  266. "finished_at": _iso_to_epoch(row.get("finished_at")),
  267. "status": str(row.get("status") or ""),
  268. "error": str(row.get("error") or "").strip(),
  269. })
  270. terminal.sort(key=lambda r: r["claimed_at"], reverse=True)
  271. return terminal
  272. def _execution_contains(
  273. attempt: dict[str, Any], started_at: float, grace_seconds: float = 300.0,
  274. ) -> bool:
  275. """Whether an output doc's timestamp falls inside a ledger attempt's window.
  276. ``save_job_output`` runs after the script finishes but before
  277. ``finish_execution`` closes the attempt, and both clocks are
  278. ``hermes_time.now()`` in the owner's zone, so a doc written by an attempt
  279. lands within ``[claimed_at, finished_at + grace]`` of THAT attempt (a still-
  280. open attempt has no ``finished_at``). Older attempts closed before this doc
  281. was written, which is what keeps a fast-firing job's status from bleeding
  282. onto an earlier run's row. The doc's filename keeps whole seconds only, so
  283. the claim is floored to its second: an attempt claimed at ``:00.500`` that
  284. writes at ``:00.750`` files its doc as ``:00``.
  285. """
  286. if started_at + 0.001 < math.floor(attempt["claimed_at"]):
  287. return False
  288. finish = attempt["finished_at"]
  289. return finish is None or started_at <= finish + grace_seconds
  290. # A cancelled or missed attempt's ``error`` is its reason ("job removed while running",
  291. # "missed ×3 (...)"), the row's whole point — show it like a failure's.
  292. _LABELS_WITH_REASON = frozenset({"FAILED", "CANCELLED", "MISSED"})
  293. # Terminal attempts that carry a real outcome (``unknown`` is an owner that vanished).
  294. _OUTCOME_STATES = ("completed", "failed", "cancelled", "missed")
  295. def _execution_status_title(status: str, error: str, fallback: str) -> str:
  296. label = (status or "").replace("_", " ").upper()
  297. if label == "UNKNOWN":
  298. return fallback
  299. if not label:
  300. return fallback
  301. return f"{label} · {error}" if (label in _LABELS_WITH_REASON and error) else f"{label} · {fallback}"
  302. def _list_cron_output_runs(
  303. job: Optional[dict[str, Any]],
  304. canonical_job_id: str,
  305. profile: Optional[str],
  306. limit: int,
  307. ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
  308. """SessionDB-less run records for jobs that never create agent sessions: ``(docs, attempts)``.
  309. Script-only (no_agent) jobs deliberately skip SessionDB (cron/scheduler.run_job),
  310. so their completed runs are invisible to the run-history endpoint. Their
  311. output docs (one .md per fire under cron/output/<job_id>/) and the execution
  312. ledger (one row per fire) are the only per-run records. ``docs`` are rows in
  313. /api/sessions shape with source='cron_output', newest first, so the frontend
  314. reuses SessionInfo; ids use a cron_output: prefix that can never collide with a
  315. SessionDB session id. ``attempts`` are the job's terminal ledger rows, newest
  316. first. A doc never records its execution, so which attempt it stands for is
  317. decided by ``_reconcile_cron_runs`` beside the sessions, never here, and never
  318. from ``last_run_at`` proximity, which mislabels an older doc when the newest
  319. run's doc is missing.
  320. """
  321. output_dir = _cron_output_runs_dir(profile, canonical_job_id)
  322. try:
  323. files = sorted(
  324. (path for path in output_dir.glob("*.md") if path.is_file()),
  325. key=lambda path: path.name,
  326. reverse=True,
  327. )
  328. except OSError:
  329. files = []
  330. attempts = _owner_profile_executions(canonical_job_id)
  331. docs: list[dict[str, Any]] = []
  332. for path in files[:limit]:
  333. started_at = _cron_output_run_timestamp(path)
  334. if started_at is None:
  335. try:
  336. started_at = path.stat().st_mtime
  337. except OSError:
  338. started_at = 0.0
  339. preview = _cron_output_run_preview(path)
  340. docs.append({
  341. "id": f"cron_output:{canonical_job_id}:{path.stem}",
  342. **_cron_output_run_row(started_at, preview or "Script-only run", preview),
  343. })
  344. if docs or any(attempt["status"] in _OUTCOME_STATES for attempt in attempts):
  345. return docs, attempts
  346. # No output docs survived (pruned or never written) but the job HAS run:
  347. # surface one metadata-only row from last_run_at/last_status instead of the
  348. # bare "No runs" the issue reports.
  349. latest_ts = _cron_job_last_run_timestamp(job)
  350. if latest_ts is None:
  351. return [], attempts
  352. preview = ""
  353. if isinstance(job, dict):
  354. preview = str(job.get("last_error") or "").strip()
  355. title = _cron_output_status_label(job) or "Script-only run"
  356. if preview:
  357. title = f"{title} · {preview}"
  358. return [{
  359. "id": f"cron_output:{canonical_job_id}:latest",
  360. **_cron_output_run_row(latest_ts, title, preview),
  361. }], attempts
  362. # A legacy run's session row is written just AFTER its execution claim; the grace only
  363. # absorbs float/ISO rounding between the two clocks on one host.
  364. _OWNERSHIP_CLAIM_GRACE_SECONDS = 5.0
  365. def _live_inflight_execution(canonical_job_id: str) -> Optional[dict[str, Any]]:
  366. """The job's claimed/running ledger attempt under a live owner, or None (fail closed).
  367. Must run inside the owner-home scope so it reads the OWNER's executions.db.
  368. """
  369. try:
  370. from cron.executions import live_inflight_execution
  371. return live_inflight_execution(canonical_job_id)
  372. except Exception:
  373. return None
  374. def _run_owned_by(session: dict[str, Any], inflight: Optional[dict[str, Any]]) -> bool:
  375. """Whether the scheduler still OWNS this never-closed run session (#88443).
  376. ``is_active`` is a 300s activity window, so a live run inside a long tool call
  377. (no heartbeat, no message) reads inactive exactly like a zombie whose process
  378. died. Ownership is the durable answer: the job's in-flight attempt is held by a
  379. live process and this session is that attempt's run. A unified-runtime run names
  380. its execution, so that is exact; a legacy run must have started under the claim,
  381. so an older never-closed run of the same job predates it and stays a zombie.
  382. """
  383. if not inflight or session.get("ended_at") is not None:
  384. return False
  385. if session.get("cron_execution_id"):
  386. return session["cron_execution_id"] == inflight.get("id")
  387. claimed_at = _iso_to_epoch(inflight.get("claimed_at"))
  388. try:
  389. started_at = float(session.get("started_at") or 0)
  390. except (TypeError, ValueError):
  391. return False
  392. return claimed_at is not None and started_at >= claimed_at - _OWNERSHIP_CLAIM_GRACE_SECONDS
  393. def cron_run_scheduler_owned(session: dict[str, Any], profile: Optional[str] = None) -> Optional[bool]:
  394. """``scheduler_owned`` for one session row, or None when it is not a cron run session.
  395. The session-detail endpoint stamps this so a client re-checking a run it
  396. already has open (a restored tab, a send) gets the same answer as the runs list.
  397. A unified-runtime run's ``local-*`` id says nothing, so the caller stamps the
  398. row's ``cron_job_id`` / ``cron_execution_id`` (``SessionDB.cron_run_identity``)
  399. first; a legacy ``cron_{job_id}_{timestamp}`` id still names its job.
  400. """
  401. if session.get("source") != "cron":
  402. return None
  403. job_id = session.get("cron_job_id") or legacy_cron_run_job(session.get("id"))
  404. if not job_id:
  405. return None
  406. if session.get("ended_at") is not None:
  407. return False
  408. with _owner_home_scope(profile):
  409. return _run_owned_by(session, _live_inflight_execution(job_id))
  410. def _list_cron_job_runs_sync(job_id: str, profile: Optional[str] = None, limit: int = 20):
  411. """Run history for a cron job, newest first: agent sessions PLUS script-only fires.
  412. Agent runs are ordinary ``source='cron'`` sessions: a legacy
  413. ``cron_{job_id}_{timestamp}`` id, or a unified-runtime ``local-*`` id bound to
  414. this job by its scheduler admission. Backed by ``SessionDB.list_cron_job_runs``
  415. — bounded index range scans, so cost scales with this job's runs, not total
  416. cron history. Script-only (no_agent) jobs never write sessions; their
  417. per-fire output docs (``_list_cron_output_runs``) fill the gaps. The two
  418. representations are reconciled PER EXECUTION, not by branch: a job can
  419. switch modes without changing its id (update_job supports no_agent on an
  420. existing record), so a surviving historical agent session must not hide
  421. newer script-only fires and a recent session must not hide older script
  422. output. Rows carrying the same execution (an agent fire writes BOTH a
  423. session and an output doc) are de-duplicated by execution id, or by
  424. timestamp window for a legacy session that names none; script-only rows
  425. that match no session are added in. All owner-scoped reads (SessionDB,
  426. output docs, execution ledger, filename timezone) run inside the owner's
  427. profile scope so the dashboard's own zone/store is never borrowed
  428. cross-profile.
  429. """
  430. selected = _job_owner_profile(job_id, profile)
  431. # job_id may be a human name; resolve to the canonical id used in run-session ids.
  432. canonical = job_id
  433. job = None
  434. if selected:
  435. job = _call_cron_for_profile(selected, "get_job", job_id)
  436. if job and job.get("id"):
  437. canonical = str(job["id"])
  438. try:
  439. limit_n = max(1, min(int(limit), 100))
  440. except (TypeError, ValueError):
  441. limit_n = 20
  442. with _owner_home_scope(selected):
  443. db = _open_session_db_for_profile(selected, read_only=True)
  444. try:
  445. session_runs = db.list_cron_job_runs(canonical, limit=limit_n, offset=0)
  446. finally:
  447. db.close()
  448. now = time.time()
  449. inflight = _live_inflight_execution(canonical)
  450. for s in session_runs:
  451. s["is_active"] = s.get("ended_at") is None and (now - s.get("last_active", s.get("started_at", 0))) < 300
  452. s["scheduler_owned"] = _run_owned_by(s, inflight)
  453. s["archived"] = bool(s.get("archived"))
  454. if selected:
  455. s["profile"] = selected
  456. docs, attempts = _list_cron_output_runs(job, canonical, selected, limit_n)
  457. return {"runs": _reconcile_cron_runs(session_runs, docs, attempts, canonical, limit_n), "limit": limit_n}
  458. # A doc within this of a session's span can be that session's run's doc (see _doc_matches_session).
  459. _SESSION_DOC_MATCH_SECONDS = 300.0
  460. def _reconcile_cron_runs(
  461. session_runs: list[dict[str, Any]],
  462. docs: list[dict[str, Any]],
  463. attempts: list[dict[str, Any]],
  464. canonical_job_id: str,
  465. limit: int,
  466. ) -> list[dict[str, Any]]:
  467. """One row per execution, newest first: each terminal ledger attempt is its session, else the
  468. output doc it wrote, else a metadata row from the ledger.
  469. An agent fire writes BOTH a session and an output doc; sessions carry the richer
  470. record (tokens, message counts, chat navigation), so the session wins and is never
  471. dropped. A unified-runtime session names its execution, so that pairing is exact.
  472. A doc only has a window guess (``_execution_contains``), so docs pair with attempts
  473. only once every attempt's session is known (``_doc_writer``), one doc per attempt:
  474. the doc folds into its writer's session, or into a legacy session (which names no
  475. execution) by time, and otherwise is its writer's row with the ledger status. A doc
  476. no attempt claims is matched against every session by time, and kept if none matches.
  477. """
  478. def matches(doc_ts: float, sessions: list[dict[str, Any]]) -> bool:
  479. return any(_doc_matches_session(doc_ts, s, _SESSION_DOC_MATCH_SECONDS) for s in sessions)
  480. by_execution = {s["cron_execution_id"]: s for s in session_runs if s.get("cron_execution_id")}
  481. legacy = [s for s in session_runs if not s.get("cron_execution_id")]
  482. terminal = [(index, a) for index, a in enumerate(attempts) if a["status"] in _OUTCOME_STATES]
  483. represented = {index for index, a in terminal if a["id"] in by_execution}
  484. paired: set = set()
  485. merged = _with_cancel_reasons(session_runs, terminal)
  486. for doc in docs:
  487. doc_ts = float(doc.get("started_at") or 0)
  488. candidates = [(index, a) for index, a in terminal
  489. if index not in paired and _execution_contains(a, doc_ts)]
  490. if not candidates:
  491. if not matches(doc_ts, session_runs):
  492. merged.append(doc)
  493. continue
  494. index, attempt = min(candidates, key=lambda c: _doc_writer(doc_ts, c[1], by_execution.get(c[1]["id"])))
  495. paired.add(index)
  496. represented.add(index)
  497. if attempt["id"] in by_execution or matches(doc_ts, legacy):
  498. continue
  499. merged.append({**doc, "title": _execution_status_title(attempt["status"], attempt["error"], doc["title"])})
  500. # Attempts with neither a session nor a surviving doc (pruned, or the run never wrote one,
  501. # e.g. a failed fire) are still real executions: each keeps its own ledger row instead of
  502. # letting a neighbour's status bleed onto its doc. Only a legacy session can claim one by time.
  503. for index, attempt in terminal:
  504. if index in represented:
  505. continue
  506. when = attempt["finished_at"] or attempt["claimed_at"]
  507. if matches(when, legacy):
  508. continue
  509. error = attempt["error"]
  510. merged.append({
  511. "id": f"cron_output:{canonical_job_id}:exec:{index}",
  512. **_cron_output_run_row(
  513. when, _execution_status_title(attempt["status"], error, "Script-only run"), error or None),
  514. })
  515. merged.sort(key=lambda r: float(r.get("started_at") or 0), reverse=True)
  516. return merged[:limit]
  517. def _with_cancel_reasons(
  518. session_runs: list[dict[str, Any]], terminal: list[tuple[int, dict[str, Any]]],
  519. ) -> list[dict[str, Any]]:
  520. """Session rows, with a stopped agent run's title naming the stop and its reason.
  521. The session stands in for its ledger attempt (richer record), so the attempt's own row is
  522. dropped; without this, a cancelled agent run would read like any finished one.
  523. """
  524. cancelled = {a["id"]: a for _, a in terminal if a["status"] == "cancelled"}
  525. return [
  526. {**s, "title": _execution_status_title(
  527. "cancelled", cancelled[s["cron_execution_id"]]["error"], s.get("title") or "")}
  528. if s.get("cron_execution_id") in cancelled else s
  529. for s in session_runs
  530. ]
  531. def _doc_writer(doc_ts: float, attempt: dict[str, Any], session: Optional[dict[str, Any]]) -> tuple:
  532. """Sort key over a doc's candidate attempts, likeliest writer first.
  533. ``save_job_output`` runs before ``finish_execution``, so an attempt that closed before
  534. the doc's second only reaches it through the grace. Among the rest, an attempt whose
  535. own session spans the doc beats an overlapping one that has none (a manual run claimed
  536. while a tick's run was still going); then the newest claim wins.
  537. """
  538. finish = attempt["finished_at"]
  539. return (
  540. finish is not None and doc_ts > finish,
  541. session is None or not _doc_matches_session(doc_ts, session, _SESSION_DOC_MATCH_SECONDS),
  542. -attempt["claimed_at"],
  543. )
  544. def _doc_matches_session(doc_ts: float, session: dict[str, Any], grace_seconds: float) -> bool:
  545. """Whether an output doc belongs to a session's run.
  546. The doc is written when the run FINISHES, so its filename timestamp sits
  547. at the run's end — compare against the session's full span, not its start:
  548. ``[started_at - grace, last_active + grace]`` (``last_active`` covers the
  549. still-open case where ``ended_at`` is None).
  550. """
  551. start = float(session.get("started_at") or 0)
  552. end = float(session.get("ended_at") or session.get("last_active") or start)
  553. return start - grace_seconds <= doc_ts <= end + grace_seconds
  554. _EXECUTION_FIELDS = {"prompt", "skill", "skills", "script", "no_agent"}
  555. def _update_cron_job_sync(job_id: str, body: CronJobUpdate, profile: Optional[str] = None):
  556. selected = _job_profile(job_id, profile)
  557. try:
  558. profile_name, profile_home = _cron_profile_home(selected)
  559. existing = _found(_call_cron_for_profile(profile_name, "get_job", job_id))
  560. updates = _normalize_dashboard_cron_updates(body.updates, profile_home)
  561. if "context_from" in updates:
  562. _validate_dashboard_cron_context_from(updates.get("context_from"), profile_name)
  563. if _EXECUTION_FIELDS.intersection(updates):
  564. effective = {**existing, **updates}
  565. if "skills" in updates and "skill" not in updates:
  566. effective["skill"] = None
  567. _validate_dashboard_cron_effective_job(effective)
  568. job = _mutate_cron_for_profile(profile_name, "update_job", job_id, updates)
  569. except HTTPException:
  570. raise
  571. except ValueError as exc:
  572. raise HTTPException(status_code=400, detail=str(exc)) from exc
  573. return _found(job)
  574. def _pause_cron_job_sync(job_id: str, profile: Optional[str] = None):
  575. return _found(_mutate_cron_for_profile(_job_profile(job_id, profile), "pause_job", job_id))
  576. def _resume_cron_job_sync(job_id: str, profile: Optional[str] = None):
  577. return _found(_mutate_cron_for_profile(_job_profile(job_id, profile), "resume_job", job_id))
  578. def _trigger_cron_job_sync(job_id: str, profile: Optional[str] = None):
  579. selected = _job_profile(job_id, profile)
  580. job = _found(_call_cron_for_profile(selected, "resolve_job_ref", job_id))
  581. # Never expose the job as due before claiming it: the built-in ticker and
  582. # external/manual fire paths share one durable claim, so only one executes
  583. # this run even racing across processes. Active jobs keep the legacy call
  584. # shape; paused jobs need the explicit force flag to resume + claim atomically.
  585. force = not job.get("enabled", True) or job.get("state") == "paused"
  586. ran = _fire_cron_job_for_profile(selected, job["id"], force=force)
  587. refreshed = _call_cron_for_profile(selected, "get_job", job["id"])
  588. if refreshed and refreshed.get("last_run_at") != job.get("last_run_at"):
  589. return refreshed
  590. if not ran:
  591. raise HTTPException(status_code=409, detail="Job is already running or was claimed by another scheduler")
  592. if refreshed:
  593. return refreshed
  594. # A one-shot may remove itself after exhausting repeat=1: keep the response
  595. # shape without inventing an outcome the store no longer holds; the list
  596. # refresh removes the completed row.
  597. return {**job, "enabled": False, "state": "completed"}
  598. def _cancel_cron_job_sync(job_id: str, profile: Optional[str] = None):
  599. """Stop the job's live run; ``cancelled: false`` when nothing is in flight. Schedule untouched."""
  600. selected = _job_profile(job_id, profile)
  601. job = _found(_call_cron_for_profile(selected, "resolve_job_ref", job_id))
  602. # The reason lands in the run history row, so it names the surface that asked.
  603. result = _call_cron_for_profile(selected, "cancel_job_run", job["id"], reason="cancelled on request (Stop)")
  604. return {**result, "job_id": job["id"]}
  605. def _delete_cron_job_sync(job_id: str, profile: Optional[str] = None):
  606. selected = _job_profile(job_id, profile)
  607. try:
  608. removed = _mutate_cron_for_profile(selected, "remove_job", job_id)
  609. except ValueError as exc:
  610. raise HTTPException(status_code=400, detail=str(exc)) from exc
  611. if not removed:
  612. raise _job_not_found()
  613. return {"ok": True}
  614. # Retry-After (seconds) on retryable cron-fire 503s: sized to clear a
  615. # scale-to-zero wake or gateway restart so a scheduler that honors it spaces its
  616. # next attempt past the outage instead of burning its retry budget in it.
  617. _CRON_FIRE_RETRY_AFTER_SECONDS = 60
  618. @router.get("/api/cron/jobs")
  619. async def list_cron_jobs(profile: str = "all"):
  620. return await _run_cron_dashboard_io(_list_cron_jobs_sync, profile)
  621. @router.get("/api/cron/jobs/{job_id}")
  622. async def get_cron_job(job_id: str, profile: Optional[str] = None):
  623. return await _run_cron_dashboard_io(_get_cron_job_sync, job_id, profile)
  624. @router.get("/api/cron/jobs/{job_id}/runs")
  625. async def list_cron_job_runs(job_id: str, profile: Optional[str] = None, limit: int = 20):
  626. return await _run_cron_dashboard_io(_list_cron_job_runs_sync, job_id, profile, limit)
  627. @router.post("/api/cron/jobs")
  628. async def create_cron_job(body: CronJobCreate, profile: Optional[str] = None):
  629. return await _run_cron_dashboard_io(_create_cron_job_sync, body, profile)
  630. @router.get("/api/cron/delivery-targets")
  631. async def get_cron_delivery_targets(profile: Optional[str] = None):
  632. """Delivery targets for the cron dropdown: implicit ``local`` plus the
  633. configured gateway platforms (a platform without a cron home channel is
  634. still listed with ``home_target_set: false`` so the UI can say so).
  635. ``cron_delivery_targets()`` reads each platform's home channel through
  636. ``get_secret``, which fails closed once this process hosts more than one
  637. profile home (the dashboard/desktop ``serve`` backend flips multi-profile
  638. hosting on the first ``?profile=`` request). The read must therefore run
  639. inside the profile scope, exactly like the sibling cron routes — otherwise
  640. the poll raises ``UnscopedSecretError`` on every tick and the dropdown
  641. silently loses every configured platform."""
  642. targets = [{"id": "local", "name": "Local (save only)", "home_target_set": True, "home_env_var": None}]
  643. try:
  644. from cron.scheduler_delivery import cron_delivery_targets
  645. with _config_profile_scope(profile):
  646. targets.extend(cron_delivery_targets())
  647. except HTTPException:
  648. raise # an unknown ?profile= is the scope's 404, not a missing platform list
  649. except Exception:
  650. _log.exception("GET /api/cron/delivery-targets failed")
  651. return {"targets": targets}
  652. @router.put("/api/cron/jobs/{job_id}")
  653. async def update_cron_job(job_id: str, body: CronJobUpdate, profile: Optional[str] = None):
  654. return await _run_cron_dashboard_io(_update_cron_job_sync, job_id, body, profile)
  655. @router.post("/api/cron/jobs/{job_id}/pause")
  656. async def pause_cron_job(job_id: str, profile: Optional[str] = None):
  657. return await _run_cron_dashboard_io(_pause_cron_job_sync, job_id, profile)
  658. @router.post("/api/cron/jobs/{job_id}/resume")
  659. async def resume_cron_job(job_id: str, profile: Optional[str] = None):
  660. return await _run_cron_dashboard_io(_resume_cron_job_sync, job_id, profile)
  661. @router.post("/api/cron/jobs/{job_id}/trigger")
  662. async def trigger_cron_job(job_id: str, profile: Optional[str] = None):
  663. return await _run_cron_dashboard_io(_trigger_cron_job_sync, job_id, profile)
  664. @router.post("/api/cron/jobs/{job_id}/cancel")
  665. async def cancel_cron_job(job_id: str, profile: Optional[str] = None):
  666. return await _run_cron_dashboard_io(_cancel_cron_job_sync, job_id, profile)
  667. @router.delete("/api/cron/jobs/{job_id}")
  668. async def delete_cron_job(job_id: str, profile: Optional[str] = None):
  669. return await _run_cron_dashboard_io(_delete_cron_job_sync, job_id, profile)
  670. @router.post("/api/cron/fire")
  671. async def cron_fire_webhook(request: Request):
  672. """Chronos managed-cron fire webhook (NAS -> agent) — gateway forwarder.
  673. Gated by the NAS-minted JWT (path is in ``PUBLIC_API_PATHS``), not the
  674. dashboard cookie. Execution belongs to the GATEWAY process (it owns the live
  675. platform adapters relay-fronted and E2EE targets need), so the fire is
  676. forwarded to the gateway api_server's own ``/api/cron/fire`` on loopback
  677. and its response passed through (the gateway re-verifies the JWT). Gateway
  678. unreachable -> 503 so NAS retries; deliberately NO local-execution fallback.
  679. """
  680. from plugins.cron_providers.chronos.verify import get_fire_verifier
  681. auth = request.headers.get("Authorization", "")
  682. token = auth[7:].strip() if auth.startswith("Bearer ") else ""
  683. cfg = await asyncio.to_thread(load_config)
  684. claims = get_fire_verifier()(
  685. token=token,
  686. expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""),
  687. jwks_or_key=cfg_get(cfg, "cron", "chronos", "nas_jwks_url", default="") or None,
  688. issuer=cfg_get(cfg, "cron", "chronos", "portal_url", default="") or None,
  689. )
  690. if claims is None:
  691. return JSONResponse({"error": "invalid fire token"}, status_code=401)
  692. try:
  693. body = await request.json()
  694. except Exception:
  695. body = {}
  696. job_id = (body or {}).get("job_id") if isinstance(body, dict) else None
  697. if not job_id:
  698. return JSONResponse({"error": "missing job_id"}, status_code=400)
  699. # Walks every profile's job list (file I/O) — off the event loop.
  700. profile = await _run_cron_dashboard_io(_find_cron_job_profile, job_id)
  701. if not profile: # job is gone (cancelled / completed): 200 so NAS does not retry
  702. return JSONResponse({"status": "gone", "job_id": job_id}, status_code=200)
  703. forwarded = await _forward_cron_fire_to_gateway(profile, job_id, auth)
  704. if forwarded is None:
  705. # Stamp the miss on the job record (last_fire_error) so the dead hop is
  706. # visible in `cronjob list` / the dashboard. Best-effort: visibility
  707. # must never break the retry contract below.
  708. try:
  709. await _run_cron_dashboard_io(
  710. _call_cron_for_profile,
  711. profile,
  712. "note_fire_forward_failure",
  713. job_id,
  714. "scheduled fire could not be forwarded to the gateway "
  715. "api_server (127.0.0.1 loopback unreachable); the gateway "
  716. "process may be down or its api_server adapter not bound "
  717. "(missing API_SERVER_KEY)",
  718. )
  719. except Exception:
  720. _log.debug("could not stamp last_fire_error for %s", job_id, exc_info=True)
  721. # Split by operator intent: a deliberately stopped gateway (durable
  722. # desired_state == "stopped") can never be reached by retrying, so drop
  723. # with 200 + a structured log line — the Chronos provider re-arms every
  724. # job on the next gateway start. A transient window (wake, restart,
  725. # crash loop) keeps the retryable 503 with a Retry-After hint.
  726. if await _run_cron_dashboard_io(_gateway_intentionally_stopped, profile):
  727. _log.info(
  728. "cron fire dropped: gateway for profile %r is deliberately "
  729. "stopped (desired_state=stopped); job %s will resume via "
  730. "Chronos reconcile on next gateway start",
  731. profile, job_id,
  732. )
  733. return JSONResponse(
  734. {
  735. "status": "gateway_stopped",
  736. "detail": "gateway deliberately stopped; fire dropped, jobs re-arm on next gateway start",
  737. "job_id": job_id,
  738. "profile": profile,
  739. },
  740. status_code=200,
  741. )
  742. return JSONResponse(
  743. {"error": "gateway unreachable; retry", "job_id": job_id, "profile": profile},
  744. status_code=503,
  745. headers={"Retry-After": str(_CRON_FIRE_RETRY_AFTER_SECONDS)},
  746. )
  747. status_code, gateway_body = forwarded
  748. if isinstance(gateway_body, dict):
  749. gateway_body.setdefault("job_id", job_id)
  750. # The gateway's own 503s (draining, admission failure) are equally transient.
  751. headers = {"Retry-After": str(_CRON_FIRE_RETRY_AFTER_SECONDS)} if status_code == 503 else None
  752. return JSONResponse(gateway_body, status_code=status_code, headers=headers)
  753. def _blueprint_catalog_sync(profile: Optional[str]) -> list:
  754. """Built-in + ``profile``'s plugin blueprints, with the ``deliver`` slot's options rewritten
  755. from that profile's configured gateway platforms. Plugin discovery can import plugin code, so
  756. this runs off the event loop, inside the profile scope that picks the plugin manager."""
  757. from cron.blueprint_catalog import blueprint_catalog_entry, list_blueprints
  758. with _config_profile_scope(profile): # an unknown ?profile= raises the scope's 404
  759. catalog = list_blueprints()
  760. deliver_options = None
  761. try:
  762. from cron.scheduler_delivery import cron_delivery_targets
  763. platforms = [t["id"] for t in cron_delivery_targets() if t.get("id")]
  764. deliver_options = ["origin", "local", *platforms]
  765. except Exception: # health: allow BLE001 -- gateway targets are optional; static options are the fallback
  766. _log.debug("cron_delivery_targets unavailable; using static deliver options", exc_info=True)
  767. entries = []
  768. for r in catalog:
  769. entry = blueprint_catalog_entry(r)
  770. if deliver_options:
  771. for f in entry.get("fields", []):
  772. if f.get("name") == "deliver":
  773. f["options"] = deliver_options
  774. entries.append(entry)
  775. return entries
  776. @router.get("/api/cron/blueprints")
  777. async def list_cron_blueprints(profile: Optional[str] = None):
  778. """Blueprint catalog (built-ins + the profile's plugin blueprints) as form schemas."""
  779. try:
  780. return {"blueprints": await _run_cron_dashboard_io(_blueprint_catalog_sync, profile)}
  781. except HTTPException:
  782. raise
  783. except Exception as e:
  784. _log.exception("GET /api/cron/blueprints failed")
  785. raise HTTPException(status_code=500, detail=str(e))
  786. def _blueprint_for_profile(key: str, profile: Optional[str]):
  787. from cron.blueprint_catalog import get_blueprint
  788. with _config_profile_scope(profile):
  789. return get_blueprint(key)
  790. @router.post("/api/cron/blueprints/instantiate")
  791. async def instantiate_blueprint(body: AutomationBlueprintInstantiate, profile: str = "default"):
  792. """Fill a blueprint's slots and create the cron job (form-submit path)."""
  793. try:
  794. from cron.blueprint_catalog import BlueprintFillError, fill_blueprint
  795. blueprint = await _run_cron_dashboard_io(_blueprint_for_profile, body.blueprint, profile)
  796. if blueprint is None:
  797. raise HTTPException(status_code=404, detail=f"Unknown blueprint: {body.blueprint}")
  798. try:
  799. spec = fill_blueprint(blueprint, body.values)
  800. except BlueprintFillError as exc: # field-level error — 422 so the form shows it inline
  801. raise HTTPException(status_code=422, detail=str(exc)) from exc
  802. # Blueprint jobs deliver to the dashboard's configured target by default;
  803. # the form's deliver slot overrides via spec["deliver"].
  804. spec.pop("origin", None)
  805. # Off-loop like the siblings; partial keeps **spec keys from colliding
  806. # with the wrapper's own parameters.
  807. _create = functools.partial(_call_cron_for_profile, profile, "create_job", **spec)
  808. created = await _run_cron_dashboard_io(_create)
  809. # Reconcile the profile-scoped provider (file I/O + NAS calls) off-loop.
  810. await _run_cron_dashboard_io(_notify_cron_provider_for_profile, profile)
  811. return created
  812. except HTTPException:
  813. raise
  814. except Exception as e:
  815. _raise_if_cron_registration_error(e)
  816. _log.exception("POST /api/cron/blueprints/instantiate failed")
  817. raise HTTPException(status_code=400, detail=str(e))