web_server_cron.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. """Dashboard cron helpers: per-profile scheduler I/O, job validation/normalisation, cron fire and
  2. gateway forwarding.
  3. """
  4. import asyncio
  5. import contextlib
  6. import logging
  7. import inspect
  8. import re
  9. from fastapi import HTTPException
  10. from pathlib import Path
  11. from typing import Any, Dict, List, Optional, Tuple
  12. from hermes_cli.config import cfg_get
  13. from hermes_cli.web_models import CronJobCreate
  14. # Same logger the code used before extraction (record parity).
  15. _log = logging.getLogger("hermes_cli.web_server")
  16. def _cron_optional_text(value: Any, *, strip_trailing_slash: bool = False) -> Optional[str]:
  17. if value is None:
  18. return None
  19. text = str(value).strip()
  20. if strip_trailing_slash:
  21. text = text.rstrip("/")
  22. return text or None
  23. def _cron_string_list(value: Any) -> Optional[list[str]]:
  24. if isinstance(value, str):
  25. raw_items = re.split(r"[\n,]", value)
  26. elif isinstance(value, (list, tuple)):
  27. raw_items = value
  28. else:
  29. return None
  30. items = [str(item).strip() for item in raw_items if str(item).strip()]
  31. return items or None
  32. def _normalize_dashboard_cron_script(value: Any, profile_home: Path) -> Optional[str]:
  33. """Validate a dashboard-selected cron script against the profile sandbox."""
  34. text = _cron_optional_text(value)
  35. if not text:
  36. return None
  37. scripts_root = (profile_home / "scripts").resolve()
  38. raw_path = Path(text).expanduser()
  39. candidate = raw_path.resolve() if raw_path.is_absolute() else (scripts_root / raw_path).resolve()
  40. try:
  41. relative = candidate.relative_to(scripts_root)
  42. except ValueError as exc:
  43. raise HTTPException(status_code=400, detail=f"script must be inside {scripts_root}") from exc
  44. if not candidate.exists():
  45. raise HTTPException(status_code=400, detail=f"script does not exist: {candidate}")
  46. if not candidate.is_file():
  47. raise HTTPException(status_code=400, detail=f"script is not a file: {candidate}")
  48. return str(relative)
  49. def _validate_dashboard_cron_effective_job(job: dict[str, Any]) -> None:
  50. prompt = _cron_optional_text(job.get("prompt"))
  51. script = _cron_optional_text(job.get("script"))
  52. skills = _cron_string_list(job.get("skills")) or _cron_string_list(job.get("skill"))
  53. if job.get("no_agent"):
  54. if not script:
  55. raise HTTPException(status_code=400, detail="no_agent=True requires a script")
  56. return
  57. if not (prompt or skills or script):
  58. raise HTTPException(status_code=400, detail="agent cron jobs require a prompt, skill, or script")
  59. def _validate_dashboard_cron_context_from(refs: Optional[list[str]], profile_name: str) -> None:
  60. for ref in refs or ():
  61. # "self" (the continuity toggle) resolves to the job's own id at run time — it can't be
  62. # validated against the store (create precedes the job's existence).
  63. if isinstance(ref, str) and ref.strip().lower() == "self":
  64. continue
  65. if not _call_cron_for_profile(profile_name, "get_job", ref):
  66. raise HTTPException(
  67. status_code=400,
  68. detail=f"context_from job '{ref}' not found in profile '{profile_name}'")
  69. def _cron_profile_dicts() -> list[dict[str, Any]]:
  70. """Minimal profile records (callers only consume ``name``); avoids ``list_profiles()``,
  71. whose config parsing, gateway probes and skill counts are GIL pressure on large pools."""
  72. from hermes_cli.web_server_profiles import _fallback_profile_dicts
  73. from hermes_cli import profiles as profiles_mod
  74. try:
  75. return [
  76. {"name": name, "path": str(home), "is_default": name == "default"}
  77. for name, home in profiles_mod.profiles_to_serve(multiplex=True, include_standalone=True, include_parked=True)]
  78. except Exception:
  79. _log.exception("Failed to list profiles for cron dashboard; falling back to directory scan")
  80. return _fallback_profile_dicts(profiles_mod)
  81. def _cron_default_profile() -> str:
  82. """Profile to target when a cron request carries no explicit ``profile``.
  83. A desktop pool backend runs one process per profile, but these endpoints route storage through
  84. the profiles tree via ``_cron_profile_home`` — a hardcoded "default" would write a non-default
  85. profile's job into ~/.hermes. ``custom`` (HERMES_HOME outside the profiles tree) has no
  86. profile-dir equivalent, so it keeps the legacy "default" fallback.
  87. """
  88. try:
  89. from hermes_cli.profiles import get_active_profile_name
  90. name = get_active_profile_name()
  91. except Exception:
  92. return "default"
  93. return "default" if name in ("default", "custom") else name
  94. def _cron_profile_home(profile: Optional[str]) -> tuple[str, Path]:
  95. """Resolve a profile query value to (profile_name, HERMES_HOME)."""
  96. from hermes_cli import profiles as profiles_mod
  97. raw = (profile or _cron_default_profile()).strip() or "default"
  98. try:
  99. canon = profiles_mod.normalize_profile_name(raw)
  100. profiles_mod.validate_profile_name(canon)
  101. except ValueError as e:
  102. raise HTTPException(status_code=400, detail=str(e))
  103. if not profiles_mod.profile_exists(canon):
  104. raise HTTPException(status_code=404, detail=f"Profile '{canon}' does not exist.")
  105. return canon, profiles_mod.get_profile_dir(canon)
  106. def _annotate_cron_job(
  107. job: dict[str, Any], profile: str, home: Path, heartbeat_age: Optional[float] = None,
  108. ) -> dict[str, Any]:
  109. return {
  110. **job,
  111. "profile": profile,
  112. "profile_name": profile,
  113. "hermes_home": str(home),
  114. "is_default_profile": profile == "default",
  115. # Seconds since this profile's ticker last iterated (None = never/unknown): a
  116. # `next_run_at` parked in the past is only explained by a scheduler that stopped
  117. # ticking, so the dashboard can date it (#114309).
  118. "scheduler_heartbeat_age_s": heartbeat_age}
  119. @contextlib.contextmanager
  120. def _cron_store_scope(home: Path):
  121. """Point HERMES_HOME and the cron.jobs store at one profile's home for the block.
  122. The dashboard is a single process inspecting many profiles; cron.jobs' execution-context
  123. override keeps these calls from retargeting a concurrent desktop ticker's load/save.
  124. """
  125. from cron import jobs as cron_jobs
  126. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  127. token = set_hermes_home_override(str(home))
  128. try:
  129. with cron_jobs.use_cron_store(home):
  130. yield cron_jobs
  131. finally:
  132. reset_hermes_home_override(token)
  133. def _call_cron_for_profile(target_profile: Optional[str], func_name: str, *args, **kwargs):
  134. """Run a cron.jobs helper against the selected profile's cron directory."""
  135. profile_name, home = _cron_profile_home(target_profile)
  136. with _cron_store_scope(home) as cron_jobs:
  137. if func_name == "create_job":
  138. from cron.scheduler import create_job_with_scheduler_registration
  139. result = create_job_with_scheduler_registration(*args, **kwargs)
  140. else:
  141. result = getattr(cron_jobs, func_name)(*args, **kwargs)
  142. heartbeat_age = cron_jobs.get_ticker_heartbeat_age()
  143. if isinstance(result, list):
  144. return [_annotate_cron_job(j, profile_name, home, heartbeat_age) for j in result]
  145. if isinstance(result, dict):
  146. return _annotate_cron_job(result, profile_name, home, heartbeat_age)
  147. return result
  148. def _notify_cron_provider_for_profile(target_profile: Optional[str]) -> None:
  149. """Best-effort provider reconcile against one profile's job store.
  150. Fail-closed for external providers on a multi-profile dashboard: an external ``reconcile``
  151. converges its REMOTE (non-profile-scoped) registry toward one profile's jobs.json and cancels
  152. every remote entry absent from it, so reconciling profile B would disarm profile A's one-shots.
  153. Until the provider contract carries a profile identity, skip unscoped external reconciles; the
  154. affected profile re-arms on its next fire/start (idempotent via dedup_key). The built-in
  155. provider re-reads jobs.json each tick and stays a no-op here.
  156. """
  157. try:
  158. _profile_name, home = _cron_profile_home(target_profile)
  159. from cron.scheduler_provider import InProcessCronScheduler, resolve_cron_scheduler
  160. with _cron_store_scope(home):
  161. provider = resolve_cron_scheduler()
  162. external = not isinstance(provider, InProcessCronScheduler)
  163. if external and sum(1 for p in _cron_profile_dicts() if p.get("name")) > 1:
  164. _log.warning(
  165. "Skipping cron provider reconcile for profile %s: "
  166. "external provider '%s' reconcile is not "
  167. "profile-scoped and would disarm other profiles' "
  168. "armed one-shots. The mutated profile re-arms "
  169. "idempotently on its next fire/start.", target_profile, provider.name,
  170. )
  171. return
  172. provider.on_jobs_changed()
  173. except Exception:
  174. _log.debug("Cron provider reconciliation failed for profile %s", target_profile, exc_info=True)
  175. def _mutate_cron_for_profile(target_profile: Optional[str], func_name: str, *args, **kwargs):
  176. """Apply a cron store mutation and reconcile its scheduler provider."""
  177. result = _call_cron_for_profile(target_profile, func_name, *args, **kwargs)
  178. if result:
  179. _notify_cron_provider_for_profile(target_profile)
  180. return result
  181. def _find_cron_job_profile(job_id: str) -> Optional[str]:
  182. for profile in _cron_profile_dicts():
  183. name = str(profile.get("name") or "")
  184. if not name:
  185. continue
  186. jobs = _call_cron_for_profile(name, "list_jobs", True)
  187. if any(j.get("id") == job_id or j.get("name") == job_id for j in jobs):
  188. return name
  189. return None
  190. async def _run_cron_dashboard_io(func, *args, **kwargs):
  191. """Run cron dashboard profile/job I/O outside the FastAPI event loop."""
  192. from starlette.concurrency import run_in_threadpool
  193. if inspect.iscoroutinefunction(func):
  194. raise TypeError("_run_cron_dashboard_io only accepts sync callables")
  195. result = await run_in_threadpool(func, *args, **kwargs)
  196. if inspect.isawaitable(result):
  197. raise TypeError("_run_cron_dashboard_io sync callable returned an awaitable")
  198. return result
  199. def _raise_if_cron_registration_error(e: Exception) -> None:
  200. """Re-raise a cron partial failure (job saved, external scheduler registration failed) as
  201. HTTP 424 with the structured envelope. Shared by every dashboard cron-create surface."""
  202. from cron.scheduler import CronSchedulerRegistrationError
  203. if isinstance(e, CronSchedulerRegistrationError):
  204. raise HTTPException(status_code=424, detail=e.to_dict()) from e
  205. def _create_cron_job_sync(body: CronJobCreate, profile: Optional[str] = None):
  206. try:
  207. profile_name, profile_home = _cron_profile_home(profile)
  208. script = _normalize_dashboard_cron_script(body.script, profile_home)
  209. skills = _cron_string_list(body.skills)
  210. context_from = _cron_string_list(body.context_from)
  211. _validate_dashboard_cron_context_from(context_from, profile_name)
  212. no_agent = bool(body.no_agent)
  213. # Finite repeat: non-positive counts are a client error (the core chokepoint
  214. # would silently make them unlimited — indistinguishable from omission);
  215. # everything else (int, 'forever'/'once'/'3') coerces through the shared
  216. # normalize_repeat_value, the same validation the CLI uses.
  217. repeat = body.repeat
  218. if repeat is not None:
  219. if isinstance(repeat, int) and not isinstance(repeat, bool) and repeat < 1:
  220. raise HTTPException(
  221. status_code=400, detail="repeat must be a positive integer")
  222. from cron.jobs import normalize_repeat_value
  223. repeat = normalize_repeat_value(repeat)
  224. _validate_dashboard_cron_effective_job(
  225. {"prompt": body.prompt, "skills": skills, "script": script, "no_agent": no_agent})
  226. return _mutate_cron_for_profile(
  227. profile_name,
  228. "create_job",
  229. prompt=body.prompt or "",
  230. schedule=body.schedule,
  231. name=body.name,
  232. repeat=body.repeat,
  233. deliver=_cron_optional_text(body.deliver) or "local",
  234. skills=skills,
  235. model=_cron_optional_text(body.model),
  236. provider=_cron_optional_text(body.provider),
  237. base_url=_cron_optional_text(body.base_url, strip_trailing_slash=True),
  238. script=script,
  239. context_from=context_from,
  240. enabled_toolsets=_cron_string_list(body.enabled_toolsets),
  241. workdir=_cron_optional_text(body.workdir),
  242. no_agent=no_agent,
  243. timezone=_cron_optional_text(body.timezone),
  244. grace_minutes=body.grace_minutes,
  245. **{key: getattr(body, key) for key in ("paused", "paused_reason", "catch_up")
  246. if key in body.model_fields_set})
  247. except HTTPException:
  248. raise
  249. except Exception as e:
  250. _raise_if_cron_registration_error(e)
  251. _log.exception("POST /api/cron/jobs failed")
  252. raise HTTPException(status_code=400, detail=str(e))
  253. def _fire_cron_job_for_profile(profile: str, job_id: str, *, force: bool = False) -> bool:
  254. """Run ONE due cron job for ``profile`` via the scheduler provider's ``fire_due``.
  255. DEPRECATED for NAS webhook fires — superseded by :func:`_forward_cron_fire_to_gateway`, since
  256. fires must run in the GATEWAY process (it owns the live adapters; the standalone path here
  257. cannot serve relay-fronted platforms or E2EE rooms). Retained for the dashboard trigger path
  258. and external callers on the web_deps late-binding seam; do not add new uses.
  259. """
  260. _profile_name, home = _cron_profile_home(profile)
  261. from cron.scheduler_provider import provider_fire_due_accepts, provider_supports_force_fire, resolve_cron_scheduler
  262. with _cron_store_scope(home):
  263. provider = resolve_cron_scheduler()
  264. if force:
  265. if not provider_supports_force_fire(provider):
  266. raise HTTPException(
  267. status_code=409,
  268. detail=(
  269. f"Cron provider '{getattr(provider, 'name', 'custom')}' "
  270. "does not support atomic forced firing of paused jobs"))
  271. return bool(provider.fire_due(job_id, adapters=None, loop=None, force=True))
  272. # Off-tick run-now: never stamp next_run_at as the occurrence (#104790); third-party
  273. # providers without the kwarg keep the legacy call.
  274. if provider_fire_due_accepts(provider, "manual"):
  275. return bool(provider.fire_due(job_id, adapters=None, loop=None, manual=True))
  276. return bool(provider.fire_due(job_id, adapters=None, loop=None))
  277. def _profile_env_value(home: Path, key: str) -> str:
  278. """One value from a profile's .env (``""`` when absent/unreadable)."""
  279. from agent.secret_scope import load_env_file
  280. return load_env_file(home / ".env").get(key, "")
  281. def _gateway_fire_endpoint(profile: str, home: Path) -> str:
  282. """Loopback URL of the gateway api_server's cron-fire route.
  283. Port resolution mirrors gateway/config.py's api_server load order for the LISTENER-OWNER
  284. profile: ``platforms.api_server.extra.port`` in its config.yaml, then ``API_SERVER_PORT``
  285. (process env for the active profile, the profile's own .env otherwise), then 8642. Loopback
  286. is safe: dashboard and gateway share a network namespace in every supported deployment.
  287. In multiplex mode only the DEFAULT profile's api_server is bound and exposes per-profile
  288. mirrors under ``/p/<profile>/…``, so a non-default profile's port must be read from the
  289. default home (a secondary's own API_SERVER_PORT is a port nothing listens on).
  290. """
  291. from hermes_cli.config import load_config
  292. import os as _os
  293. multiplex = False
  294. try:
  295. # The live default gateway's own record, else the explicit flag — never the merged default:
  296. # an unset gateway.multiplex_profiles is settled by the gateway at boot, not by this process.
  297. from hermes_cli.gateway_multiplex_mode import default_gateway_multiplexes
  298. multiplex = default_gateway_multiplexes()
  299. except Exception:
  300. _log.debug("cron fire: multiplex detection failed; assuming single-profile", exc_info=True)
  301. listener_profile, listener_home = profile, home
  302. if multiplex and profile != "default":
  303. from hermes_constants import get_default_hermes_root
  304. listener_profile, listener_home = "default", get_default_hermes_root()
  305. _log.info(
  306. "cron fire: multiplex gateway — resolving api_server port for %s "
  307. "from the default profile's listener (%s)", profile, listener_home,
  308. )
  309. port = 0
  310. try:
  311. # Profile-scoped read through the CANONICAL loader (managed-scope overlay, ${ENV_VAR}
  312. # expansion) — never a raw yaml.safe_load (tests/hermes_cli/test_config_read_guard.py).
  313. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  314. token = set_hermes_home_override(str(listener_home))
  315. try:
  316. profile_cfg = load_config()
  317. finally:
  318. reset_hermes_home_override(token)
  319. raw = cfg_get(profile_cfg, "platforms", "api_server", "extra", "port", default=None)
  320. if raw:
  321. port = int(raw)
  322. except Exception:
  323. port = 0
  324. if not port:
  325. raw = (
  326. _os.getenv("API_SERVER_PORT", "")
  327. if listener_profile == _cron_default_profile()
  328. else _profile_env_value(listener_home, "API_SERVER_PORT"))
  329. try:
  330. port = int(raw) if raw else 0
  331. except ValueError:
  332. port = 0
  333. port = port or 8642
  334. if multiplex and profile != "default":
  335. return f"http://127.0.0.1:{port}/p/{profile}/api/cron/fire"
  336. return f"http://127.0.0.1:{port}/api/cron/fire"
  337. async def _forward_cron_fire_to_gateway(
  338. profile: str, job_id: str, authorization: str) -> Optional[tuple[int, dict[str, Any]]]:
  339. """Forward a Chronos fire callback byte-preserved to the gateway api_server on loopback.
  340. The dashboard is the hosted deployment's only public HTTP door, but cron execution belongs to
  341. the GATEWAY process (live adapters → relay-fronted platforms and E2EE rooms work). Same job_id
  342. and NAS bearer; the gateway re-verifies the JWT itself.
  343. Returns ``(status_code, body)``, or ``None`` when the gateway is unreachable (scale-to-zero
  344. wake, restart, api_server disabled). The caller maps None to 503 so NAS retries (store CAS
  345. de-dupes a double fire) — unless :func:`_gateway_intentionally_stopped`, in which case it
  346. drops the fire with 200: retrying into an operator-stopped gateway can never succeed.
  347. """
  348. _profile_name, home = _cron_profile_home(profile)
  349. url = await asyncio.to_thread(_gateway_fire_endpoint, _profile_name, home)
  350. import httpx
  351. try:
  352. async with httpx.AsyncClient(timeout=10.0) as client:
  353. resp = await client.post(url, json={"job_id": job_id}, headers={"Authorization": authorization})
  354. except Exception as exc:
  355. _log.warning("cron fire forward to %s failed (%s: %s); returning 503 for NAS retry", url, type(exc).__name__, exc)
  356. return None
  357. try:
  358. body = resp.json()
  359. except Exception:
  360. body = {"raw": (resp.text or "")[:500]}
  361. if not isinstance(body, dict):
  362. body = {"raw": body}
  363. return resp.status_code, body
  364. def _gateway_intentionally_stopped(profile: Optional[str]) -> bool:
  365. """True when the profile's gateway is stopped BY OPERATOR INTENT.
  366. Reads the durable ``desired_state`` of gateway_state.json, written only by the s6 lifecycle
  367. commands (``gateway stop`` persists "stopped"; start/restart persist "running") and never
  368. during transient windows (crash loops, drains, wakes) — so it splits "retry will eventually
  369. succeed" from "retry can never succeed". Deliberately does NOT fall back to the volatile
  370. ``gateway_state`` runtime field: a legacy/crashed file must stay on the retryable-503 path.
  371. Any resolution or parse failure returns False (fail open toward retry).
  372. """
  373. import json as _json
  374. try:
  375. data = _json.loads((_cron_profile_home(profile)[1] / "gateway_state.json").read_text(encoding="utf-8-sig"))
  376. return isinstance(data, dict) and data.get("desired_state") == "stopped"
  377. except Exception:
  378. return False