scheduler_preflight.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. """Cron pre-run preflight: transient provider-resolution error classification, provider-key /
  2. delivery-target / skills checks, and the shared-route adapter view used by satellite profiles.
  3. Split out of ``cron.scheduler``. Import names from this module directly (``cron.scheduler`` only
  4. imports the few it calls itself). Origin-resident helpers and sibling split modules are reached
  5. late-bound (``_sched`` / module refs at the bottom) so monkeypatching the defining module works.
  6. """
  7. from __future__ import annotations
  8. import errno
  9. import json
  10. import logging
  11. import os
  12. from typing import Optional
  13. from cron.env_settings import cron_env_setting
  14. # Log-record parity with the origin module.
  15. logger = logging.getLogger("cron.scheduler")
  16. # Error-string prefixes from ``run_job``; ``run_one_job`` keys off them for last_status and the
  17. # alert-once dedup. ``:silent`` = already alerted on a previous tick — do not deliver again.
  18. BLOCKED_CONFIG_MARKER = "[blocked_config]"
  19. BLOCKED_CONFIG_SILENT_MARKER = "[blocked_config:silent]"
  20. _TRANSIENT_NET_EXC_NAMES = frozenset({
  21. "ConnectError", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout", "NetworkError",
  22. "TimeoutException", "ClientConnectorError", "ClientConnectorDNSError", "ServerTimeoutError",
  23. "ClientOSError"})
  24. _DNS_FAILURE_NEEDLES = ("nodename nor servname", "name or service not known")
  25. _TRANSIENT_OSERROR_NEEDLES = _DNS_FAILURE_NEEDLES + (
  26. "temporary failure in name resolution", "network is unreachable")
  27. _TRANSIENT_HTTP_NEEDLES = _TRANSIENT_OSERROR_NEEDLES + (
  28. "failed to resolve", "connection refused", "timed out", "timeout")
  29. _TRANSIENT_ERRNOS = frozenset({
  30. errno.ECONNREFUSED, errno.ECONNRESET, errno.EHOSTUNREACH, errno.ENETUNREACH, errno.ENETDOWN,
  31. errno.ETIMEDOUT, errno.EAGAIN})
  32. def _is_transient_provider_resolve_error(exc: BaseException) -> bool:
  33. """True when primary provider resolution failed for a transient network reason (DNS blip,
  34. ConnectError...). Must be eligible for ``fallback_providers`` like AuthError, else a healthy
  35. fallback rung is never tried and the job dies before the first model call."""
  36. import socket
  37. # gaierror carries EAI_* codes, plain OSError carries errno — never mix the namespaces (raw
  38. # literals like {8, 7, 11} are macOS-only and wrong on Linux).
  39. eai_transient = {
  40. getattr(socket, n) for n in ("EAI_NONAME", "EAI_AGAIN", "EAI_FAIL", "EAI_NODATA")
  41. if hasattr(socket, n)
  42. }
  43. # Walk the cause chain; the scheduler wraps raw transport errors.
  44. seen: set[int] = set()
  45. cur: Optional[BaseException] = exc
  46. while cur is not None and id(cur) not in seen:
  47. seen.add(id(cur))
  48. module = type(cur).__module__ or ""
  49. msg = str(cur).lower()
  50. if type(cur).__name__ in _TRANSIENT_NET_EXC_NAMES:
  51. return True
  52. if any(m in module for m in ("httpx", "httpcore", "aiohttp")) and any(
  53. needle in msg for needle in _TRANSIENT_HTTP_NEEDLES):
  54. return True
  55. if isinstance(cur, OSError):
  56. if isinstance(cur, socket.gaierror):
  57. if cur.errno in eai_transient:
  58. return True
  59. elif getattr(cur, "errno", None) in _TRANSIENT_ERRNOS:
  60. return True
  61. if any(needle in msg for needle in _TRANSIENT_OSERROR_NEEDLES):
  62. return True
  63. # Bare exceptions that carry the raw DNS text (format_runtime_provider_error).
  64. if any(needle in msg for needle in _DNS_FAILURE_NEEDLES):
  65. return True
  66. cur = cur.__cause__ or cur.__context__
  67. return False
  68. def _cron_preflight_enabled(cfg: dict) -> bool:
  69. """Preflight is ON unless ``cron.preflight`` is literally ``false``."""
  70. cron_cfg = (cfg or {}).get("cron")
  71. return not isinstance(cron_cfg, dict) or cron_cfg.get("preflight", True) is not False
  72. def cron_default_provider(cfg: dict) -> str:
  73. """``cron.model_provider``: the provider every cron job without its own pin resolves through."""
  74. cron = cfg.get("cron") if isinstance(cfg, dict) else None
  75. return str(cron.get("model_provider") or "").strip() if isinstance(cron, dict) else ""
  76. def _preflight_check_provider_key(job: dict, cfg: dict) -> Optional[str]:
  77. """READ-ONLY probe: would provider resolution fail for lack of a key? Mirrors run_job's
  78. requested-provider computation. Skipped when the job has a fallback chain — auth-fallback may
  79. legitimately rescue a missing primary key. A pinned job has none (``_job_fallback_chain``), so
  80. its missing key blocks even when the global chain is configured."""
  81. try:
  82. if _sched._job_fallback_chain(job, cfg):
  83. return None
  84. except Exception:
  85. return None # fail-open: never block on a preflight-internal error
  86. requested = job.get("provider") or cron_default_provider(cfg) or None
  87. model = job.get("model") or cron_env_setting("HERMES_MODEL") or ""
  88. from hermes_cli.auth import AuthError, is_rate_limited_auth_error
  89. try:
  90. from hermes_cli.runtime_provider import resolve_runtime_provider
  91. kwargs = {"requested": requested, "target_model": model}
  92. if job.get("base_url"):
  93. kwargs["explicit_base_url"] = job.get("base_url")
  94. resolve_runtime_provider(**kwargs)
  95. except AuthError as exc:
  96. if is_rate_limited_auth_error(exc):
  97. # Quota/rate-limit is not a missing credential: let the real path report it and hold
  98. # the job through the provider's window (cron/quota_hold.py, #89376).
  99. return None
  100. return (
  101. f"provider credential missing: {exc} {_credential_store_scope_label()}. "
  102. "Set the provider API key in .env (or `hermes setup`) for that home, or pin a "
  103. "working provider via `hermes cron edit "
  104. f"{job.get('id')} --provider <p>`."
  105. )
  106. except Exception:
  107. return None # non-auth errors are not a missing-credential verdict; real path reports them
  108. return None
  109. def _credential_store_scope_label() -> str:
  110. """``[profile '<name>', HERMES_HOME <path>]`` for the home this preflight read credentials from.
  111. The verdict must name the store it judged: a scheduler process whose home differs from the
  112. shell where "the same credential works" (Docker HOME vs HERMES_HOME, a multiplexed satellite
  113. profile, a gateway launched without the shell's env) otherwise reports a bare "No credentials
  114. stored" that cannot be told apart from a real login gap (#116213).
  115. """
  116. from hermes_cli.profiles import get_active_profile_name
  117. from hermes_constants import get_hermes_home
  118. return f"[profile '{get_active_profile_name() or 'default'}', HERMES_HOME {get_hermes_home()}]"
  119. def _primary_profile_routes_for_current_home() -> list:
  120. """Primary gateway ``profile_routes`` targeting the profile being served; ``[]`` if this IS the
  121. primary home. Satellite crons are ticked and delivered by the primary gateway (a satellite
  122. holding its own token is a ``duplicate_credential`` fatal). Reads the primary home's YAML
  123. layers (user file + managed-scope overlay, top-level or nested ``gateway.``) through the same
  124. loader ``load_gateway_config()`` uses, without bridging platform config into this process.
  125. Shared by preflight rescue and delivery-time resolution so they cannot drift.
  126. Under ``gateway.multiplex_profiles`` a satellite profile's cron jobs are ticked by the primary gateway's
  127. in-process ticker (#69377) and delivered through the primary gateway's live adapters — the satellite
  128. home never holds the platform credentials itself (giving it a token of its own is a
  129. ``duplicate_credential`` fatal).
  130. """
  131. try:
  132. from hermes_constants import get_default_hermes_root, get_hermes_home
  133. primary_home = get_default_hermes_root()
  134. current_home = _sched.Path(get_hermes_home())
  135. if (
  136. primary_home.expanduser().resolve(strict=False)
  137. == current_home.expanduser().resolve(strict=False)
  138. ):
  139. return [] # this IS the primary home — nothing to consult
  140. # Same layers the primary gateway's own loader reads: routes pinned in the managed scope
  141. # (/etc/hermes/config.yaml) never reached the raw user-file read (#121212), so preflight
  142. # false-blocked and routed delivery failed closed on centrally-managed installs.
  143. from gateway.config_loader import read_yaml_layers
  144. layered = read_yaml_layers(primary_home.expanduser())
  145. routes_raw = layered.get("profile_routes")
  146. if routes_raw is None and isinstance(layered.get("gateway"), dict):
  147. routes_raw = layered["gateway"].get("profile_routes")
  148. if not isinstance(routes_raw, list):
  149. return []
  150. from gateway.profile_routing import parse_profile_routes
  151. from hermes_cli.profiles import profile_matches_home
  152. return [
  153. route for route in parse_profile_routes(routes_raw)
  154. if route.enabled and profile_matches_home(route.profile)
  155. ]
  156. except Exception:
  157. logger.debug("primary-gateway profile-route lookup unavailable", exc_info=True)
  158. return []
  159. def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool:
  160. """True when the primary gateway routes this platform to the profile being served.
  161. scheduler is currently serving (preflight rescue, #97476).
  162. """
  163. platform_key = platform_name.lower()
  164. return any(
  165. str(route.platform).lower() == platform_key
  166. for route in _primary_profile_routes_for_current_home()
  167. )
  168. class SharedRouteAdapters:
  169. """Read-only adapter map for a credentialless satellite profile. ``get(platform, target)``
  170. resolves the PRIMARY adapter iff the inbound route matcher (``ProfileRoute.matches``) accepts
  171. the target; anything else (unmatched target, disabled route, other profile, or target-less
  172. ``get(platform)``) is a miss — fail closed, never the default bot.
  173. See #101113.
  174. """
  175. def __init__(self, primary_adapters, routes) -> None:
  176. self._primary = dict(primary_adapters or {})
  177. self._routes = list(routes or [])
  178. def __bool__(self) -> bool:
  179. return bool(self._primary) and bool(self._routes)
  180. def get(self, platform, target=None, default=None):
  181. if not target:
  182. return default
  183. adapter = self._primary.get(platform)
  184. if adapter is None:
  185. return default
  186. platform_key = str(getattr(platform, "value", platform)).lower()
  187. chat_id = str(target.get("chat_id") or "") or None
  188. thread_id = target.get("thread_id")
  189. thread_id = str(thread_id) if thread_id else None
  190. # A cron target carries no inbound guild anchor, so a route's guild_id is matched against
  191. # itself — the target-exact discriminators (chat_id/thread_id) authorize the send. Without
  192. # this the documented ``guild_id + chat_id`` Discord route never authorized cron output.
  193. for route in self._routes:
  194. if str(route.platform).lower() != platform_key:
  195. continue
  196. if not (route.chat_id or route.thread_id):
  197. continue # guild-only routes are not target-exact
  198. if route.matches(
  199. str(route.platform), guild_id=route.guild_id, chat_id=chat_id, thread_id=thread_id,
  200. ):
  201. return adapter
  202. return default
  203. def has_credential_platform(adapters, owned=None) -> bool:
  204. """True when the profile has a bot of its own. Any local session (Desktop chat, CLI run, Kanban
  205. worker) adds a ``LOCAL`` adapter, which holds no platform credential: a profile with only that is
  206. still a credentialless satellite that may ride routed primary adapters. ``owned`` is the
  207. profile's ``_profile_bot_platforms`` entry: a bot that dropped leaves the live map while it
  208. disconnects and reconnects, and borrowing the primary meanwhile would send through the wrong bot."""
  209. return bool(owned) or any(
  210. getattr(platform, "value", platform) != "local" for platform in adapters or ())
  211. def _preflight_check_delivery(job: dict) -> Optional[str]:
  212. """Check delivery targets resolve to configured platforms. ``local``/``origin``/``all`` are
  213. never checked (no gateway-config load). Unknown platform always blocks; known platform blocks
  214. only if the gateway config loads AND reports it unconnected; config load failures fail OPEN.
  215. ``failure_deliver`` gets the same rules — a typo'd failure platform would otherwise only
  216. surface when a failure occurs (NS-788)."""
  217. deliver_value = _delivery._normalize_deliver_value(job.get("deliver", "local"))
  218. failure_deliver_value = _delivery._normalize_deliver_value(
  219. _delivery._delivery_lane_value(job, for_failure=True))
  220. lane_values = [deliver_value]
  221. if failure_deliver_value != deliver_value:
  222. lane_values.append(failure_deliver_value)
  223. platform_parts: list[str] = []
  224. for lane_value in lane_values:
  225. for part in lane_value.split(","):
  226. part = part.strip()
  227. if not part or part.lower() in {"local", "origin", "all"}:
  228. continue
  229. # bot-chat targets deliver via a local subprocess; failures land in last_delivery_error.
  230. if _delivery.parse_bot_chat_deliver_token(part) is not None:
  231. continue
  232. if _delivery.parse_room_deliver_token(part) is not None:
  233. continue # Group Chat posts write the room store directly; no adapter needed.
  234. platform_parts.append(part.split(":", 1)[0].strip())
  235. if not platform_parts:
  236. return None
  237. connected: Optional[set] = None
  238. for platform_name in platform_parts:
  239. if not _delivery._is_known_delivery_platform(platform_name):
  240. return (
  241. f"delivery platform '{platform_name}' is not a known cron "
  242. "delivery target. Fix the job's `deliver` value or configure "
  243. "the platform's gateway credentials."
  244. )
  245. if connected is None:
  246. try:
  247. from gateway.config import load_gateway_config
  248. gateway_config = load_gateway_config()
  249. connected = {p.value for p in gateway_config.get_connected_platforms()}
  250. connected |= _delivery._relay_fronted_delivery_platforms(connected)
  251. except Exception:
  252. logger.debug(
  253. "preflight: gateway config unavailable — skipping "
  254. "delivery credential check", exc_info=True)
  255. return None # fail-open
  256. # Multiplex: a satellite served by the primary's adapters reads unconnected — no block.
  257. if (
  258. platform_name.lower() not in connected
  259. # Multiplex escape hatch: a satellite profile whose deliveries are routed by the primary
  260. # gateway's profile_routes is served by the primary's adapters, so its own unconnected reading
  261. # is a false block (#97476).
  262. and not _delivery_platform_routed_from_primary_gateway(platform_name)
  263. ):
  264. return (
  265. f"delivery platform '{platform_name}' has no gateway "
  266. "credentials configured (not connected). Configure it via "
  267. "`hermes setup` or change the job's `deliver` target."
  268. )
  269. return None
  270. # ``skill_view`` payload keys naming missing prerequisites -> label for the preflight verdict.
  271. _SKILL_MISSING_FIELDS = (
  272. ("missing_required_environment_variables", "env ${}"),
  273. ("missing_required_commands", "command '{}'"),
  274. ("missing_credential_files", "credential file {}"))
  275. def _preflight_check_skills(job: dict) -> Optional[str]:
  276. """Block only on an affirmative ``setup_needed`` verdict from ``skill_view``; skills that fail
  277. to load fall through to ``_build_job_prompt``'s skipped-skill handling (fail-open)."""
  278. from cron.scheduler_prompt import _job_skill_names
  279. skill_names = _job_skill_names(job)
  280. if not skill_names:
  281. return None
  282. from tools.skills_tool import skill_view
  283. for skill_name in skill_names:
  284. try:
  285. payload = json.loads(skill_view(skill_name))
  286. except Exception:
  287. continue # unreadable/missing skill → existing skip handling
  288. if not isinstance(payload, dict) or not payload.get("success"):
  289. continue
  290. if payload.get("setup_needed") or payload.get("readiness_status") == "setup_needed":
  291. missing = [
  292. fmt.format(name)
  293. for key, fmt in _SKILL_MISSING_FIELDS
  294. for name in payload.get(key) or []
  295. ]
  296. detail = ", ".join(missing) or "required setup incomplete"
  297. return (
  298. f"attached skill '{skill_name}' is not ready: missing "
  299. f"{detail}. Provide the missing prerequisites or detach the "
  300. "skill from this job."
  301. )
  302. return None
  303. # (job id, server name) pairs already warned about as reconnecting; see _empty_requested_mcp_toolsets.
  304. _RECONNECTING_WARNED: set = set()
  305. def _empty_requested_mcp_toolsets(job: dict, cfg: dict) -> Optional[str]:
  306. """Reason when an MCP server the job's own ``enabled_toolsets`` names resolves to zero tools.
  307. Runs AFTER cron MCP discovery. The server's toolset alias is process-global while its tools
  308. are registered per profile overlay, so under a multiplexer a job can name a server that is
  309. connected for another profile and build a tool-less agent that ``quiet_mode`` never reports.
  310. Only servers the job explicitly asked for count; the implicit enabled-server merge does not.
  311. """
  312. requested = [str(name) for name in (job.get("enabled_toolsets") or [])]
  313. if not requested:
  314. return None
  315. from hermes_cli.tools_config import enabled_mcp_server_names
  316. from toolsets import resolve_toolset
  317. from tools.mcp_tool_discovery import mcp_server_reconnecting
  318. missing = [name for name in requested
  319. if name in enabled_mcp_server_names(cfg) and not resolve_toolset(name)]
  320. # A server that worked in this process and is parked/self-probing after a network blip
  321. # (router reboot, DNS failure) is recovering, not misconfigured: the job runs with the tools
  322. # that did resolve rather than losing a whole tick to a minute of downtime (#112871). Only a
  323. # server that never connected for this profile is judged below.
  324. reconnecting = sorted(name for name in missing if mcp_server_reconnecting(name))
  325. job_id = str(job.get("id", "?"))
  326. # One WARNING per job+server per outage (like the one-shot blocked_config alert), not one per
  327. # tick; the entry drops once the server is back so the next outage warns again.
  328. _RECONNECTING_WARNED.difference_update(
  329. key for key in list(_RECONNECTING_WARNED) if key[0] == job_id and key[1] not in reconnecting)
  330. unwarned = [name for name in reconnecting if (job_id, name) not in _RECONNECTING_WARNED]
  331. if unwarned:
  332. _RECONNECTING_WARNED.update((job_id, name) for name in unwarned)
  333. logger.warning(
  334. "Job '%s': MCP server(s) %s named in enabled_toolsets are reconnecting — running "
  335. "without their tools until they recover (a server parked on a permanent error blocks "
  336. "the job instead)", job_id, ", ".join(unwarned))
  337. if reconnecting:
  338. missing = [name for name in missing if name not in reconnecting]
  339. if not missing:
  340. return None
  341. # The reason is what the operator reads in the gateway log and the alert. It must say the
  342. # block is not sticky: a server whose first connection failed on a network blip is parked and
  343. # self-probed by the MCP layer, and this check re-runs on every dispatch, so the job resumes
  344. # on its own — two operators misread the old text as a config error to repair by hand (#112871).
  345. return (
  346. f"MCP server(s) {', '.join(sorted(missing))} named in this job's enabled_toolsets "
  347. "resolved to zero tools for this profile (never connected for this profile, or connected "
  348. "for another profile only). If the server is only temporarily unreachable this clears by itself — "
  349. "the check re-runs on every dispatch and the job resumes once the server reconnects. "
  350. "If the name is wrong or belongs to another profile, fix the server or remove it from "
  351. "the job's toolsets.")
  352. def _preflight_job_config(job: dict, cfg: dict) -> Optional[str]:
  353. """Pre-dispatch validation: return a reason (missing key, unconfigured delivery, unready skill)
  354. so the caller refuses BEFORE building agent machinery or burning an LLM call. Every check fails
  355. open — preflight blocks only on an affirmative misconfiguration verdict.
  356. Same fail-before-spend spirit as the fail-loud-on-hidden-tools direction in #27948; alert dedup
  357. follows the alert-once pattern from the dead-pin auto-pause (#73506).
  358. """
  359. for name, check in (
  360. ("provider_key", lambda: _preflight_check_provider_key(job, cfg)),
  361. ("skills", lambda: _preflight_check_skills(job)),
  362. ("delivery", lambda: _preflight_check_delivery(job))):
  363. try:
  364. reason = check()
  365. except Exception:
  366. logger.debug("preflight check %s raised — failing open", name, exc_info=True)
  367. continue
  368. if reason:
  369. return reason
  370. return None
  371. # Late-bound origin namespace (see module docstring). Imported LAST so this module is fully
  372. # populated before ``scheduler`` re-exports from it.
  373. from cron import scheduler as _sched
  374. from cron import scheduler_delivery as _delivery