scheduler_script.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601
  1. """Cron pre-run script execution: timeouts, Windows venv bootstrap, process-tree termination,
  2. and the claim-heartbeat thread that keeps a long script's run claim alive.
  3. Split out of ``cron.scheduler``. Import names from this module directly (``cron.scheduler`` only
  4. imports the few it calls itself). Origin-resident helpers and sibling split modules are reached
  5. late-bound (``_sched`` / module refs at the bottom) so monkeypatching the defining module works.
  6. """
  7. from __future__ import annotations
  8. import contextlib
  9. import contextvars
  10. import logging
  11. import os
  12. import shutil
  13. import signal
  14. import stat
  15. import subprocess
  16. import sys
  17. import threading
  18. import time
  19. from cron.env_settings import cron_env_setting
  20. from cron.jobs import _ensure_cron_dir
  21. from pathlib import Path
  22. from typing import Any, Callable, Optional, TYPE_CHECKING
  23. from hermes_cli._subprocess_compat import windows_hide_flags
  24. if TYPE_CHECKING:
  25. from cron.cancellation import _CancelEventLike
  26. # Log-record parity with the origin module.
  27. logger = logging.getLogger("cron.scheduler")
  28. def _positive_int(raw) -> Optional[int]:
  29. """``int(float(raw))`` when > 0, else None; raises on unparsable input."""
  30. timeout = int(float(raw))
  31. return timeout if timeout > 0 else None
  32. def _timeout_from_env_or_config(
  33. env_var: str, config_key: str, parse: Callable[[Any], Any], label: str):
  34. """Shared env → ``cron.<config_key>`` resolution. ``parse`` returns the value or None to keep
  35. looking; a parse error on the env var WARNs, on config DEBUGs. None when neither yields."""
  36. env_value = cron_env_setting(env_var).strip()
  37. if env_value:
  38. try:
  39. value = parse(env_value)
  40. if value is not None:
  41. return value
  42. except Exception:
  43. logger.warning("Invalid %s=%r; using config/default", env_var, env_value)
  44. try:
  45. cfg = _sched.load_config() or {}
  46. cron_cfg = cfg.get("cron", {}) if isinstance(cfg, dict) else {}
  47. configured = cron_cfg.get(config_key)
  48. if configured is not None:
  49. value = parse(configured)
  50. if value is not None:
  51. return value
  52. except Exception as exc:
  53. logger.debug("Failed to load %s from config: %s", label, exc)
  54. return None
  55. def _get_script_timeout() -> int:
  56. """Resolve cron pre-run script timeout from module/env/config with a safe default."""
  57. if _sched._SCRIPT_TIMEOUT != _sched._DEFAULT_SCRIPT_TIMEOUT:
  58. try:
  59. timeout = _positive_int(_sched._SCRIPT_TIMEOUT)
  60. if timeout is not None:
  61. return timeout
  62. except Exception:
  63. logger.warning(
  64. "Invalid patched _SCRIPT_TIMEOUT=%r; using env/config/default",
  65. _sched._SCRIPT_TIMEOUT)
  66. resolved = _timeout_from_env_or_config(
  67. "HERMES_CRON_SCRIPT_TIMEOUT", "script_timeout_seconds", _positive_int,
  68. "cron script timeout",
  69. )
  70. return _sched._DEFAULT_SCRIPT_TIMEOUT if resolved is None else resolved
  71. _DEFAULT_MEDIA_SEND_TIMEOUT = 300
  72. def _get_media_send_timeout() -> int:
  73. """Per-attachment media-send timeout: HERMES_CRON_MEDIA_SEND_TIMEOUT env, then
  74. ``cron.media_send_timeout_seconds``, then 300s (long TTS audio can exceed a 30s window)."""
  75. resolved = _timeout_from_env_or_config(
  76. "HERMES_CRON_MEDIA_SEND_TIMEOUT", "media_send_timeout_seconds", _positive_int,
  77. "cron media-send timeout")
  78. return _DEFAULT_MEDIA_SEND_TIMEOUT if resolved is None else resolved
  79. def _get_session_db_timeout() -> float:
  80. """Bound on run_job's SessionDB init: HERMES_CRON_SESSION_DB_TIMEOUT env, then
  81. ``cron.session_db_timeout_seconds`` (in DEFAULT_CONFIG), then 10s. Unlike sibling timeouts,
  82. 0 is meaningful (unlimited, debugging opt-in), so values pass through untouched."""
  83. resolved = _timeout_from_env_or_config(
  84. "HERMES_CRON_SESSION_DB_TIMEOUT", "session_db_timeout_seconds", float,
  85. "cron.session_db_timeout_seconds")
  86. return 10.0 if resolved is None else resolved
  87. def _read_windows_pyvenv_cfg(venv_dir: Path) -> dict[str, str]:
  88. try:
  89. lines = (venv_dir / "pyvenv.cfg").read_text(encoding="utf-8-sig").splitlines()
  90. except OSError:
  91. return {}
  92. return {
  93. key.strip().lower(): value.strip()
  94. for key, value in (raw.split("=", 1) for raw in lines if "=" in raw)
  95. }
  96. # Mirrors ``python script.py``: sys.path[0] is the script's directory (none under -P), a real
  97. # ``__main__`` module that outlives the body (atexit/threads can still pickle its classes; a
  98. # runpy temp module is swapped out when the body returns), plus the live checkout next.
  99. _POSIX_SCRIPT_BOOTSTRAP = """\
  100. import importlib.machinery, os, sys, types
  101. repo, script = sys.argv[1], sys.argv[2]
  102. sys.argv = [script] + sys.argv[3:]
  103. if sys.flags.safe_path:
  104. sys.path.insert(0, repo)
  105. else:
  106. sys.path[0:1] = [os.path.dirname(script), repo]
  107. main = types.ModuleType("__main__")
  108. main.__file__ = script
  109. main.__loader__ = importlib.machinery.SourceFileLoader("__main__", script)
  110. main.__cached__ = None
  111. main.__builtins__ = __builtins__
  112. sys.modules["__main__"] = main
  113. with open(script, "rb") as f:
  114. code = compile(f.read(), script, "exec")
  115. exec(code, main.__dict__)
  116. """
  117. def _posix_cron_script_argv(script: Path) -> tuple[list[str], dict[str, str]]:
  118. """POSIX managed-store installs run cron ``.py`` scripts on the selected dependency venv's
  119. interpreter: the store Python has the repo and managed site-packages only on its in-process
  120. ``sys.path``, so its children import neither (#123044). No ``PYTHONPATH``: everything the
  121. script spawns would inherit it and a foreign interpreter would load the store's compiled
  122. extensions (#123440). The venv resolves Hermes from its generation's workspace snapshot,
  123. rebuilt only on a dependency change, so the bootstrap puts the live checkout first.
  124. Lazy installs are off for the script's process tree: a script importing ``hermes_bootstrap``
  125. could otherwise complete a source update and ``execv`` itself onto the bare store Python."""
  126. from hermes_cli._launchers import resolve_store_python
  127. from pm.environments import project_python
  128. repo = Path(__file__).resolve().parents[1]
  129. if resolve_store_python(repo) is None:
  130. return [sys.executable, str(script)], {}
  131. # project_python, not committed_venv as on Windows: a pre-PM venv selected before the first
  132. # commit runs on its OWN interpreter here, so there is no ABI mix (#122183).
  133. python = project_python(repo)
  134. if not python.is_file():
  135. # The caller's interpreter is the bare store Python here — the #123044 failure mode.
  136. raise RuntimeError(f"dependency environment interpreter is missing: {python}")
  137. return ([str(python), "-c", _POSIX_SCRIPT_BOOTSTRAP, str(repo), str(script)],
  138. {"HERMES_DISABLE_LAZY_INSTALLS": "1"})
  139. def _windows_cron_python_invocation(python_exe: str) -> tuple[str, dict[str, str]]:
  140. """Hidden, output-capable Python invocation for Windows cron scripts. ``pythonw.exe`` loses
  141. captured output; uv venv launchers can re-exec the base console python and flash a window
  142. even with CREATE_NO_WINDOW, so run the base python directly with venv paths overlaid in env."""
  143. if sys.platform != "win32":
  144. return python_exe, {}
  145. interpreter = _sched.Path(python_exe)
  146. venv_dir = interpreter.parent.parent
  147. env_overlay: dict[str, str] = {}
  148. if interpreter.name.lower() == "pythonw.exe":
  149. sibling = interpreter.with_name("python.exe")
  150. if sibling.exists():
  151. interpreter = sibling
  152. from hermes_cli._launchers import resolve_store_python
  153. from pm.environments import committed_venv, site_packages as dependency_site
  154. repo = Path(__file__).resolve().parents[1]
  155. managed_python = resolve_store_python(repo)
  156. if managed_python is not None:
  157. # Only the committed generation may overlay the store Python: ``selected_venv``
  158. # falls back to the leftover pre-PM <root>/venv, a foreign ABI (#122183). With
  159. # nothing committed, fall through so the handed venv keeps its own interpreter
  160. # and site-packages (a bare store Python would die on its first import).
  161. environment = committed_venv(repo)
  162. if environment is not None:
  163. return str(managed_python), {"PYTHONPATH": os.pathsep.join(
  164. [str(repo), str(dependency_site(environment))])}
  165. cfg = _read_windows_pyvenv_cfg(venv_dir)
  166. home = cfg.get("home", "")
  167. site_packages = venv_dir / "Lib" / "site-packages"
  168. if "uv" in cfg and home:
  169. base_python = _sched.Path(home) / "python.exe"
  170. if base_python.exists() and site_packages.exists():
  171. interpreter = base_python
  172. env_overlay["VIRTUAL_ENV"] = str(venv_dir)
  173. pythonpath_entries = [
  174. str(_sched.Path(__file__).resolve().parents[1]), str(site_packages)]
  175. existing_pythonpath = os.environ.get("PYTHONPATH", "")
  176. if existing_pythonpath:
  177. pythonpath_entries.append(existing_pythonpath)
  178. env_overlay["PYTHONPATH"] = os.pathsep.join(pythonpath_entries)
  179. return str(interpreter), env_overlay
  180. def _terminate_cron_script_process(proc: subprocess.Popen) -> None:
  181. """Best-effort hard stop of a cron script and every child it spawned."""
  182. if proc.poll() is not None:
  183. return
  184. if sys.platform != "win32":
  185. _terminate_process_group(proc)
  186. else:
  187. try:
  188. subprocess.run(
  189. ["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=10,
  190. creationflags=windows_hide_flags(), check=False)
  191. except (OSError, subprocess.TimeoutExpired):
  192. proc.kill()
  193. try:
  194. proc.wait(timeout=1.0)
  195. except subprocess.TimeoutExpired:
  196. proc.kill()
  197. proc.wait(timeout=1.0)
  198. def _terminate_process_group(proc: subprocess.Popen) -> None:
  199. """POSIX: TERM the script's process group, then KILL if ANY member survived (a survivor holds
  200. the pipe write ends open and the caller's communicate() would block on EOF forever)."""
  201. try:
  202. process_group = os.getpgid(proc.pid)
  203. os.killpg(process_group, signal.SIGTERM) # windows-footgun: ok — POSIX-only branch
  204. except (ProcessLookupError, PermissionError, OSError):
  205. return
  206. with contextlib.suppress(subprocess.TimeoutExpired):
  207. proc.wait(timeout=1.0)
  208. try:
  209. os.killpg(process_group, 0) # windows-footgun: ok — POSIX-only branch
  210. except (ProcessLookupError, OSError):
  211. return
  212. with contextlib.suppress((ProcessLookupError, PermissionError, OSError)):
  213. os.killpg(process_group, getattr(signal, "SIGKILL", signal.SIGTERM))
  214. def _terminate_cron_script_tree(proc: subprocess.Popen) -> None:
  215. """Terminate a script tree, then fall back to the local process-group path."""
  216. if proc.poll() is not None:
  217. # Already reaped: kill_process_tree would log a spurious "no signal" warning.
  218. return
  219. def fallback(reason: str, *args, exc_info: bool = False) -> None:
  220. logger.warning(
  221. reason + "; falling back to process-group termination", *args, exc_info=exc_info)
  222. _terminate_cron_script_process(proc)
  223. pid = getattr(proc, "pid", None)
  224. if not isinstance(pid, int) or pid <= 0:
  225. return fallback("Cron script tree-kill received invalid pid %r", pid)
  226. try:
  227. # Function-local (monkeypatchable); separate try so an import problem is not
  228. # misreported as a kill failure.
  229. from agent.deadline import kill_process_tree
  230. except Exception:
  231. return fallback("agent.deadline.kill_process_tree unavailable", exc_info=True)
  232. try:
  233. if kill_process_tree(pid):
  234. return
  235. except Exception:
  236. return fallback("Cron script tree-kill failed for pid %s", pid, exc_info=True)
  237. fallback("Cron script tree-kill reported no signal for pid %s", pid)
  238. def _drain_script_pipes(proc: subprocess.Popen) -> None:
  239. """Reap a terminated script without blocking forever: a surviving descendant can hold the pipe
  240. write ends open, so bound the drain and abandon the pipes (output is not needed)."""
  241. with contextlib.suppress(subprocess.TimeoutExpired):
  242. proc.communicate(timeout=5.0)
  243. return
  244. with contextlib.suppress(OSError):
  245. proc.kill()
  246. for stream in (proc.stdout, proc.stderr):
  247. with contextlib.suppress(OSError):
  248. if stream is not None:
  249. stream.close()
  250. with contextlib.suppress(subprocess.TimeoutExpired):
  251. proc.wait(timeout=5.0)
  252. def _windows_cron_bootstrap_argv(
  253. python_exe: str, env_overlay: dict[str, str], script_path: str) -> list[str]:
  254. """Bootstrap a cron script under the base interpreter with ``.pth`` support. Overlay mode puts
  255. the venv on ``PYTHONPATH``, but ``.pth`` files are only processed by ``site.addsitedir()``, so
  256. editable installs would be invisible; bootstrap via addsitedir + ``runpy.run_path`` (keeps
  257. ``__file__``/``sys.path[0]`` semantics). Plain invocation if the venv is unresolvable."""
  258. site_packages = next((Path(item) for item in env_overlay.get("PYTHONPATH", "").split(os.pathsep)
  259. if Path(item).name == "site-packages"),
  260. _sched.Path(env_overlay.get("VIRTUAL_ENV", "")) / "Lib" / "site-packages")
  261. if not site_packages.is_dir():
  262. # Warn: silent fallback would make "editable installs invisible" undiagnosable.
  263. logger.warning(
  264. "Windows cron script: venv site-packages %s not found; running "
  265. "without .pth processing (editable installs may be unimportable)",
  266. site_packages)
  267. return [python_exe, script_path]
  268. bootstrap = (
  269. "import os, runpy, site, sys;"
  270. f"site.addsitedir({str(site_packages)!r});"
  271. "script = sys.argv[1];"
  272. "sys.argv = [script] + sys.argv[2:];"
  273. "sys.path.insert(0, os.path.dirname(os.path.abspath(script)));"
  274. "runpy.run_path(script, run_name='__main__')"
  275. )
  276. return [python_exe, "-c", bootstrap, script_path]
  277. def _resolve_script_path(script_path: str) -> tuple[Optional[Path], Optional[str]]:
  278. """Validate a job script path; ``(path, None)`` or ``(None, error)``. Scripts MUST resolve
  279. inside HERMES_HOME/scripts/ (relative, absolute and ``~`` paths are all validated — path
  280. traversal / absolute-path injection); contract of lifecycle_guard._expand_candidate_path."""
  281. scripts_dir = _sched._get_hermes_home() / "scripts"
  282. _ensure_cron_dir(scripts_dir)
  283. scripts_dir_resolved = scripts_dir.resolve()
  284. # Reject NUL eagerly: on Windows Path ops raise ValueError *after* expanduser so the try below
  285. # would not catch it. str() first so the guard itself cannot raise on a non-str script_path.
  286. # Same ingestion contract as cron.lifecycle_guard._expand_candidate_path: a NUL-bearing value can never
  287. # name a real script, and on Windows the Path operations raise ValueError *after* expanduser (expanduser
  288. # never expands "~user" there, so the try below never fires) — reject eagerly so both platforms fail
  289. # cleanly instead of crashing the scheduler. str() first so the guard itself can never raise TypeError
  290. # on a non-str script_path (e.g. a Path passed by a future caller) — the guard must be crash-proof even
  291. # though every current call site passes a plain str (#86832 review).
  292. if "\x00" in str(script_path):
  293. return None, f"Blocked: script path contains a NUL byte: {script_path!r}"
  294. try:
  295. raw = _sched.Path(script_path).expanduser()
  296. except (ValueError, RuntimeError, OSError):
  297. # RuntimeError: unexpandable ``~`` (no resolvable HOME).
  298. return None, f"Blocked: script path is not a valid filesystem path: {script_path!r}"
  299. path = raw.resolve() if raw.is_absolute() else (scripts_dir / raw).resolve()
  300. # Traversal / absolute-path / symlink escape guard — MUST stay inside HERMES_HOME/scripts/.
  301. try:
  302. path.relative_to(scripts_dir_resolved)
  303. except ValueError:
  304. return None, (
  305. f"Blocked: script path resolves outside the scripts directory "
  306. f"({scripts_dir_resolved}): {script_path!r}"
  307. )
  308. if not path.exists():
  309. # Scripts resolve against THIS profile's scripts/ dir by design (profiles never share files),
  310. # which is the usual reason a copied job cannot find a script that exists elsewhere (#94821).
  311. return None, (
  312. f"Script not found: {path}. Cron scripts are looked up only in this profile's folder "
  313. f"({scripts_dir_resolved}); if the job was copied from another profile, copy the script "
  314. f"there too, or edit the job with `hermes cron edit`."
  315. )
  316. if not path.is_file():
  317. return None, f"Script path is not a file: {path}"
  318. return path, None
  319. def _resolve_cron_interpreter(interpreter: str) -> tuple[Optional[str], Optional[str]]:
  320. """``(python_exe, error)`` for a job's ``interpreter`` field. Checked at run time, not create
  321. time: a user venv can be rebuilt or moved while the job lives. Bare names are refused — they
  322. silently change meaning with PATH. The name (and the symlink target's name) must look like a
  323. Python: the lifecycle guard classifies ``.py`` scripts as Python and skips its shell reference
  324. walk, so ``interpreter=/bin/bash`` would run an unscanned ``.py`` body as shell. ``pythonw``
  325. is refused because it discards captured output (an agent job would go silently quiet)."""
  326. from cron.lifecycle_guard import _INTERPRETER_IMAGE_RE
  327. raw = interpreter.strip()
  328. try:
  329. resolved = Path(raw).expanduser()
  330. if not resolved.is_absolute():
  331. return None, (f"Interpreter must be an absolute or ~-prefixed path (got {raw!r}). "
  332. "Bare names like 'python3' are not stable across PATH changes.")
  333. mode = resolved.stat().st_mode
  334. names = {resolved.name.lower(), resolved.resolve().name.lower()}
  335. except FileNotFoundError:
  336. return None, f"Interpreter not found: {raw}"
  337. except (RuntimeError, OSError) as exc: # unknown ~user, unreadable parent, symlink loop
  338. return None, f"Unable to resolve interpreter path {raw!r}: {exc}"
  339. if not stat.S_ISREG(mode):
  340. return None, f"Interpreter path is not a file: {resolved}"
  341. if sys.platform != "win32" and not mode & 0o111:
  342. return None, f"Interpreter is not executable: {resolved}"
  343. if not all(_INTERPRETER_IMAGE_RE.match(name) and not name.startswith("pythonw")
  344. for name in names):
  345. return None, f"Interpreter must be a Python executable (python, python3, python3.12, ...): {resolved}"
  346. return str(resolved), None
  347. def _script_argv(
  348. path: Path, interpreter: Optional[str] = None,
  349. ) -> tuple[Optional[list[str]], dict[str, str], Optional[str]]:
  350. """``(argv, env_overlay, error)`` for a validated script. Interpreter by extension — the
  351. shebang is deliberately NOT honoured (small, auditable surface): ``.sh``/``.bash`` → bash,
  352. else the job's ``interpreter`` when set, else a Python chosen by ``_posix_cron_script_argv``
  353. / ``_windows_cron_python_invocation``. Interpreter selection reads PM's install records and
  354. may raise; callers run this inside their ``try``."""
  355. if path.suffix.lower() in {".sh", ".bash"}:
  356. # which() finds Git Bash on Windows; None there → clear error instead of a "[WinError 2]".
  357. _bash = shutil.which("bash") or ("/bin/bash" if os.path.isfile("/bin/bash") else None)
  358. if _bash is None:
  359. return None, {}, (
  360. f"Cannot run .sh/.bash script {path.name!r}: bash not found on PATH. "
  361. "On Windows, install Git for Windows (which ships Git Bash) "
  362. "or rewrite the script as Python (.py)."
  363. )
  364. return [_bash, str(path)], {}, None
  365. if isinstance(interpreter, str) and interpreter.strip():
  366. # A user venv gets none of the managed-store overlays: the repo bootstrap / PYTHONPATH
  367. # exist to run Hermes' own dependency venv and would shadow the user's packages.
  368. python_exe, err = _resolve_cron_interpreter(interpreter)
  369. return ([python_exe, str(path)] if python_exe else None), {}, err
  370. if sys.platform != "win32":
  371. argv, env_overlay = _posix_cron_script_argv(path)
  372. return argv, env_overlay, None
  373. python_exe, env_overlay = _windows_cron_python_invocation(sys.executable)
  374. if env_overlay:
  375. return _windows_cron_bootstrap_argv(python_exe, env_overlay, str(path)), env_overlay, None
  376. return [python_exe, str(path)], env_overlay, None
  377. def _run_job_script(
  378. script_path: str, workdir: Optional[str] = None,
  379. cancel_event: Optional[_CancelEventLike] = None, interpreter: Optional[str] = None,
  380. ) -> tuple[bool, str]:
  381. """Execute a cron job's script and return ``(success, output)``; on failure *output* is the
  382. error message for the LLM to report. Env goes through ``build_subprocess_env`` (SECURITY.md
  383. §2.3). ``workdir`` sets the subprocess cwd only; the Python process cwd is NEVER mutated (an
  384. ``os.chdir()`` would leak into concurrent gateway sessions).
  385. Args: script_path: Path to the script. Relative paths are resolved against HERMES_HOME/scripts/.
  386. Absolute and ~-prefixed paths are also validated to ensure they stay within the scripts dir. workdir:
  387. Optional absolute path to use as the script's cwd. When set, the subprocess runs in this directory
  388. instead of the scripts-dir parent. See #69396. interpreter: the job's optional Python for
  389. ``.py`` scripts (#8714).
  390. """
  391. path, err = _resolve_script_path(script_path)
  392. if path is None:
  393. return False, err
  394. script_timeout = _get_script_timeout()
  395. try:
  396. argv, env_overlay, err = _script_argv(path, interpreter)
  397. if argv is None:
  398. return False, err
  399. from tools.environments.local import build_subprocess_env
  400. # Lossy decode only: keep the platform-default (locale) encoding — gating ``encoding=``
  401. # to win32 was deliberate (#66566: unconditional UTF-8 leaked into POSIX) — but
  402. # ``errors=`` must not stay 'strict': one stray non-UTF-8 byte in the script's stdout
  403. # or stderr raises UnicodeDecodeError in communicate() and fails the whole run,
  404. # discarding the output (#105582; the Windows branch decodes lossily per #45099).
  405. popen_kwargs: dict[str, Any] = {
  406. # detached-spawn: ok — awaited; cancel and timeout tree-kill the group
  407. "start_new_session": True,
  408. "errors": "replace",
  409. }
  410. if sys.platform == "win32":
  411. popen_kwargs = {
  412. "creationflags": windows_hide_flags()
  413. | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0),
  414. # Lossy UTF-8 decode — locale-mismatched bytes from the STT command must not raise in the
  415. # reader threads on non-UTF-8 Windows (#45099).
  416. # Lossy UTF-8 decode — locale-mismatched bytes from the TTS command must not raise in the
  417. # reader threads on non-UTF-8 Windows (#45099).
  418. "encoding": "utf-8",
  419. "errors": "replace"}
  420. # The process env is the LAUNCH profile's. For a job owned by a routed profile, drop that
  421. # profile's .env residue from the base first (no-op for the launch profile's own jobs), then
  422. # overlay the routed profile's own scope (which never enters os.environ under multiplex
  423. # semantics) before the sanitizer, which also overlays the names the owning profile declares
  424. # in terminal.env_passthrough from that scope (#114209). The factory snapshots the process
  425. # env itself — no raw copy at the spawn site (test_subprocess_env_guard).
  426. env = build_subprocess_env(strip_launch_profile=True)
  427. env.update(env_overlay)
  428. # Subprocess cwd only (default: scripts-dir parent). NEVER os.chdir() the process.
  429. # Use the job's workdir as the subprocess cwd when configured, otherwise default to the scripts-dir
  430. # parent (back-compat). NEVER mutate the Python process cwd — that would leak into concurrent
  431. # gateway sessions (#69396).
  432. proc = subprocess.Popen(
  433. argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
  434. cwd=workdir or str(path.parent), env=env, **popen_kwargs)
  435. deadline = time.monotonic() + script_timeout
  436. while True:
  437. # Tree-kill on cancel AND timeout: killpg misses setsid grandchildren (watchdogs,
  438. # backgrounded shell jobs); kill_process_tree snapshots descendants BEFORE signalling.
  439. if cancel_event is not None and cancel_event.is_set():
  440. _terminate_cron_script_tree(proc)
  441. _drain_script_pipes(proc)
  442. from cron.cancellation import cancel_reason
  443. return False, f"Script cancelled: {cancel_reason(cancel_event)}"
  444. remaining = deadline - time.monotonic()
  445. if remaining <= 0:
  446. _terminate_cron_script_tree(proc)
  447. _drain_script_pipes(proc)
  448. # Phase 4a (#85125): a script timeout must leave ZERO living descendants. killpg only
  449. # reaches the script's own process group — a grandchild that called setsid (backgrounded
  450. # shell jobs, watchdogs) escapes it and keeps running after the job reports failure (#71148
  451. # / #59549). agent.deadline.kill_process_tree snapshots the descendant set via psutil BEFORE
  452. # signalling, so own-session grandchildren are reached too — the unified deadline layer's
  453. # tree-kill (#85147, d6a5cb9725).
  454. return False, f"Script timed out after {script_timeout}s: {path}"
  455. try:
  456. stdout_raw, stderr_raw = proc.communicate(timeout=min(0.1, remaining))
  457. break
  458. except subprocess.TimeoutExpired:
  459. continue
  460. stdout = (stdout_raw or "").strip()
  461. stderr = (stderr_raw or "").strip()
  462. # Redact secrets before ANY return path.
  463. try:
  464. from agent.redact import redact_sensitive_text
  465. stdout = redact_sensitive_text(stdout)
  466. stderr = redact_sensitive_text(stderr)
  467. except Exception as e:
  468. logger.warning("Failed to redact sensitive text from output: %s", e)
  469. stdout = stderr = "[REDACTED - redaction failed]"
  470. if proc.returncode != 0:
  471. parts = [f"Script exited with code {proc.returncode}"]
  472. if stderr:
  473. parts.append(f"stderr:\n{stderr}")
  474. if stdout:
  475. parts.append(f"stdout:\n{stdout}")
  476. return False, "\n".join(parts)
  477. return True, stdout
  478. except Exception as exc:
  479. return False, f"Script execution failed: {exc}"
  480. def _start_heartbeat_thread(loop_fn, name: str, fail_log) -> Optional[threading.Thread]:
  481. """Start ``loop_fn`` on a daemon thread inside a copy of the current context (multiplexed
  482. profile ContextVars). On failure calls ``fail_log()`` inside the except (traceback intact) and
  483. returns None."""
  484. thread = threading.Thread(
  485. target=contextvars.copy_context().run, args=(loop_fn,), name=name, daemon=True)
  486. try:
  487. thread.start()
  488. except Exception:
  489. fail_log()
  490. return None
  491. return thread
  492. def _run_job_script_with_claim_heartbeat(
  493. job: dict, script_path: str, workdir: Optional[str] = None,
  494. cancel_event: Optional[_CancelEventLike] = None,
  495. ) -> tuple[bool, str]:
  496. """Run a cron script while heartbeating its owned one-shot claim. A long script can outlive
  497. the stale-claim TTL; without a heartbeat another scheduler would re-dispatch the one-shot.
  498. Recurring/unclaimed runs have no durable claim → no thread. The owner is captured from the
  499. dispatched job, never re-read, so a stale runner cannot extend a replacement owner's claim."""
  500. def run() -> tuple[bool, str]:
  501. return _run_job_script(script_path, workdir=workdir, cancel_event=cancel_event,
  502. interpreter=job.get("interpreter"))
  503. schedule = job.get("schedule")
  504. claim = job.get("run_claim")
  505. owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
  506. if not (isinstance(schedule, dict) and schedule.get("kind") == "once" and owner):
  507. return run()
  508. job_id = str(job.get("id") or "")
  509. stop = threading.Event()
  510. def _heartbeat_loop() -> None:
  511. while not stop.wait(_sched._RUN_CLAIM_HEARTBEAT_SECONDS):
  512. try:
  513. _sched.heartbeat_run_claim(job_id, expected_owner=owner)
  514. except Exception:
  515. logger.debug("Job '%s': script run_claim heartbeat failed", job_id, exc_info=True)
  516. heartbeat_thread = _start_heartbeat_thread(
  517. _heartbeat_loop, "cron-script-claim-heartbeat",
  518. lambda: logger.debug(
  519. "Job '%s': could not start script run_claim heartbeat", job_id, exc_info=True),
  520. )
  521. if heartbeat_thread is None:
  522. return run()
  523. try:
  524. return run()
  525. finally:
  526. stop.set()
  527. # Bounded join: the heartbeat may be blocked on another process's jobs-file lock.
  528. heartbeat_thread.join(timeout=1.0)
  529. # Late-bound origin namespace (see module docstring). Imported LAST so this module is fully
  530. # populated before ``scheduler`` re-exports from it.
  531. from cron import scheduler as _sched