cancellation.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. """On-demand cancellation of an in-flight cron run (``hermes cron cancel``).
  2. The request has to reach the process that owns the run, and that process is usually not
  3. the one taking the request: ``run_one_job`` hands an attempt to a detached external worker
  4. before anything executes in-process, so an in-memory ``Event`` can never be the transport.
  5. The request is therefore a marker file per EXECUTION — the ledger already mints one id per
  6. attempt — polled by the same cancel seam the fire-claim heartbeat uses, so a cancel reaches
  7. script runs (tree-kill + TERM/KILL escalation) and agent runs (interrupt) through the code
  8. those paths already own.
  9. Keying by execution id is what makes a leftover marker harmless: only the run it was written
  10. for can read it, so a cancel can never bleed into the job's next fire.
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import os
  15. import tempfile
  16. import time
  17. from pathlib import Path
  18. from typing import Any, Dict, Optional, Protocol
  19. from hermes_constants import get_hermes_home
  20. from cron.executions import cancellable_execution
  21. # A marker whose run was SIGKILLed before it could clean up is unreadable garbage. Canceling a
  22. # job is rare, so the prune rides along with the write instead of needing its own sweep.
  23. _MARKER_TTL_SECONDS = 24 * 3600
  24. _DEFAULT_REASON = "cancelled on request (hermes cron cancel)"
  25. def _cancels_dir() -> Path:
  26. """Resolved per call so a multiplexed gateway never writes into another profile's home."""
  27. return get_hermes_home().resolve() / "cron" / "cancels"
  28. def _marker_path(execution_id: str) -> Path:
  29. execution_id = str(execution_id or "").strip()
  30. if not execution_id or "/" in execution_id or "\\" in execution_id or execution_id in {".", ".."}:
  31. raise ValueError(f"Not a usable execution id: {execution_id!r}")
  32. return _cancels_dir() / f"{execution_id}.cancel"
  33. def _prune_stale_markers(now: Optional[float] = None) -> None:
  34. cutoff = (now if now is not None else time.time()) - _MARKER_TTL_SECONDS
  35. with contextlib.suppress(OSError):
  36. for entry in _cancels_dir().iterdir():
  37. with contextlib.suppress(OSError):
  38. if entry.stat().st_mtime < cutoff:
  39. entry.unlink()
  40. def request_cancel(execution_id: str, *, reason: Optional[str] = None) -> Path:
  41. """Record a cancel for one execution. Idempotent: the marker holds the first reason.
  42. A second request for the same execution keeps the reason already on disk, because the run
  43. may have read it into its report before the later request lands. The first ask is the one
  44. that describes why the operator acted.
  45. """
  46. path = _marker_path(execution_id)
  47. path.parent.mkdir(parents=True, exist_ok=True)
  48. if not path.exists():
  49. _publish_first(path, (reason or _DEFAULT_REASON).strip().encode("utf-8"))
  50. _prune_stale_markers()
  51. return path
  52. def _publish_first(path: Path, body: bytes) -> None:
  53. """Create ``path`` holding ``body`` unless it already exists (the first reason wins).
  54. A private temp file published with link() means overlapping requests never share a temp path
  55. and a reader never sees the reason half-written. A filesystem without hard links (exFAT, some
  56. network shares) falls back to an exclusive create: a reader racing that write sees an empty
  57. marker, which still reads as a cancel, with the default reason.
  58. """
  59. fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f"{path.name}.", suffix=".tmp")
  60. try:
  61. with os.fdopen(fd, "wb") as handle:
  62. handle.write(body)
  63. with contextlib.suppress(FileExistsError):
  64. os.link(tmp, path)
  65. linked = True
  66. except OSError: # no hard links on this filesystem
  67. linked = False
  68. finally:
  69. with contextlib.suppress(OSError):
  70. os.unlink(tmp)
  71. if not linked:
  72. with contextlib.suppress(FileExistsError), open(path, "xb") as handle:
  73. handle.write(body)
  74. def cancel_requested(execution_id: str) -> bool:
  75. with contextlib.suppress(ValueError, OSError):
  76. return _marker_path(execution_id).exists()
  77. return False
  78. def clear_cancel(execution_id: str) -> None:
  79. """Called by the run itself once it has wound down, so a finished attempt leaves no marker."""
  80. with contextlib.suppress(ValueError, OSError):
  81. _marker_path(execution_id).unlink(missing_ok=True)
  82. class _CancelEventLike(Protocol):
  83. """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""
  84. def is_set(self) -> bool: ...
  85. def set(self) -> None: ...
  86. class _CombinedCancelEvent:
  87. """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat
  88. ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.
  89. """
  90. def __init__(self, *events: Optional[_CancelEventLike]) -> None:
  91. self._events = [event for event in events if event is not None]
  92. def is_set(self) -> bool:
  93. return any(event.is_set() for event in self._events)
  94. def set(self) -> None:
  95. for event in self._events:
  96. event.set()
  97. @property
  98. def reason(self) -> Optional[str]:
  99. """Why the run is being torn down, from whichever source fired — so a script or agent
  100. stop never reports "fire claim lost" for an operator cancel (or the reverse)."""
  101. for event in self._events:
  102. if event.is_set():
  103. return getattr(event, "reason", None)
  104. return None
  105. class CancelRequest:
  106. """``_CancelEventLike`` view of one execution's cancel marker.
  107. Implements ``set()`` for the protocol but never latches: this source is owned by another
  108. process, so a local ``set()`` would claim a cancel nobody asked for.
  109. """
  110. def __init__(self, execution_id: str) -> None:
  111. self.execution_id = str(execution_id or "")
  112. self._reason: Optional[str] = None
  113. def is_set(self) -> bool:
  114. return cancel_requested(self.execution_id)
  115. def set(self) -> None: # pragma: no cover — protocol shape only
  116. raise NotImplementedError("a cancel request is set by the requesting process, not the run")
  117. @property
  118. def reason(self) -> str:
  119. if self._reason is None:
  120. with contextlib.suppress(ValueError, OSError):
  121. recorded = _marker_path(self.execution_id).read_text(encoding="utf-8-sig").strip()
  122. if recorded:
  123. self._reason = recorded
  124. self._reason = self._reason or _DEFAULT_REASON
  125. return self._reason
  126. def cancel_reason(cancel_event: Any) -> str:
  127. """Human-facing cause for a cancelled run; the legacy wording when no source carries one."""
  128. return getattr(cancel_event, "reason", None) or "cron fire ownership was lost"
  129. def finish_cancelled_run(job: dict[str, Any], execution_id: str, fire_owner: Optional[str]) -> None:
  130. """Book an operator-cancelled run: ledger row ``cancelled`` with the recorded reason, and the
  131. job's ``last_status`` — without a failure alert or a failure-streak bump, since the operator
  132. asked for it. A job removed mid-run has no record left to mark; its ledger row still lands."""
  133. from cron import scheduler
  134. from cron.unreachable_retry import is_retry_run
  135. reason = CancelRequest(execution_id).reason
  136. if not scheduler.self_removal_delivery_allowed(job["id"]):
  137. owner = {"expected_fire_owner": fire_owner} if fire_owner is not None else {}
  138. # A stopped ladder re-run re-ran an occurrence that already counted: no second repeat slot.
  139. rung = {"ladder_rung": True} if is_retry_run(job) else {}
  140. scheduler.mark_job_run(
  141. job["id"], False, f"Cancelled: {reason}", status="cancelled", **owner, **rung)
  142. scheduler.finish_execution(execution_id, success=False, error=reason, cancelled=True)
  143. def finish_lost_execution(execution_id: str, error: str) -> Optional[dict[str, Any]]:
  144. """Close an attempt whose fire claim vanished. Removing a job mid-run cancels it AND drops the
  145. claim, so a run that finds its claim gone books the operator's ``cancelled`` (with the reason)
  146. when a cancel marker is waiting, and ``failed`` with ``error`` only for a real claim loss."""
  147. from cron import scheduler
  148. if cancel_requested(execution_id):
  149. return scheduler.finish_execution(
  150. execution_id, success=False, error=CancelRequest(execution_id).reason, cancelled=True)
  151. return scheduler.finish_execution(execution_id, success=False, error=error)
  152. def run_job_unless_cancelled(run_job, job, run_kwargs, teardown, fence, execution_id, fire_owner):
  153. """``run_job``'s result, or None once an operator cancel stopped the run and was booked.
  154. Checked before the caller's claim-loss handling: removing a job mid-run cancels it AND drops
  155. its claim, and the run must book the cancel, not "fire claim ownership lost" (#123045).
  156. ``run_job`` hands the agent back even when raising, so ``teardown`` runs on every path
  157. (BaseException, so KeyboardInterrupt/SystemExit mid-run still tear down)."""
  158. try:
  159. result = run_job(job, **run_kwargs)
  160. except BaseException as exc:
  161. teardown()
  162. if not (isinstance(exc, Exception) and fence.operator_cancelled()):
  163. raise
  164. finish_cancelled_run(job, execution_id, fire_owner)
  165. return None
  166. if not fence.operator_cancelled():
  167. return result
  168. teardown()
  169. finish_cancelled_run(job, execution_id, fire_owner)
  170. return None
  171. def cancel_run(job_id: str, *, reason: Optional[str] = None) -> dict[str, Any]:
  172. """Request cancellation of the job's live run. A job with nothing in flight is a no-op."""
  173. record = cancellable_execution(job_id)
  174. if record is None:
  175. return {
  176. "success": True,
  177. "cancelled": False,
  178. "execution_id": None,
  179. "detail": "No run of this job is in flight — nothing to cancel.",
  180. }
  181. request_cancel(record["id"], reason=reason)
  182. return {
  183. "success": True,
  184. "cancelled": True,
  185. "execution_id": record["id"],
  186. "status": record.get("status"),
  187. "pid": record.get("pid"),
  188. "detail": "Cancel requested; the run stops at its next cancel check.",
  189. }