executions.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755
  1. """Profile-local durable audit ledger for cron execution attempts.
  2. The ledger records what is known about each attempt; it is not a retry queue. Interrupted attempts
  3. become ``unknown`` only after their owner process is proved gone — a start-time reading that fails
  4. to match the claim-time fingerprint is not proof of death. Terminal states are immutable.
  5. """
  6. from __future__ import annotations
  7. import logging
  8. import math
  9. import os
  10. import sqlite3
  11. import threading
  12. import time
  13. import uuid
  14. from contextlib import contextmanager
  15. from datetime import datetime
  16. from pathlib import Path
  17. from typing import Any, Dict, Iterator, List, Optional
  18. from hermes_constants import get_hermes_home
  19. from hermes_time import now as _hermes_now
  20. from cron.constants import CLAIM_TTL_INACTIVITY_HEADROOM
  21. from hermes_cli.observability.shared_metrics_gateway import record_cron_finish
  22. logger = logging.getLogger(__name__)
  23. # Optional test override. Production resolves the path at transaction time so dashboard operations
  24. # that temporarily enter another profile cannot leak that profile's records into the import-time
  25. # home.
  26. EXECUTIONS_FILE: Optional[Path] = None
  27. MAX_TERMINAL_EXECUTIONS = 1000
  28. HANDOFF_ADOPTION_GRACE_SECONDS = 30.0
  29. # Floor for the live-owner stale-claim bound (#115692); see _live_owner_stale_after_seconds.
  30. LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS = 7200.0
  31. # ``cancelled``: an operator stopped the run (``hermes cron cancel``, removing the job mid-run).
  32. # ``missed``: never started — one row books every occurrence the scheduler skipped past.
  33. _TERMINAL_STATES = ("completed", "failed", "unknown", "cancelled", "missed")
  34. _STATUS_CHECK = "('claimed','running','completed','failed','unknown','cancelled','missed')"
  35. # Outcomes the attempt's own firer booked: receipt recovery, the external-worker waiter and the
  36. # dead-owner sweep must treat these as final, never replay or re-book them.
  37. FIRER_FINISHED_STATES = frozenset({"completed", "failed", "cancelled"})
  38. _lock = threading.RLock()
  39. _PROCESS_ID = uuid.uuid4().hex
  40. # --- executions ledger --------------------------------------------------------------------------
  41. def _connect() -> sqlite3.Connection:
  42. # Late imports: a scheduler daemon that outlives an on-disk upgrade already has the OLD
  43. # ``hermes_cli.sqlite_util`` / ``cron.jobs`` cached, so new names must be resolved at call time,
  44. # not at import time (the guarantee cron/ledger.py used to carry, see e24c8499).
  45. from cron.jobs import _ensure_cron_dir
  46. from hermes_cli.sqlite_util import open_db
  47. path = EXECUTIONS_FILE or (get_hermes_home().resolve() / "cron" / "executions.db")
  48. _ensure_cron_dir(path.parent)
  49. return open_db(path, db_label="cron/executions.db", synchronous_full=True, initialize=_initialize_schema)
  50. def _initialize_schema(conn: sqlite3.Connection) -> None:
  51. from hermes_cli.sqlite_util import add_column_if_missing
  52. conn.execute(
  53. f"""CREATE TABLE IF NOT EXISTS executions (
  54. id TEXT PRIMARY KEY,
  55. job_id TEXT NOT NULL,
  56. source TEXT NOT NULL,
  57. process_id TEXT NOT NULL,
  58. pid INTEGER NOT NULL,
  59. process_started_at INTEGER,
  60. status TEXT NOT NULL CHECK(status IN
  61. {_STATUS_CHECK}),
  62. handoff_pending INTEGER NOT NULL DEFAULT 0,
  63. handoff_started_at REAL,
  64. claimed_at TEXT NOT NULL,
  65. started_at TEXT,
  66. finished_at TEXT,
  67. error TEXT
  68. )"""
  69. )
  70. _widen_status_check(conn)
  71. add_column_if_missing(
  72. conn, "executions", "handoff_pending",
  73. "handoff_pending INTEGER NOT NULL DEFAULT 0",
  74. )
  75. add_column_if_missing(
  76. conn, "executions", "handoff_started_at", "handoff_started_at REAL"
  77. )
  78. conn.execute(
  79. "CREATE INDEX IF NOT EXISTS idx_executions_job_claimed "
  80. "ON executions(job_id, claimed_at DESC, id DESC)"
  81. )
  82. conn.execute(
  83. "CREATE INDEX IF NOT EXISTS idx_executions_status_claimed "
  84. "ON executions(status, claimed_at DESC, id DESC)"
  85. )
  86. add_column_if_missing(conn, "executions", "delivery_outcome", "delivery_outcome TEXT")
  87. add_column_if_missing(conn, "executions", "scheduled_instant", "scheduled_instant TEXT")
  88. add_column_if_missing(conn, "executions", "progress_at", "progress_at TEXT")
  89. conn.execute(
  90. "CREATE INDEX IF NOT EXISTS idx_executions_occurrence "
  91. "ON executions(job_id, scheduled_instant) WHERE status='completed'"
  92. )
  93. def _widen_status_check(conn: sqlite3.Connection) -> None:
  94. """Rebuild a ledger created before ``cancelled``/``missed`` existed: SQLite cannot alter a
  95. CHECK constraint in place. Copies every column the old table has; runs once per file, and
  96. re-reads the schema under the write lock so two processes opening it together rebuild once."""
  97. def current_sql() -> Optional[str]:
  98. row = conn.execute(
  99. "SELECT sql FROM sqlite_master WHERE type='table' AND name='executions'").fetchone()
  100. return row[0] if row is not None else None
  101. if "'missed'" in (current_sql() or "'missed'"):
  102. return
  103. conn.commit()
  104. conn.execute("BEGIN IMMEDIATE")
  105. try:
  106. old_sql = current_sql() or "'missed'"
  107. if "'missed'" not in old_sql:
  108. columns = ", ".join(r[1] for r in conn.execute("PRAGMA table_info(executions)"))
  109. conn.execute("ALTER TABLE executions RENAME TO executions_pre_widen")
  110. conn.execute(old_sql.replace("'unknown')", "'unknown','cancelled','missed')", 1))
  111. conn.execute(
  112. f"INSERT INTO executions ({columns}) SELECT {columns} FROM executions_pre_widen")
  113. conn.execute("DROP TABLE executions_pre_widen")
  114. conn.commit()
  115. except BaseException:
  116. conn.rollback()
  117. raise
  118. @contextmanager
  119. def _transaction() -> Iterator[sqlite3.Connection]:
  120. from hermes_cli.sqlite_util import transaction
  121. with _lock, transaction(_connect()) as conn:
  122. yield conn
  123. def _fetch(conn: sqlite3.Connection, execution_id: str) -> Optional[dict[str, Any]]:
  124. row = conn.execute("SELECT * FROM executions WHERE id=?", (execution_id,)).fetchone()
  125. return dict(row) if row is not None else None
  126. def _emit_execution_state(
  127. record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None
  128. ) -> None:
  129. """Project durable state to monitoring without affecting ledger behavior."""
  130. try:
  131. from agent.monitoring.cron_health import emit_execution_state
  132. emit_execution_state(record, delivery_outcome=delivery_outcome)
  133. except Exception:
  134. pass
  135. def _process_start_time(pid: int) -> Optional[int]:
  136. try:
  137. from gateway.status import get_process_start_time
  138. return get_process_start_time(pid)
  139. except Exception:
  140. return None
  141. def _owner_is_live(pid: int, started_at: Optional[int]) -> bool:
  142. try:
  143. from gateway.status import _pid_exists
  144. if not _pid_exists(pid):
  145. return False
  146. except Exception:
  147. return True # fail safe: inability to prove death must not rewrite state
  148. if started_at is None:
  149. return pid == os.getpid()
  150. current = _process_start_time(pid)
  151. if current is None:
  152. return True # cannot compare -> cannot prove death; a misread must not rewrite state
  153. # Drifted same-host readings (#117505) are not proof of death; a live misread is still
  154. # bounded by the stale-claim sweep below.
  155. from gateway.status import start_time_fingerprints_match
  156. return start_time_fingerprints_match(started_at, current)
  157. def _live_owner_stale_after_seconds() -> Optional[float]:
  158. """Age past which a claimed/running row with a LIVE owner is treated as wedged.
  159. Derived from the existing knobs, never a bare wall-clock constant:
  160. ``max(3 × HERMES_CRON_TIMEOUT, cron script timeout, 7200)``. Returns ``None`` (never reclaim
  161. live owners — today's behaviour) when the inactivity timeout is 0/unlimited or not a finite
  162. positive number: with no bound to derive from, fail closed.
  163. """
  164. from cron.scheduler import _cron_inactivity_seconds
  165. from cron.scheduler_script import _get_script_timeout
  166. inactivity = float(_cron_inactivity_seconds())
  167. if not math.isfinite(inactivity) or inactivity <= 0:
  168. return None
  169. return max(
  170. inactivity * CLAIM_TTL_INACTIVITY_HEADROOM,
  171. float(_get_script_timeout()),
  172. LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS,
  173. )
  174. def _claim_age_seconds(claimed_at: str) -> float:
  175. """Seconds since ``claimed_at`` (NOT NULL, always the aware ISO string from hermes_time.now)."""
  176. return (_hermes_now() - datetime.fromisoformat(claimed_at)).total_seconds()
  177. def _stale_age_seconds(claimed_at: str, progress_at: Optional[str]) -> float:
  178. """Seconds since the owner last proved it was making progress.
  179. A wedged worker (#115692) stops stamping ``progress_at``; a long but healthy run keeps
  180. stamping it, so the derived stale bound measures silence, not run length. Rows written
  181. before the column existed have no stamp and fall back to claim age.
  182. """
  183. return _claim_age_seconds(progress_at or claimed_at)
  184. def touch_execution_progress(execution_id: str) -> bool:
  185. """Stamp ``progress_at`` on a running attempt this process owns. Returns False when the row
  186. is no longer ours or no longer running (the caller treats that as informational only)."""
  187. now = _hermes_now().isoformat()
  188. with _transaction() as conn:
  189. cur = conn.execute(
  190. """UPDATE executions SET progress_at=?
  191. WHERE id=? AND status IN ('claimed','running') AND process_id=? AND pid=?""",
  192. (now, execution_id, _PROCESS_ID, os.getpid()),
  193. )
  194. return cur.rowcount == 1
  195. def _prune_unlocked(conn: sqlite3.Connection) -> None:
  196. conn.execute(
  197. """DELETE FROM executions WHERE id IN (
  198. SELECT id FROM executions
  199. WHERE status IN ('completed','failed','unknown','cancelled','missed')
  200. ORDER BY julianday(finished_at) DESC, finished_at DESC,
  201. julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?
  202. )""",
  203. (max(0, int(MAX_TERMINAL_EXECUTIONS)),),
  204. )
  205. def create_execution(
  206. job_id: str, *, source: str, scheduled_instant: Optional[str] = None,
  207. ) -> dict[str, Any]:
  208. """Persist a claimed attempt before executor/provider dispatch."""
  209. from cron.occurrences import scheduled_instant as canonical_instant
  210. now = _hermes_now().isoformat()
  211. execution_id = uuid.uuid4().hex
  212. pid = os.getpid()
  213. with _transaction() as conn:
  214. conn.execute(
  215. """INSERT INTO executions
  216. (id, job_id, source, process_id, pid, process_started_at,
  217. status, claimed_at, scheduled_instant)
  218. VALUES (?, ?, ?, ?, ?, ?, 'claimed', ?, ?)""",
  219. (execution_id, str(job_id), str(source), _PROCESS_ID, pid,
  220. _process_start_time(pid), now, canonical_instant(scheduled_instant)),
  221. )
  222. record = _fetch(conn, execution_id)
  223. _emit_execution_state(record)
  224. return record # type: ignore[return-value]
  225. def set_execution_occurrence(execution_id: str, instant: Optional[str]) -> None:
  226. """Bind the store-claimed snapshot before a provider hands it to a worker."""
  227. from cron.occurrences import scheduled_instant
  228. with _transaction() as conn:
  229. cur = conn.execute(
  230. "UPDATE executions SET scheduled_instant=? WHERE id=? AND status='claimed' "
  231. "AND handoff_pending=0 AND process_id=? AND pid=?",
  232. (scheduled_instant(instant), execution_id, _PROCESS_ID, os.getpid()),
  233. )
  234. if cur.rowcount != 1:
  235. raise RuntimeError("Cron occurrence could not be bound before dispatch")
  236. def mark_execution_handoff_pending(execution_id: str) -> Optional[dict[str, Any]]:
  237. """Fence restart recovery while an external worker is adopting a claim."""
  238. with _transaction() as conn:
  239. cur = conn.execute(
  240. """UPDATE executions
  241. SET handoff_pending=1, handoff_started_at=?
  242. WHERE id=? AND status='claimed'
  243. AND process_id=? AND pid=?""",
  244. (time.time(), execution_id, _PROCESS_ID, os.getpid()),
  245. )
  246. if cur.rowcount != 1:
  247. return None
  248. record = _fetch(conn, execution_id)
  249. _emit_execution_state(record)
  250. return record
  251. def adopt_claimed_execution(execution_id: str) -> Optional[dict[str, Any]]:
  252. """Atomically transfer and start an attempt in its worker process.
  253. The dispatching gateway creates the row before spawning a restart-safe
  254. worker. Adoption is the single ``claimed`` → ``running`` gate: only the
  255. winner may acknowledge ownership or run side effects.
  256. """
  257. pid = os.getpid()
  258. process_started_at = _process_start_time(pid)
  259. now = _hermes_now().isoformat()
  260. with _transaction() as conn:
  261. cur = conn.execute(
  262. """UPDATE executions
  263. SET process_id=?, pid=?, process_started_at=?,
  264. status='running', started_at=?, handoff_pending=0,
  265. handoff_started_at=NULL
  266. WHERE id=? AND status='claimed' AND handoff_pending=1""",
  267. (_PROCESS_ID, pid, process_started_at, now, execution_id),
  268. )
  269. if cur.rowcount != 1:
  270. return None
  271. record = _fetch(conn, execution_id)
  272. _emit_execution_state(record)
  273. return record
  274. def mark_execution_running(execution_id: str) -> Optional[dict[str, Any]]:
  275. """Transition one claimed attempt to running exactly once."""
  276. now = _hermes_now().isoformat()
  277. with _transaction() as conn:
  278. cur = conn.execute(
  279. """UPDATE executions
  280. SET status='running', started_at=?, handoff_pending=0,
  281. handoff_started_at=NULL
  282. WHERE id=? AND status='claimed' AND handoff_pending=0
  283. AND process_id=? AND pid=?""",
  284. (now, execution_id, _PROCESS_ID, os.getpid()),
  285. )
  286. if cur.rowcount != 1:
  287. return None
  288. record = _fetch(conn, execution_id)
  289. _emit_execution_state(record)
  290. return record
  291. def finish_execution(
  292. execution_id: str, *, success: bool, error: Optional[str] = None,
  293. delivery_outcome: Optional[str] = None, cancelled: bool = False,
  294. ) -> Optional[dict[str, Any]]:
  295. """Write a terminal result once; terminal attempts cannot be rewritten.
  296. Only the owning process may finish its attempt; receipt recovery for a departed owner
  297. settles through ``recover_receipted_execution``.
  298. """
  299. now = _hermes_now().isoformat()
  300. status = "cancelled" if cancelled else ("completed" if success else "failed")
  301. detail = None if success else (str(error) if error else "unknown failure")
  302. with _transaction() as conn:
  303. owner = (_PROCESS_ID, os.getpid())
  304. cur = conn.execute(
  305. """UPDATE executions
  306. SET status=?, finished_at=?, error=?, handoff_pending=0,
  307. handoff_started_at=NULL, delivery_outcome=?
  308. WHERE id=? AND status IN ('claimed','running')
  309. AND process_id=? AND pid=?""",
  310. (status, now, detail, delivery_outcome, execution_id, *owner),
  311. )
  312. if cur.rowcount != 1:
  313. return None
  314. _prune_unlocked(conn)
  315. record = _fetch(conn, execution_id)
  316. _emit_execution_state(record, delivery_outcome=delivery_outcome)
  317. record_cron_finish(record, delivery_outcome)
  318. return record
  319. def record_missed_occurrences(
  320. job_id: str, *, count: int, first_instant: Optional[str], detail: str,
  321. ) -> Optional[dict[str, Any]]:
  322. """Book occurrences the scheduler passed over without running as ONE terminal ``missed`` row
  323. (``missed ×N``), so run history shows the gap instead of nothing. Never started, so no
  324. owner fencing applies; ``first_instant`` is the earliest skipped slot."""
  325. from cron.occurrences import scheduled_instant as canonical_instant
  326. if count < 1:
  327. return None
  328. now = _hermes_now().isoformat()
  329. execution_id = uuid.uuid4().hex
  330. pid = os.getpid()
  331. with _transaction() as conn:
  332. conn.execute(
  333. """INSERT INTO executions
  334. (id, job_id, source, process_id, pid, process_started_at,
  335. status, claimed_at, finished_at, error, scheduled_instant)
  336. VALUES (?, ?, 'scheduler', ?, ?, ?, 'missed', ?, ?, ?, ?)""",
  337. (execution_id, str(job_id), _PROCESS_ID, pid, _process_start_time(pid), now, now,
  338. f"missed \u00d7{count} ({detail})", canonical_instant(first_instant)),
  339. )
  340. _prune_unlocked(conn)
  341. record = _fetch(conn, execution_id)
  342. _emit_execution_state(record)
  343. return record
  344. def recover_receipted_execution(execution_id, job_id, *, success, error=None, delivery_outcome=None):
  345. """Settle a verified canonical receipt after owner loss, including its unknown audit row.
  346. Terminal verdicts stay immutable. A live firer still owns its bookkeeping
  347. (``execution_owner_live``).
  348. """
  349. now = _hermes_now().isoformat()
  350. with _transaction() as conn:
  351. row = _recoverable_row(conn, execution_id, job_id)
  352. if row is None or _row_owner_live(row):
  353. return None
  354. conn.execute(
  355. """UPDATE executions SET status=?, finished_at=?, error=?, handoff_pending=0,
  356. handoff_started_at=NULL, delivery_outcome=? WHERE id=?""",
  357. ('completed' if success else 'failed', now, None if success else (str(error) if error else 'unknown failure'),
  358. delivery_outcome, execution_id))
  359. _prune_unlocked(conn)
  360. record = _fetch(conn, execution_id)
  361. _emit_execution_state(record, delivery_outcome=delivery_outcome)
  362. record_cron_finish(record, delivery_outcome)
  363. return record
  364. def _recoverable_row(conn, execution_id, job_id):
  365. return conn.execute(
  366. """SELECT status, process_id, pid, process_started_at FROM executions
  367. WHERE id=? AND job_id=? AND status IN ('claimed','running','unknown')""",
  368. (str(execution_id), str(job_id))).fetchone()
  369. def _row_owner_live(row) -> bool:
  370. if row['process_id'] == _PROCESS_ID:
  371. # The in-gateway ticker fires on pool threads: a claimed/running row of THIS process is
  372. # still in flight here. Only an unknown row was given up by its own firer.
  373. return row['status'] != 'unknown'
  374. return _owner_is_live(int(row['pid']), row['process_started_at'])
  375. def execution_owner_live(execution_id: str, job_id: str) -> bool:
  376. """True while the attempt's own firer (this process included) can still finish it, so receipt
  377. recovery must not act for it: no output re-save, no job mark, no ledger write."""
  378. with _transaction() as conn:
  379. row = _recoverable_row(conn, execution_id, job_id)
  380. return row is not None and _row_owner_live(row)
  381. def settle_delivery_outcome(execution_id: str, outcome: str) -> bool:
  382. """Replace a terminal attempt's ``queued`` delivery outcome with the drain's real one.
  383. The only post-terminal rewrite the ledger allows: the attempt's status is untouched and
  384. only a ``queued`` handoff can settle, so a replay or a late caller is a no-op.
  385. """
  386. with _transaction() as conn:
  387. cur = conn.execute(
  388. """UPDATE executions SET delivery_outcome=?
  389. WHERE id=? AND delivery_outcome='queued'
  390. AND status IN ('completed','failed','unknown')""",
  391. (outcome, execution_id),
  392. )
  393. return cur.rowcount == 1
  394. _OWNER_GONE_REASON = (
  395. "Scheduler restarted after this execution's owner exited before a durable "
  396. "terminal state; whether side effects ran is unknown."
  397. )
  398. _OWNER_WEDGED_REASON = (
  399. "Owner process is still alive but the claim outlived the derived stale bound; "
  400. "treated as wedged (#115692). The process was not terminated; whether side effects "
  401. "ran is unknown."
  402. )
  403. def settle_unstarted_execution(execution_id: str, job_id: str, error: str) -> None:
  404. """Close the receipt of a run that never started: a ``claimed`` row never resolves. Best-effort
  405. so a ledger write cannot mask the failure the caller logs or re-raises."""
  406. try:
  407. finish_execution(execution_id, success=False, error=error)
  408. except (sqlite3.Error, OSError) as record_err:
  409. logger.error("Job '%s': failed to close execution receipt %s (%s): %s",
  410. job_id, execution_id, error, record_err)
  411. def recover_interrupted_executions() -> int:
  412. """Mark abandoned attempts unknown without scheduling retries: rows whose owner is provably
  413. dead, plus rows whose live owner holds a claim older than the derived stale bound (the
  414. process is not killed)."""
  415. now = _hermes_now().isoformat()
  416. changed = 0
  417. recovered: list[dict[str, Any]] = []
  418. # Derived on the first live-owned row only: the bound reads config, and the idle gateway
  419. # tick must stay config-free (tests/cron/test_idle_tick_config_skip.py).
  420. stale_after: Optional[float] = None
  421. stale_after_resolved = False
  422. with _transaction() as conn:
  423. rows = conn.execute(
  424. """SELECT id, status, process_id, pid, process_started_at,
  425. handoff_pending, handoff_started_at, claimed_at, progress_at
  426. FROM executions
  427. WHERE status IN ('claimed','running')"""
  428. ).fetchall()
  429. for row in rows:
  430. if row["process_id"] == _PROCESS_ID:
  431. continue
  432. reason = _OWNER_GONE_REASON
  433. if _owner_is_live(int(row["pid"]), row["process_started_at"]):
  434. # A live owner is normally a legitimately running job. A worker permanently
  435. # deadlocked (e.g. futex_wait behind a route/proxy flip, #115692) also passes
  436. # this check, so a claim SILENT for longer than the derived bound is treated
  437. # as wedged and released — the external-worker wait loop polls this ledger
  438. # for a terminal status, so the job can fire again. Silence is measured from
  439. # the owner's last ``progress_at`` stamp (the run monitor refreshes it while
  440. # the agent is active), so a healthy multi-hour run is never reclaimed while
  441. # it is still working. The wedged worker PROCESS is NOT terminated here
  442. # (leaked until host restart); rows owned by this process (process_id ==
  443. # _PROCESS_ID, in-process runs) are skipped above and remain out of scope.
  444. if not stale_after_resolved:
  445. stale_after = _live_owner_stale_after_seconds()
  446. stale_after_resolved = True
  447. if (
  448. stale_after is None
  449. or _stale_age_seconds(row["claimed_at"], row["progress_at"]) <= stale_after
  450. ):
  451. continue
  452. reason = _OWNER_WEDGED_REASON
  453. handoff_started_at = row["handoff_started_at"]
  454. if (
  455. row["handoff_pending"]
  456. and handoff_started_at is not None
  457. and time.time() - float(handoff_started_at)
  458. < HANDOFF_ADOPTION_GRACE_SECONDS
  459. ):
  460. continue
  461. # The SELECT ran outside a write transaction and the liveness probe takes time: the
  462. # owner may have stamped progress since, so act only on the snapshot that was judged.
  463. cur = conn.execute(
  464. """UPDATE executions
  465. SET status='unknown', finished_at=?, error=?,
  466. handoff_pending=0, handoff_started_at=NULL
  467. WHERE id=? AND status=? AND process_id=? AND pid=?
  468. AND handoff_pending=?
  469. AND handoff_started_at IS ?
  470. AND claimed_at=? AND progress_at IS ?""",
  471. (now, reason, row["id"], row["status"], row["process_id"], row["pid"],
  472. row["handoff_pending"], row["handoff_started_at"],
  473. row["claimed_at"], row["progress_at"]),
  474. )
  475. changed += cur.rowcount
  476. if cur.rowcount:
  477. record = _fetch(conn, row["id"])
  478. if record is not None:
  479. recovered.append(record)
  480. if changed:
  481. _prune_unlocked(conn)
  482. for record in recovered:
  483. _emit_execution_state(record)
  484. return changed
  485. def terminalize_dead_owner(execution_id: str, *, reason: str) -> bool:
  486. """Record one attempt as ``unknown`` with a cause this process actually observed.
  487. ``recover_interrupted_executions`` sweeps every attempt whose owner is provably
  488. dead, and knows nothing but that absence — so all it can write is
  489. ``_OWNER_GONE_REASON``, which asserts a scheduler restart. A waiter that held the
  490. worker's ``Popen`` knows more: the owner was that external worker, and it exited
  491. with a known status. Without this, a manual run whose worker dies is filed as
  492. "Scheduler restarted ..." (a restart that never happened) and, because the sweep
  493. leaves the row terminal, the waiter reports success and never records the run —
  494. the job's ``fire_claim`` then blocks the next manual fire for the whole lease
  495. (#128509).
  496. The attempt stays ``unknown``, not ``failed``: whether side effects ran is still
  497. unknown. Only the CAUSE becomes truthful. Returns False — leaving the caller to
  498. fall back to the generic sweep — when the row is absent, already terminal, owned by
  499. this process, inside the handoff adoption grace, or owned by a live process: a
  500. worker that is still running must never be terminalized out from under itself.
  501. """
  502. now = _hermes_now().isoformat()
  503. with _transaction() as conn:
  504. row = conn.execute(
  505. """SELECT id, status, process_id, pid, process_started_at,
  506. handoff_pending, handoff_started_at
  507. FROM executions WHERE id=?""",
  508. (execution_id,),
  509. ).fetchone()
  510. if row is None or row["status"] not in ("claimed", "running"):
  511. return False
  512. if row["process_id"] == _PROCESS_ID:
  513. return False
  514. if _owner_is_live(int(row["pid"]), row["process_started_at"]):
  515. return False
  516. handoff_started_at = row["handoff_started_at"]
  517. if (
  518. row["handoff_pending"]
  519. and handoff_started_at is not None
  520. and time.time() - float(handoff_started_at) < HANDOFF_ADOPTION_GRACE_SECONDS
  521. ):
  522. return False
  523. cur = conn.execute(
  524. """UPDATE executions
  525. SET status='unknown', finished_at=?, error=?,
  526. handoff_pending=0, handoff_started_at=NULL
  527. WHERE id=? AND status=? AND process_id=? AND pid=?""",
  528. (now, reason, row["id"], row["status"], row["process_id"], row["pid"]),
  529. )
  530. if cur.rowcount != 1:
  531. return False
  532. record = _fetch(conn, execution_id)
  533. _prune_unlocked(conn)
  534. _emit_execution_state(record)
  535. return True
  536. def give_up_execution(execution_id: str, *, reason: str) -> bool:
  537. """The firer itself gave up on its in-flight attempt (``CronExecutionUnknown``): record it
  538. ``unknown`` now, owner-fenced to this process, instead of leaving it ``running`` until a
  539. restart's dead-owner sweep. Receipt recovery never acts for a same-process running row, so
  540. the owner's later terminal receipt can settle it only once it is marked given up."""
  541. now = _hermes_now().isoformat()
  542. with _transaction() as conn:
  543. cur = conn.execute(
  544. """UPDATE executions SET status='unknown', finished_at=?, error=?,
  545. handoff_pending=0, handoff_started_at=NULL
  546. WHERE id=? AND status IN ('claimed','running') AND process_id=? AND pid=?""",
  547. (now, reason, str(execution_id), _PROCESS_ID, os.getpid()))
  548. if cur.rowcount != 1:
  549. return False
  550. record = _fetch(conn, str(execution_id))
  551. _prune_unlocked(conn)
  552. _emit_execution_state(record)
  553. return True
  554. def list_executions(
  555. *, job_id: Optional[str] = None, limit: int = 50, before_claimed_at: Optional[str] = None,
  556. ) -> list[dict[str, Any]]:
  557. """Return indexed, newest-first execution history with cursor pagination."""
  558. clauses: list[str] = []
  559. params: list[Any] = []
  560. if job_id is not None:
  561. clauses.append("job_id=?")
  562. params.append(str(job_id))
  563. if before_claimed_at is not None:
  564. # Same (instant, text) key as the ORDER BY, so a page never skips or repeats a row.
  565. clauses.append("(julianday(claimed_at), claimed_at) < (julianday(?), ?)")
  566. params.extend([str(before_claimed_at)] * 2)
  567. where = " WHERE " + " AND ".join(clauses) if clauses else ""
  568. params.append(max(1, min(int(limit), 500)))
  569. # Stamps carry the local offset, which changes at DST and on a timezone change, so text order
  570. # is not time order. julianday() compares instants (ms); the text breaks same-ms ties.
  571. with _transaction() as conn:
  572. rows = conn.execute(
  573. "SELECT * FROM executions" + where
  574. + " ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT ?",
  575. params,
  576. ).fetchall()
  577. return [dict(row) for row in rows]
  578. def get_execution(execution_id: str) -> Optional[dict[str, Any]]:
  579. """Return one exact execution attempt, or ``None`` when it is absent."""
  580. with _transaction() as conn:
  581. row = conn.execute(
  582. "SELECT * FROM executions WHERE id=?",
  583. (str(execution_id),),
  584. ).fetchone()
  585. return dict(row) if row is not None else None
  586. def latest_execution(job_id: str) -> Optional[dict[str, Any]]:
  587. rows = list_executions(job_id=job_id, limit=1)
  588. return rows[0] if rows else None
  589. def live_inflight_execution(job_id: str) -> Optional[dict[str, Any]]:
  590. """The job's latest attempt while it is still claimed/running under a LIVE owner, else ``None``.
  591. This is scheduler OWNERSHIP, not recent activity: a run inside a long tool call writes no
  592. heartbeat yet stays owned, while a run whose process died (watchdog kill, crash) does not.
  593. Read-only — unlike ``recover_interrupted_executions`` it never rewrites a row.
  594. """
  595. # A ``missed`` row is the due scan's note about skipped slots, not an attempt: one booked while
  596. # a long run is still live (machine slept through its next slots) must not hide that run.
  597. with _transaction() as conn:
  598. row = conn.execute(
  599. """SELECT * FROM executions WHERE job_id=? AND status != 'missed'
  600. ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT 1""",
  601. (job_id,),
  602. ).fetchone()
  603. record = dict(row) if row is not None else None
  604. if not record or record.get("status") not in ("claimed", "running"):
  605. return None
  606. if not _owner_is_live(int(record["pid"]), record.get("process_started_at")):
  607. return None
  608. return record
  609. def cancellable_execution(job_id: str) -> Optional[dict[str, Any]]:
  610. """The job's newest claimed/running attempt under a LIVE owner, even when it is not the latest row.
  611. A Run-now refused while a run is live books its own newer ``failed`` row ("Fire claim was not
  612. acquired"), so ``live_inflight_execution`` (latest row only) would call the live run idle and a
  613. cancel would never reach it.
  614. """
  615. return live_executions([job_id]).get(str(job_id))
  616. def live_executions(job_ids: list[str]) -> dict[str, dict[str, Any]]:
  617. """Each job's newest claimed/running attempt under a live owner, in one query (see
  618. ``cancellable_execution``). The job list carries it as ``live_execution`` so a Stop button
  619. still finds a run that a refused Run-now's newer row hides from ``latest_execution``."""
  620. clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
  621. if not clean:
  622. return {}
  623. placeholders = ",".join("?" for _ in clean)
  624. with _transaction() as conn:
  625. rows = conn.execute(
  626. f"""SELECT * FROM executions WHERE job_id IN ({placeholders}) AND status IN ('claimed', 'running')
  627. ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC""",
  628. clean,
  629. ).fetchall()
  630. live: dict[str, dict[str, Any]] = {}
  631. for row in rows:
  632. if row["job_id"] not in live and _row_owner_live(row):
  633. live[row["job_id"]] = dict(row)
  634. return live
  635. def latest_executions(job_ids: list[str]) -> dict[str, dict[str, Any]]:
  636. """Load latest execution for many jobs in one query."""
  637. clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
  638. if not clean:
  639. return {}
  640. placeholders = ",".join("?" for _ in clean)
  641. # One windowed sort: a per-row correlated ORDER BY julianday() cannot use the index and
  642. # grows quadratically with history (~90 ms at 1000 rows).
  643. with _transaction() as conn:
  644. rows = conn.execute(
  645. f"""SELECT e.* FROM executions e WHERE e.id IN (
  646. SELECT id FROM (
  647. SELECT id, ROW_NUMBER() OVER (
  648. PARTITION BY job_id
  649. ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC
  650. ) AS rn
  651. FROM executions WHERE job_id IN ({placeholders}))
  652. WHERE rn=1)""",
  653. clean,
  654. ).fetchall()
  655. return {row["job_id"]: dict(row) for row in rows}