| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755 |
- """Profile-local durable audit ledger for cron execution attempts.
- The ledger records what is known about each attempt; it is not a retry queue. Interrupted attempts
- become ``unknown`` only after their owner process is proved gone — a start-time reading that fails
- to match the claim-time fingerprint is not proof of death. Terminal states are immutable.
- """
- from __future__ import annotations
- import logging
- import math
- import os
- import sqlite3
- import threading
- import time
- import uuid
- from contextlib import contextmanager
- from datetime import datetime
- from pathlib import Path
- from typing import Any, Dict, Iterator, List, Optional
- from hermes_constants import get_hermes_home
- from hermes_time import now as _hermes_now
- from cron.constants import CLAIM_TTL_INACTIVITY_HEADROOM
- from hermes_cli.observability.shared_metrics_gateway import record_cron_finish
- logger = logging.getLogger(__name__)
- # Optional test override. Production resolves the path at transaction time so dashboard operations
- # that temporarily enter another profile cannot leak that profile's records into the import-time
- # home.
- EXECUTIONS_FILE: Optional[Path] = None
- MAX_TERMINAL_EXECUTIONS = 1000
- HANDOFF_ADOPTION_GRACE_SECONDS = 30.0
- # Floor for the live-owner stale-claim bound (#115692); see _live_owner_stale_after_seconds.
- LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS = 7200.0
- # ``cancelled``: an operator stopped the run (``hermes cron cancel``, removing the job mid-run).
- # ``missed``: never started — one row books every occurrence the scheduler skipped past.
- _TERMINAL_STATES = ("completed", "failed", "unknown", "cancelled", "missed")
- _STATUS_CHECK = "('claimed','running','completed','failed','unknown','cancelled','missed')"
- # Outcomes the attempt's own firer booked: receipt recovery, the external-worker waiter and the
- # dead-owner sweep must treat these as final, never replay or re-book them.
- FIRER_FINISHED_STATES = frozenset({"completed", "failed", "cancelled"})
- _lock = threading.RLock()
- _PROCESS_ID = uuid.uuid4().hex
- # --- executions ledger --------------------------------------------------------------------------
- def _connect() -> sqlite3.Connection:
- # Late imports: a scheduler daemon that outlives an on-disk upgrade already has the OLD
- # ``hermes_cli.sqlite_util`` / ``cron.jobs`` cached, so new names must be resolved at call time,
- # not at import time (the guarantee cron/ledger.py used to carry, see e24c8499).
- from cron.jobs import _ensure_cron_dir
- from hermes_cli.sqlite_util import open_db
- path = EXECUTIONS_FILE or (get_hermes_home().resolve() / "cron" / "executions.db")
- _ensure_cron_dir(path.parent)
- return open_db(path, db_label="cron/executions.db", synchronous_full=True, initialize=_initialize_schema)
- def _initialize_schema(conn: sqlite3.Connection) -> None:
- from hermes_cli.sqlite_util import add_column_if_missing
- conn.execute(
- f"""CREATE TABLE IF NOT EXISTS executions (
- id TEXT PRIMARY KEY,
- job_id TEXT NOT NULL,
- source TEXT NOT NULL,
- process_id TEXT NOT NULL,
- pid INTEGER NOT NULL,
- process_started_at INTEGER,
- status TEXT NOT NULL CHECK(status IN
- {_STATUS_CHECK}),
- handoff_pending INTEGER NOT NULL DEFAULT 0,
- handoff_started_at REAL,
- claimed_at TEXT NOT NULL,
- started_at TEXT,
- finished_at TEXT,
- error TEXT
- )"""
- )
- _widen_status_check(conn)
- add_column_if_missing(
- conn, "executions", "handoff_pending",
- "handoff_pending INTEGER NOT NULL DEFAULT 0",
- )
- add_column_if_missing(
- conn, "executions", "handoff_started_at", "handoff_started_at REAL"
- )
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_executions_job_claimed "
- "ON executions(job_id, claimed_at DESC, id DESC)"
- )
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_executions_status_claimed "
- "ON executions(status, claimed_at DESC, id DESC)"
- )
- add_column_if_missing(conn, "executions", "delivery_outcome", "delivery_outcome TEXT")
- add_column_if_missing(conn, "executions", "scheduled_instant", "scheduled_instant TEXT")
- add_column_if_missing(conn, "executions", "progress_at", "progress_at TEXT")
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_executions_occurrence "
- "ON executions(job_id, scheduled_instant) WHERE status='completed'"
- )
- def _widen_status_check(conn: sqlite3.Connection) -> None:
- """Rebuild a ledger created before ``cancelled``/``missed`` existed: SQLite cannot alter a
- CHECK constraint in place. Copies every column the old table has; runs once per file, and
- re-reads the schema under the write lock so two processes opening it together rebuild once."""
- def current_sql() -> Optional[str]:
- row = conn.execute(
- "SELECT sql FROM sqlite_master WHERE type='table' AND name='executions'").fetchone()
- return row[0] if row is not None else None
- if "'missed'" in (current_sql() or "'missed'"):
- return
- conn.commit()
- conn.execute("BEGIN IMMEDIATE")
- try:
- old_sql = current_sql() or "'missed'"
- if "'missed'" not in old_sql:
- columns = ", ".join(r[1] for r in conn.execute("PRAGMA table_info(executions)"))
- conn.execute("ALTER TABLE executions RENAME TO executions_pre_widen")
- conn.execute(old_sql.replace("'unknown')", "'unknown','cancelled','missed')", 1))
- conn.execute(
- f"INSERT INTO executions ({columns}) SELECT {columns} FROM executions_pre_widen")
- conn.execute("DROP TABLE executions_pre_widen")
- conn.commit()
- except BaseException:
- conn.rollback()
- raise
- @contextmanager
- def _transaction() -> Iterator[sqlite3.Connection]:
- from hermes_cli.sqlite_util import transaction
- with _lock, transaction(_connect()) as conn:
- yield conn
- def _fetch(conn: sqlite3.Connection, execution_id: str) -> Optional[dict[str, Any]]:
- row = conn.execute("SELECT * FROM executions WHERE id=?", (execution_id,)).fetchone()
- return dict(row) if row is not None else None
- def _emit_execution_state(
- record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None
- ) -> None:
- """Project durable state to monitoring without affecting ledger behavior."""
- try:
- from agent.monitoring.cron_health import emit_execution_state
- emit_execution_state(record, delivery_outcome=delivery_outcome)
- except Exception:
- pass
- def _process_start_time(pid: int) -> Optional[int]:
- try:
- from gateway.status import get_process_start_time
- return get_process_start_time(pid)
- except Exception:
- return None
- def _owner_is_live(pid: int, started_at: Optional[int]) -> bool:
- try:
- from gateway.status import _pid_exists
- if not _pid_exists(pid):
- return False
- except Exception:
- return True # fail safe: inability to prove death must not rewrite state
- if started_at is None:
- return pid == os.getpid()
- current = _process_start_time(pid)
- if current is None:
- return True # cannot compare -> cannot prove death; a misread must not rewrite state
- # Drifted same-host readings (#117505) are not proof of death; a live misread is still
- # bounded by the stale-claim sweep below.
- from gateway.status import start_time_fingerprints_match
- return start_time_fingerprints_match(started_at, current)
- def _live_owner_stale_after_seconds() -> Optional[float]:
- """Age past which a claimed/running row with a LIVE owner is treated as wedged.
- Derived from the existing knobs, never a bare wall-clock constant:
- ``max(3 × HERMES_CRON_TIMEOUT, cron script timeout, 7200)``. Returns ``None`` (never reclaim
- live owners — today's behaviour) when the inactivity timeout is 0/unlimited or not a finite
- positive number: with no bound to derive from, fail closed.
- """
- from cron.scheduler import _cron_inactivity_seconds
- from cron.scheduler_script import _get_script_timeout
- inactivity = float(_cron_inactivity_seconds())
- if not math.isfinite(inactivity) or inactivity <= 0:
- return None
- return max(
- inactivity * CLAIM_TTL_INACTIVITY_HEADROOM,
- float(_get_script_timeout()),
- LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS,
- )
- def _claim_age_seconds(claimed_at: str) -> float:
- """Seconds since ``claimed_at`` (NOT NULL, always the aware ISO string from hermes_time.now)."""
- return (_hermes_now() - datetime.fromisoformat(claimed_at)).total_seconds()
- def _stale_age_seconds(claimed_at: str, progress_at: Optional[str]) -> float:
- """Seconds since the owner last proved it was making progress.
- A wedged worker (#115692) stops stamping ``progress_at``; a long but healthy run keeps
- stamping it, so the derived stale bound measures silence, not run length. Rows written
- before the column existed have no stamp and fall back to claim age.
- """
- return _claim_age_seconds(progress_at or claimed_at)
- def touch_execution_progress(execution_id: str) -> bool:
- """Stamp ``progress_at`` on a running attempt this process owns. Returns False when the row
- is no longer ours or no longer running (the caller treats that as informational only)."""
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions SET progress_at=?
- WHERE id=? AND status IN ('claimed','running') AND process_id=? AND pid=?""",
- (now, execution_id, _PROCESS_ID, os.getpid()),
- )
- return cur.rowcount == 1
- def _prune_unlocked(conn: sqlite3.Connection) -> None:
- conn.execute(
- """DELETE FROM executions WHERE id IN (
- SELECT id FROM executions
- WHERE status IN ('completed','failed','unknown','cancelled','missed')
- ORDER BY julianday(finished_at) DESC, finished_at DESC,
- julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?
- )""",
- (max(0, int(MAX_TERMINAL_EXECUTIONS)),),
- )
- def create_execution(
- job_id: str, *, source: str, scheduled_instant: Optional[str] = None,
- ) -> dict[str, Any]:
- """Persist a claimed attempt before executor/provider dispatch."""
- from cron.occurrences import scheduled_instant as canonical_instant
- now = _hermes_now().isoformat()
- execution_id = uuid.uuid4().hex
- pid = os.getpid()
- with _transaction() as conn:
- conn.execute(
- """INSERT INTO executions
- (id, job_id, source, process_id, pid, process_started_at,
- status, claimed_at, scheduled_instant)
- VALUES (?, ?, ?, ?, ?, ?, 'claimed', ?, ?)""",
- (execution_id, str(job_id), str(source), _PROCESS_ID, pid,
- _process_start_time(pid), now, canonical_instant(scheduled_instant)),
- )
- record = _fetch(conn, execution_id)
- _emit_execution_state(record)
- return record # type: ignore[return-value]
- def set_execution_occurrence(execution_id: str, instant: Optional[str]) -> None:
- """Bind the store-claimed snapshot before a provider hands it to a worker."""
- from cron.occurrences import scheduled_instant
- with _transaction() as conn:
- cur = conn.execute(
- "UPDATE executions SET scheduled_instant=? WHERE id=? AND status='claimed' "
- "AND handoff_pending=0 AND process_id=? AND pid=?",
- (scheduled_instant(instant), execution_id, _PROCESS_ID, os.getpid()),
- )
- if cur.rowcount != 1:
- raise RuntimeError("Cron occurrence could not be bound before dispatch")
- def mark_execution_handoff_pending(execution_id: str) -> Optional[dict[str, Any]]:
- """Fence restart recovery while an external worker is adopting a claim."""
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions
- SET handoff_pending=1, handoff_started_at=?
- WHERE id=? AND status='claimed'
- AND process_id=? AND pid=?""",
- (time.time(), execution_id, _PROCESS_ID, os.getpid()),
- )
- if cur.rowcount != 1:
- return None
- record = _fetch(conn, execution_id)
- _emit_execution_state(record)
- return record
- def adopt_claimed_execution(execution_id: str) -> Optional[dict[str, Any]]:
- """Atomically transfer and start an attempt in its worker process.
- The dispatching gateway creates the row before spawning a restart-safe
- worker. Adoption is the single ``claimed`` → ``running`` gate: only the
- winner may acknowledge ownership or run side effects.
- """
- pid = os.getpid()
- process_started_at = _process_start_time(pid)
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions
- SET process_id=?, pid=?, process_started_at=?,
- status='running', started_at=?, handoff_pending=0,
- handoff_started_at=NULL
- WHERE id=? AND status='claimed' AND handoff_pending=1""",
- (_PROCESS_ID, pid, process_started_at, now, execution_id),
- )
- if cur.rowcount != 1:
- return None
- record = _fetch(conn, execution_id)
- _emit_execution_state(record)
- return record
- def mark_execution_running(execution_id: str) -> Optional[dict[str, Any]]:
- """Transition one claimed attempt to running exactly once."""
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions
- SET status='running', started_at=?, handoff_pending=0,
- handoff_started_at=NULL
- WHERE id=? AND status='claimed' AND handoff_pending=0
- AND process_id=? AND pid=?""",
- (now, execution_id, _PROCESS_ID, os.getpid()),
- )
- if cur.rowcount != 1:
- return None
- record = _fetch(conn, execution_id)
- _emit_execution_state(record)
- return record
- def finish_execution(
- execution_id: str, *, success: bool, error: Optional[str] = None,
- delivery_outcome: Optional[str] = None, cancelled: bool = False,
- ) -> Optional[dict[str, Any]]:
- """Write a terminal result once; terminal attempts cannot be rewritten.
- Only the owning process may finish its attempt; receipt recovery for a departed owner
- settles through ``recover_receipted_execution``.
- """
- now = _hermes_now().isoformat()
- status = "cancelled" if cancelled else ("completed" if success else "failed")
- detail = None if success else (str(error) if error else "unknown failure")
- with _transaction() as conn:
- owner = (_PROCESS_ID, os.getpid())
- cur = conn.execute(
- """UPDATE executions
- SET status=?, finished_at=?, error=?, handoff_pending=0,
- handoff_started_at=NULL, delivery_outcome=?
- WHERE id=? AND status IN ('claimed','running')
- AND process_id=? AND pid=?""",
- (status, now, detail, delivery_outcome, execution_id, *owner),
- )
- if cur.rowcount != 1:
- return None
- _prune_unlocked(conn)
- record = _fetch(conn, execution_id)
- _emit_execution_state(record, delivery_outcome=delivery_outcome)
- record_cron_finish(record, delivery_outcome)
- return record
- def record_missed_occurrences(
- job_id: str, *, count: int, first_instant: Optional[str], detail: str,
- ) -> Optional[dict[str, Any]]:
- """Book occurrences the scheduler passed over without running as ONE terminal ``missed`` row
- (``missed ×N``), so run history shows the gap instead of nothing. Never started, so no
- owner fencing applies; ``first_instant`` is the earliest skipped slot."""
- from cron.occurrences import scheduled_instant as canonical_instant
- if count < 1:
- return None
- now = _hermes_now().isoformat()
- execution_id = uuid.uuid4().hex
- pid = os.getpid()
- with _transaction() as conn:
- conn.execute(
- """INSERT INTO executions
- (id, job_id, source, process_id, pid, process_started_at,
- status, claimed_at, finished_at, error, scheduled_instant)
- VALUES (?, ?, 'scheduler', ?, ?, ?, 'missed', ?, ?, ?, ?)""",
- (execution_id, str(job_id), _PROCESS_ID, pid, _process_start_time(pid), now, now,
- f"missed \u00d7{count} ({detail})", canonical_instant(first_instant)),
- )
- _prune_unlocked(conn)
- record = _fetch(conn, execution_id)
- _emit_execution_state(record)
- return record
- def recover_receipted_execution(execution_id, job_id, *, success, error=None, delivery_outcome=None):
- """Settle a verified canonical receipt after owner loss, including its unknown audit row.
- Terminal verdicts stay immutable. A live firer still owns its bookkeeping
- (``execution_owner_live``).
- """
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- row = _recoverable_row(conn, execution_id, job_id)
- if row is None or _row_owner_live(row):
- return None
- conn.execute(
- """UPDATE executions SET status=?, finished_at=?, error=?, handoff_pending=0,
- handoff_started_at=NULL, delivery_outcome=? WHERE id=?""",
- ('completed' if success else 'failed', now, None if success else (str(error) if error else 'unknown failure'),
- delivery_outcome, execution_id))
- _prune_unlocked(conn)
- record = _fetch(conn, execution_id)
- _emit_execution_state(record, delivery_outcome=delivery_outcome)
- record_cron_finish(record, delivery_outcome)
- return record
- def _recoverable_row(conn, execution_id, job_id):
- return conn.execute(
- """SELECT status, process_id, pid, process_started_at FROM executions
- WHERE id=? AND job_id=? AND status IN ('claimed','running','unknown')""",
- (str(execution_id), str(job_id))).fetchone()
- def _row_owner_live(row) -> bool:
- if row['process_id'] == _PROCESS_ID:
- # The in-gateway ticker fires on pool threads: a claimed/running row of THIS process is
- # still in flight here. Only an unknown row was given up by its own firer.
- return row['status'] != 'unknown'
- return _owner_is_live(int(row['pid']), row['process_started_at'])
- def execution_owner_live(execution_id: str, job_id: str) -> bool:
- """True while the attempt's own firer (this process included) can still finish it, so receipt
- recovery must not act for it: no output re-save, no job mark, no ledger write."""
- with _transaction() as conn:
- row = _recoverable_row(conn, execution_id, job_id)
- return row is not None and _row_owner_live(row)
- def settle_delivery_outcome(execution_id: str, outcome: str) -> bool:
- """Replace a terminal attempt's ``queued`` delivery outcome with the drain's real one.
- The only post-terminal rewrite the ledger allows: the attempt's status is untouched and
- only a ``queued`` handoff can settle, so a replay or a late caller is a no-op.
- """
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions SET delivery_outcome=?
- WHERE id=? AND delivery_outcome='queued'
- AND status IN ('completed','failed','unknown')""",
- (outcome, execution_id),
- )
- return cur.rowcount == 1
- _OWNER_GONE_REASON = (
- "Scheduler restarted after this execution's owner exited before a durable "
- "terminal state; whether side effects ran is unknown."
- )
- _OWNER_WEDGED_REASON = (
- "Owner process is still alive but the claim outlived the derived stale bound; "
- "treated as wedged (#115692). The process was not terminated; whether side effects "
- "ran is unknown."
- )
- def settle_unstarted_execution(execution_id: str, job_id: str, error: str) -> None:
- """Close the receipt of a run that never started: a ``claimed`` row never resolves. Best-effort
- so a ledger write cannot mask the failure the caller logs or re-raises."""
- try:
- finish_execution(execution_id, success=False, error=error)
- except (sqlite3.Error, OSError) as record_err:
- logger.error("Job '%s': failed to close execution receipt %s (%s): %s",
- job_id, execution_id, error, record_err)
- def recover_interrupted_executions() -> int:
- """Mark abandoned attempts unknown without scheduling retries: rows whose owner is provably
- dead, plus rows whose live owner holds a claim older than the derived stale bound (the
- process is not killed)."""
- now = _hermes_now().isoformat()
- changed = 0
- recovered: list[dict[str, Any]] = []
- # Derived on the first live-owned row only: the bound reads config, and the idle gateway
- # tick must stay config-free (tests/cron/test_idle_tick_config_skip.py).
- stale_after: Optional[float] = None
- stale_after_resolved = False
- with _transaction() as conn:
- rows = conn.execute(
- """SELECT id, status, process_id, pid, process_started_at,
- handoff_pending, handoff_started_at, claimed_at, progress_at
- FROM executions
- WHERE status IN ('claimed','running')"""
- ).fetchall()
- for row in rows:
- if row["process_id"] == _PROCESS_ID:
- continue
- reason = _OWNER_GONE_REASON
- if _owner_is_live(int(row["pid"]), row["process_started_at"]):
- # A live owner is normally a legitimately running job. A worker permanently
- # deadlocked (e.g. futex_wait behind a route/proxy flip, #115692) also passes
- # this check, so a claim SILENT for longer than the derived bound is treated
- # as wedged and released — the external-worker wait loop polls this ledger
- # for a terminal status, so the job can fire again. Silence is measured from
- # the owner's last ``progress_at`` stamp (the run monitor refreshes it while
- # the agent is active), so a healthy multi-hour run is never reclaimed while
- # it is still working. The wedged worker PROCESS is NOT terminated here
- # (leaked until host restart); rows owned by this process (process_id ==
- # _PROCESS_ID, in-process runs) are skipped above and remain out of scope.
- if not stale_after_resolved:
- stale_after = _live_owner_stale_after_seconds()
- stale_after_resolved = True
- if (
- stale_after is None
- or _stale_age_seconds(row["claimed_at"], row["progress_at"]) <= stale_after
- ):
- continue
- reason = _OWNER_WEDGED_REASON
- handoff_started_at = row["handoff_started_at"]
- if (
- row["handoff_pending"]
- and handoff_started_at is not None
- and time.time() - float(handoff_started_at)
- < HANDOFF_ADOPTION_GRACE_SECONDS
- ):
- continue
- # The SELECT ran outside a write transaction and the liveness probe takes time: the
- # owner may have stamped progress since, so act only on the snapshot that was judged.
- cur = conn.execute(
- """UPDATE executions
- SET status='unknown', finished_at=?, error=?,
- handoff_pending=0, handoff_started_at=NULL
- WHERE id=? AND status=? AND process_id=? AND pid=?
- AND handoff_pending=?
- AND handoff_started_at IS ?
- AND claimed_at=? AND progress_at IS ?""",
- (now, reason, row["id"], row["status"], row["process_id"], row["pid"],
- row["handoff_pending"], row["handoff_started_at"],
- row["claimed_at"], row["progress_at"]),
- )
- changed += cur.rowcount
- if cur.rowcount:
- record = _fetch(conn, row["id"])
- if record is not None:
- recovered.append(record)
- if changed:
- _prune_unlocked(conn)
- for record in recovered:
- _emit_execution_state(record)
- return changed
- def terminalize_dead_owner(execution_id: str, *, reason: str) -> bool:
- """Record one attempt as ``unknown`` with a cause this process actually observed.
- ``recover_interrupted_executions`` sweeps every attempt whose owner is provably
- dead, and knows nothing but that absence — so all it can write is
- ``_OWNER_GONE_REASON``, which asserts a scheduler restart. A waiter that held the
- worker's ``Popen`` knows more: the owner was that external worker, and it exited
- with a known status. Without this, a manual run whose worker dies is filed as
- "Scheduler restarted ..." (a restart that never happened) and, because the sweep
- leaves the row terminal, the waiter reports success and never records the run —
- the job's ``fire_claim`` then blocks the next manual fire for the whole lease
- (#128509).
- The attempt stays ``unknown``, not ``failed``: whether side effects ran is still
- unknown. Only the CAUSE becomes truthful. Returns False — leaving the caller to
- fall back to the generic sweep — when the row is absent, already terminal, owned by
- this process, inside the handoff adoption grace, or owned by a live process: a
- worker that is still running must never be terminalized out from under itself.
- """
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- row = conn.execute(
- """SELECT id, status, process_id, pid, process_started_at,
- handoff_pending, handoff_started_at
- FROM executions WHERE id=?""",
- (execution_id,),
- ).fetchone()
- if row is None or row["status"] not in ("claimed", "running"):
- return False
- if row["process_id"] == _PROCESS_ID:
- return False
- if _owner_is_live(int(row["pid"]), row["process_started_at"]):
- return False
- handoff_started_at = row["handoff_started_at"]
- if (
- row["handoff_pending"]
- and handoff_started_at is not None
- and time.time() - float(handoff_started_at) < HANDOFF_ADOPTION_GRACE_SECONDS
- ):
- return False
- cur = conn.execute(
- """UPDATE executions
- SET status='unknown', finished_at=?, error=?,
- handoff_pending=0, handoff_started_at=NULL
- WHERE id=? AND status=? AND process_id=? AND pid=?""",
- (now, reason, row["id"], row["status"], row["process_id"], row["pid"]),
- )
- if cur.rowcount != 1:
- return False
- record = _fetch(conn, execution_id)
- _prune_unlocked(conn)
- _emit_execution_state(record)
- return True
- def give_up_execution(execution_id: str, *, reason: str) -> bool:
- """The firer itself gave up on its in-flight attempt (``CronExecutionUnknown``): record it
- ``unknown`` now, owner-fenced to this process, instead of leaving it ``running`` until a
- restart's dead-owner sweep. Receipt recovery never acts for a same-process running row, so
- the owner's later terminal receipt can settle it only once it is marked given up."""
- now = _hermes_now().isoformat()
- with _transaction() as conn:
- cur = conn.execute(
- """UPDATE executions SET status='unknown', finished_at=?, error=?,
- handoff_pending=0, handoff_started_at=NULL
- WHERE id=? AND status IN ('claimed','running') AND process_id=? AND pid=?""",
- (now, reason, str(execution_id), _PROCESS_ID, os.getpid()))
- if cur.rowcount != 1:
- return False
- record = _fetch(conn, str(execution_id))
- _prune_unlocked(conn)
- _emit_execution_state(record)
- return True
- def list_executions(
- *, job_id: Optional[str] = None, limit: int = 50, before_claimed_at: Optional[str] = None,
- ) -> list[dict[str, Any]]:
- """Return indexed, newest-first execution history with cursor pagination."""
- clauses: list[str] = []
- params: list[Any] = []
- if job_id is not None:
- clauses.append("job_id=?")
- params.append(str(job_id))
- if before_claimed_at is not None:
- # Same (instant, text) key as the ORDER BY, so a page never skips or repeats a row.
- clauses.append("(julianday(claimed_at), claimed_at) < (julianday(?), ?)")
- params.extend([str(before_claimed_at)] * 2)
- where = " WHERE " + " AND ".join(clauses) if clauses else ""
- params.append(max(1, min(int(limit), 500)))
- # Stamps carry the local offset, which changes at DST and on a timezone change, so text order
- # is not time order. julianday() compares instants (ms); the text breaks same-ms ties.
- with _transaction() as conn:
- rows = conn.execute(
- "SELECT * FROM executions" + where
- + " ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT ?",
- params,
- ).fetchall()
- return [dict(row) for row in rows]
- def get_execution(execution_id: str) -> Optional[dict[str, Any]]:
- """Return one exact execution attempt, or ``None`` when it is absent."""
- with _transaction() as conn:
- row = conn.execute(
- "SELECT * FROM executions WHERE id=?",
- (str(execution_id),),
- ).fetchone()
- return dict(row) if row is not None else None
- def latest_execution(job_id: str) -> Optional[dict[str, Any]]:
- rows = list_executions(job_id=job_id, limit=1)
- return rows[0] if rows else None
- def live_inflight_execution(job_id: str) -> Optional[dict[str, Any]]:
- """The job's latest attempt while it is still claimed/running under a LIVE owner, else ``None``.
- This is scheduler OWNERSHIP, not recent activity: a run inside a long tool call writes no
- heartbeat yet stays owned, while a run whose process died (watchdog kill, crash) does not.
- Read-only — unlike ``recover_interrupted_executions`` it never rewrites a row.
- """
- # A ``missed`` row is the due scan's note about skipped slots, not an attempt: one booked while
- # a long run is still live (machine slept through its next slots) must not hide that run.
- with _transaction() as conn:
- row = conn.execute(
- """SELECT * FROM executions WHERE job_id=? AND status != 'missed'
- ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT 1""",
- (job_id,),
- ).fetchone()
- record = dict(row) if row is not None else None
- if not record or record.get("status") not in ("claimed", "running"):
- return None
- if not _owner_is_live(int(record["pid"]), record.get("process_started_at")):
- return None
- return record
- def cancellable_execution(job_id: str) -> Optional[dict[str, Any]]:
- """The job's newest claimed/running attempt under a LIVE owner, even when it is not the latest row.
- A Run-now refused while a run is live books its own newer ``failed`` row ("Fire claim was not
- acquired"), so ``live_inflight_execution`` (latest row only) would call the live run idle and a
- cancel would never reach it.
- """
- return live_executions([job_id]).get(str(job_id))
- def live_executions(job_ids: list[str]) -> dict[str, dict[str, Any]]:
- """Each job's newest claimed/running attempt under a live owner, in one query (see
- ``cancellable_execution``). The job list carries it as ``live_execution`` so a Stop button
- still finds a run that a refused Run-now's newer row hides from ``latest_execution``."""
- clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
- if not clean:
- return {}
- placeholders = ",".join("?" for _ in clean)
- with _transaction() as conn:
- rows = conn.execute(
- f"""SELECT * FROM executions WHERE job_id IN ({placeholders}) AND status IN ('claimed', 'running')
- ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC""",
- clean,
- ).fetchall()
- live: dict[str, dict[str, Any]] = {}
- for row in rows:
- if row["job_id"] not in live and _row_owner_live(row):
- live[row["job_id"]] = dict(row)
- return live
- def latest_executions(job_ids: list[str]) -> dict[str, dict[str, Any]]:
- """Load latest execution for many jobs in one query."""
- clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
- if not clean:
- return {}
- placeholders = ",".join("?" for _ in clean)
- # One windowed sort: a per-row correlated ORDER BY julianday() cannot use the index and
- # grows quadratically with history (~90 ms at 1000 rows).
- with _transaction() as conn:
- rows = conn.execute(
- f"""SELECT e.* FROM executions e WHERE e.id IN (
- SELECT id FROM (
- SELECT id, ROW_NUMBER() OVER (
- PARTITION BY job_id
- ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC
- ) AS rn
- FROM executions WHERE job_id IN ({placeholders}))
- WHERE rn=1)""",
- clean,
- ).fetchall()
- return {row["job_id"]: dict(row) for row in rows}
|