api_server_run_idempotency.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. """Durable idempotency reservations for API server runs."""
  2. import hmac
  3. import json
  4. import logging
  5. import sqlite3
  6. import threading
  7. import time
  8. from contextlib import contextmanager
  9. from pathlib import Path
  10. from typing import Any, Dict
  11. from hermes_cli.sqlite_util import add_column_if_missing
  12. # Keep the extracted store's log records on the API server logger.
  13. logger = logging.getLogger("gateway.platforms.api_server")
  14. TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "interrupted"})
  15. _SELECT_BY_KEY = (
  16. "SELECT fingerprint, run_id, status_json, owner_pid, owner_started, updated_at "
  17. "FROM run_idempotency WHERE scope=? AND idempotency_key=?")
  18. _EXTEND_RETENTION_BY_KEY = (
  19. "UPDATE run_idempotency SET retention_until=MAX(retention_until, ?) "
  20. "WHERE scope=? AND idempotency_key=? AND fingerprint=?")
  21. _EXTEND_RETENTION_BY_RUN = (
  22. "UPDATE run_idempotency SET retention_until=MAX(retention_until, ?) "
  23. "WHERE scope=? AND run_id=?")
  24. # Columns added after the first schema shipped; applied when missing.
  25. _MIGRATIONS = {
  26. "owner_pid": "INTEGER NOT NULL DEFAULT 0",
  27. "owner_started": "INTEGER NOT NULL DEFAULT 0",
  28. "retention_until": "REAL NOT NULL DEFAULT 0",
  29. "acknowledged_at": "REAL"}
  30. def _encode_status(status: dict[str, Any]) -> str:
  31. return json.dumps(status, sort_keys=True, separators=(",", ":"))
  32. def _record(run_id, status_json, owner_pid, owner_started, updated_at) -> dict[str, Any]:
  33. return {
  34. "run_id": run_id, "status": json.loads(status_json), "owner_pid": int(owner_pid or 0),
  35. "owner_started": int(owner_started or 0), "updated_at": float(updated_at or 0)}
  36. def _outcome(row, fingerprint):
  37. """Classify a stored ``(scope, key)`` row against the caller's fingerprint."""
  38. return ("reused" if hmac.compare_digest(row[0], fingerprint) else "conflict"), _record(*row[1:])
  39. class RunIdempotencyStore:
  40. """Durable, tenant-scoped reservations for ``POST /v1/runs``: a unique ``(scope, key)`` row
  41. inserted inside ``BEGIN IMMEDIATE`` so separate workers cannot both admit one request. Only
  42. fingerprints and public run status are stored — never request bodies or credentials."""
  43. RETENTION_SECONDS = 24 * 60 * 60
  44. ACKNOWLEDGED_RETENTION_SECONDS = 24 * 60 * 60
  45. @property
  46. def durable(self) -> bool:
  47. """Whether reservations survive this process."""
  48. return self._db_path is not None
  49. def __init__(self, db_path: str | None = None):
  50. if db_path is None:
  51. try:
  52. from hermes_cli.config import get_hermes_home
  53. db_path = str(get_hermes_home() / "runs_idempotency.db")
  54. except Exception:
  55. db_path = ":memory:"
  56. self._db_path = None if db_path == ":memory:" else db_path
  57. try:
  58. self._conn = sqlite3.connect(db_path, check_same_thread=False, timeout=30)
  59. except Exception as exc:
  60. # Docker may create the container object before `docker run` fails to start it (e.g. exit code
  61. # 125 when the daemon isn't ready, or a timeout mid-pull). That orphan is left in "Created"
  62. # state — which the exited-only orphan reaper (reap_orphan_containers, status=exited) never
  63. # catches, so it leaks permanently. Remove it by its known name before re-raising. See #7439.
  64. logger.warning(
  65. "Run idempotency storage is unavailable; falling back to "
  66. "process memory, so replay will not survive a restart: %s", exc)
  67. self._conn = sqlite3.connect(":memory:", check_same_thread=False)
  68. self._db_path = None
  69. from hermes_state_wal import apply_wal_with_fallback
  70. apply_wal_with_fallback(self._conn, db_label="runs_idempotency.db")
  71. self._conn.execute(
  72. """CREATE TABLE IF NOT EXISTS run_idempotency (
  73. scope TEXT NOT NULL,
  74. idempotency_key TEXT NOT NULL,
  75. fingerprint TEXT NOT NULL,
  76. run_id TEXT NOT NULL,
  77. status_json TEXT NOT NULL,
  78. owner_pid INTEGER NOT NULL DEFAULT 0,
  79. owner_started INTEGER NOT NULL DEFAULT 0,
  80. retention_until REAL NOT NULL DEFAULT 0,
  81. acknowledged_at REAL,
  82. created_at REAL NOT NULL,
  83. updated_at REAL NOT NULL,
  84. PRIMARY KEY (scope, idempotency_key)
  85. )"""
  86. )
  87. columns = {str(row[1]) for row in self._conn.execute("PRAGMA table_info(run_idempotency)")}
  88. for column, ddl in _MIGRATIONS.items():
  89. if column not in columns:
  90. add_column_if_missing(self._conn, "run_idempotency", column, f"{column} {ddl}")
  91. self._conn.execute(
  92. "CREATE UNIQUE INDEX IF NOT EXISTS run_idempotency_run_id ON run_idempotency(run_id)")
  93. self._conn.commit()
  94. self._lock = threading.Lock()
  95. self._tighten_permissions()
  96. def _tighten_permissions(self) -> None:
  97. for suffix in ("", "-wal", "-shm") if self._db_path else ():
  98. candidate = Path(self._db_path + suffix)
  99. try:
  100. if candidate.exists():
  101. candidate.chmod(0o600)
  102. except OSError:
  103. logger.debug("Failed to restrict run idempotency store permissions", exc_info=True)
  104. @contextmanager
  105. def _immediate_txn(self):
  106. """Hold the lock inside ``BEGIN IMMEDIATE``; the body commits, errors roll back."""
  107. with self._lock:
  108. self._conn.execute("BEGIN IMMEDIATE")
  109. try:
  110. yield
  111. except Exception:
  112. self._conn.rollback()
  113. raise
  114. def reserve(self, scope: str, key: str, fingerprint: str, run_id: str, status: dict[str, Any], *,
  115. owner_pid: int = 0, owner_started: int = 0, retention_until: float = 0):
  116. """Atomically reserve a key; return ``(outcome, stored_record)``."""
  117. now = time.time()
  118. retention_until = max(0.0, float(retention_until or 0))
  119. encoded = _encode_status(status)
  120. with self._immediate_txn():
  121. self._prune_stale_terminal_locked(now)
  122. row = self._conn.execute(_SELECT_BY_KEY, (scope, key)).fetchone()
  123. if row is not None:
  124. if retention_until:
  125. self._conn.execute(_EXTEND_RETENTION_BY_KEY, (retention_until, scope, key, fingerprint))
  126. self._conn.commit()
  127. return _outcome(row, fingerprint)
  128. self._conn.execute(
  129. "INSERT INTO run_idempotency("
  130. "scope,idempotency_key,fingerprint,run_id,status_json,"
  131. "owner_pid,owner_started,retention_until,created_at,updated_at"
  132. ") VALUES(?,?,?,?,?,?,?,?,?,?)",
  133. (scope, key, fingerprint, run_id, encoded, int(owner_pid or 0), int(owner_started or 0),
  134. retention_until, now, now))
  135. self._conn.commit()
  136. return "created", _record(run_id, encoded, owner_pid, owner_started, now) | {"status": status}
  137. def lookup(self, scope: str, key: str, fingerprint: str, *, retention_until: float = 0):
  138. """Return ``missing``, ``reused`` or ``conflict`` without reserving."""
  139. now = time.time()
  140. retention_until = max(0.0, float(retention_until or 0))
  141. with self._immediate_txn():
  142. if retention_until:
  143. self._conn.execute(_EXTEND_RETENTION_BY_KEY, (retention_until, scope, key, fingerprint))
  144. self._prune_stale_terminal_locked(now)
  145. row = self._conn.execute(_SELECT_BY_KEY, (scope, key)).fetchone()
  146. self._conn.commit()
  147. return ("missing", None) if row is None else _outcome(row, fingerprint)
  148. def _prune_stale_terminal_locked(self, now: float) -> None:
  149. """Prune aged replay records only once their stored run is terminal (caller holds the
  150. lock + transaction): a long or disconnected room turn may outlive the retention window."""
  151. stale = self._conn.execute(
  152. """SELECT scope, idempotency_key, status_json
  153. FROM run_idempotency
  154. WHERE acknowledged_at <= ?
  155. OR (retention_until > 0 AND retention_until <= ?)
  156. OR (retention_until <= 0 AND updated_at < ?)""",
  157. (now - self.ACKNOWLEDGED_RETENTION_SECONDS, now, now - self.RETENTION_SECONDS),
  158. ).fetchall()
  159. for stale_scope, stale_key, stale_status in stale:
  160. try:
  161. terminal = json.loads(stale_status).get("status") in TERMINAL_STATUSES
  162. except Exception:
  163. terminal = False
  164. if terminal:
  165. self._conn.execute(
  166. "DELETE FROM run_idempotency WHERE scope=? AND idempotency_key=?", (stale_scope, stale_key))
  167. def status_for_run(self, scope: str, run_id: str, *, retention_until: float = 0) -> dict[str, Any] | None:
  168. """Load one durable run status inside its authenticated scope."""
  169. retention_until = max(0.0, float(retention_until or 0))
  170. with self._lock:
  171. if retention_until:
  172. self._conn.execute(_EXTEND_RETENTION_BY_RUN, (retention_until, scope, run_id))
  173. self._conn.commit()
  174. row = self._conn.execute(
  175. "SELECT status_json, owner_pid, owner_started, updated_at "
  176. "FROM run_idempotency WHERE scope=? AND run_id=?",
  177. (scope, run_id)).fetchone()
  178. if row is None:
  179. return None
  180. return {k: v for k, v in _record(None, *row).items() if k != "run_id"}
  181. def extend_retention(self, scope: str, run_id: str, until: float) -> bool:
  182. """Persist the latest verified recovery horizon for an active grant."""
  183. checked_until = max(0.0, float(until or 0))
  184. if not checked_until:
  185. return False
  186. with self._lock:
  187. changed = self._conn.execute(_EXTEND_RETENTION_BY_RUN, (checked_until, scope, run_id)).rowcount
  188. self._conn.commit()
  189. return changed == 1
  190. def owns_run(self, scope: str, run_id: str) -> bool:
  191. with self._lock:
  192. row = self._conn.execute(
  193. "SELECT 1 FROM run_idempotency WHERE scope=? AND run_id=?", (scope, run_id)).fetchone()
  194. return row is not None
  195. def update_status(self, run_id: str, status: dict[str, Any]) -> None:
  196. with self._lock:
  197. self._conn.execute(
  198. "UPDATE run_idempotency SET status_json=?, updated_at=? WHERE run_id=?",
  199. (_encode_status(status), time.time(), run_id))
  200. self._conn.commit()
  201. def forget(self, scope: str, key: str) -> None:
  202. """Release a reservation whose run was refused before it existed."""
  203. with self._lock:
  204. self._conn.execute("DELETE FROM run_idempotency WHERE scope=? AND idempotency_key=?", (scope, key))
  205. self._conn.commit()
  206. def close(self) -> None:
  207. with self._lock:
  208. self._conn.close()