hosted_room_driver.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916
  1. """Durable execution state for a same-gateway hosted room driver.
  2. Owns only the driver lease and task state machine: no model calls, no sessions, no dependency on the
  3. hosted-room event log. Callers supply the database path and clock so recovery and fencing are testable
  4. without process-global state.
  5. """
  6. from __future__ import annotations
  7. import dataclasses
  8. import hashlib
  9. import json
  10. import math
  11. import sqlite3
  12. from contextlib import closing
  13. from collections.abc import Mapping
  14. from dataclasses import dataclass
  15. from functools import partial
  16. from typing import Any, Callable, Literal, get_args
  17. from gateway.hosted_rooms_common import (
  18. DbPath, bounded_int, canonical_json, compact_json, connect, fenced_update, identifier, table_columns, text,
  19. transaction)
  20. Clock = Callable[[], float]
  21. TaskStatus = Literal["queued", "running", "settled", "failed", "cancelled", "indeterminate", "deferred", "stopping"]
  22. TerminalStatus = Literal["settled", "failed"]
  23. MAX_IDENTIFIER_CHARS = 128
  24. MAX_PROMPT_BYTES = 128 * 1024
  25. MAX_RESULT_JSON_BYTES = 256 * 1024
  26. TERMINAL_TASK_RETENTION_SECONDS = 30 * 24 * 60 * 60
  27. MAX_RETAINED_TERMINAL_TASKS = 2048
  28. MAX_TASK_PRUNE_BATCH = 1000
  29. TASK_STATUSES = frozenset(get_args(TaskStatus))
  30. TERMINAL_STATUSES = frozenset({"settled", "failed", "cancelled"})
  31. _TASK_PAYLOAD_REQUIRED_FIELDS = frozenset({"target_profile", "prompt", "source_event_seq"})
  32. _TASK_PAYLOAD_OPTIONAL_FIELDS = frozenset({"target_member_id", "input_context", "attachments", "lead_routing"})
  33. _LEASE_COLUMNS = frozenset({
  34. "room_id", "gateway_id", "authority_epoch", "process_generation", "lease_generation", "expires_at", "acquired_at",
  35. "updated_at", "released_at"})
  36. _TASK_COLUMN_ORDER = (
  37. "room_id", "task_id", "thread_id", "turn_id", "source_event_seq", "payload_json", "payload_digest", "status",
  38. "execution_generation", "cancel_generation", "run_gateway_id", "run_process_generation", "run_lease_generation",
  39. "cancel_id", "settlement_id", "settlement_status", "result_json", "created_at", "updated_at", "started_at",
  40. "terminal_at", "indeterminate_at")
  41. _TASK_COLUMNS = frozenset(_TASK_COLUMN_ORDER)
  42. _TASK_ORDER = "ORDER BY source_event_seq, created_at, task_id"
  43. _SELECT_LEASE = "SELECT * FROM hosted_room_driver_leases WHERE room_id=?"
  44. _SELECT_TASK = "SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND task_id=?"
  45. _TASK_INDEX_SQL = """CREATE INDEX {if_not_exists}idx_hosted_room_driver_tasks_status
  46. ON hosted_room_driver_tasks(room_id, status, source_event_seq, created_at, task_id)"""
  47. # --- Fenced task UPDATE statements (one per state-machine transition) ---------
  48. # Every transition is "UPDATE ... SET <set> WHERE room_id=? AND task_id=? AND <fence>"; the fence names the
  49. # expected status plus the generations that must not have moved.
  50. _GENERATION_FENCE = "execution_generation=? AND cancel_generation=?"
  51. _RUN_FENCE = "run_gateway_id=? AND run_process_generation=? AND run_lease_generation=?"
  52. _SETTLE_SET = "status=?, settlement_id=?, settlement_status=?, result_json=?, terminal_at=?, updated_at=?"
  53. _REQUEUE_SET = "status='queued', run_gateway_id=NULL, run_process_generation=NULL, run_lease_generation=NULL"
  54. _CANCEL_SET = "status='cancelled', cancel_generation=?, cancel_id=?, terminal_at=?, updated_at=?"
  55. def _task_update(set_clause: str, fence: str) -> str:
  56. return f"UPDATE hosted_room_driver_tasks SET {set_clause} WHERE room_id=? AND task_id=? AND {fence}"
  57. def _generation_update(set_clause: str, status: str) -> str:
  58. """Transition fenced on ``status`` + both generations (terminal settlements and the recovery family)."""
  59. return _task_update(set_clause, f"status='{status}' AND {_GENERATION_FENCE}")
  60. _SETTLE_RUNNING_SQL = _generation_update(_SETTLE_SET, "running") + f" AND {_RUN_FENCE}"
  61. _SETTLE_STOPPING_SQL = _generation_update(_SETTLE_SET, "stopping")
  62. _REQUEUE_RUNNING_SQL = _task_update(
  63. f"{_REQUEUE_SET}, started_at=NULL, updated_at=?", f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}")
  64. _CANCEL_QUEUED_SQL = _task_update(_CANCEL_SET, "status IN ('queued', 'deferred') AND cancel_generation=?")
  65. _BEGIN_STOP_SQL = _task_update(
  66. "status='stopping', cancel_generation=?, cancel_id=?, updated_at=?",
  67. "status IN ('running', 'indeterminate') AND cancel_generation=?")
  68. _COMPLETE_STOP_SQL = _task_update(
  69. "status='cancelled', terminal_at=?, updated_at=?", "status='stopping' AND cancel_id=? AND cancel_generation=?")
  70. # Lease-first recovery transitions: name -> (fenced status, SET clause, generation-guard stale message,
  71. # row stale message); the UPDATE is _generation_update(set_clause, status).
  72. _INDETERMINATE_STALE = "indeterminate task generation changed"
  73. _GENERATION_TRANSITIONS = {
  74. "resolve": ("indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation"),
  75. "resolve_cancel": (
  76. "indeterminate", _CANCEL_SET, "indeterminate cancellation proof is stale",
  77. "indeterminate cancellation proof lost its fence"),
  78. "requeue": (
  79. "indeterminate", f"{_REQUEUE_SET}, started_at=NULL, indeterminate_at=NULL, updated_at=?", _INDETERMINATE_STALE,
  80. "indeterminate task changed during requeue"),
  81. "defer": (
  82. "indeterminate", "status='deferred', result_json=?, terminal_at=?, updated_at=?", _INDETERMINATE_STALE,
  83. "indeterminate task changed during deferral"),
  84. "requeue_deferred": (
  85. "deferred",
  86. f"{_REQUEUE_SET}, result_json=NULL, started_at=NULL, terminal_at=NULL, indeterminate_at=NULL, updated_at=?",
  87. "deferred task generation changed", "deferred task changed during requeue")}
  88. class DriverStateError(ValueError): """Base class for invalid or conflicting driver-state operations."""
  89. class DriverValidationError(DriverStateError): """Raised when an identifier, clock, TTL, or payload is invalid."""
  90. class RoomUnavailableError(DriverStateError): """Raised when the hosted room does not exist or was disbanded."""
  91. class LeaseHeldError(DriverStateError): """Raised when another unexpired driver generation owns the room."""
  92. class StaleLeaseError(DriverStateError): """Raised when a lease generation can no longer mutate room state."""
  93. class TaskConflictError(DriverStateError): """Raised when an idempotency key is reused for different task state."""
  94. class StaleTaskError(DriverStateError): """Raised when an obsolete task attempt or cancellation tries to commit."""
  95. class InvalidTaskTransitionError(DriverStateError): """Raised when a requested task transition is not allowed."""
  96. _identifier = partial(identifier, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS)
  97. _bounded_int = partial(bounded_int, error=DriverValidationError)
  98. _canonical_json = partial(
  99. canonical_json, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES, ensure_ascii=True)
  100. def _finite(compute: Callable[[], Any], message: str, *, positive: bool = False) -> float:
  101. try:
  102. value = float(compute())
  103. except (TypeError, ValueError, OverflowError) as exc:
  104. raise DriverValidationError(message) from exc
  105. if not math.isfinite(value) or (positive and value <= 0):
  106. raise DriverValidationError(message)
  107. return value
  108. def _timestamp(clock: Clock) -> float:
  109. if not callable(clock):
  110. raise DriverValidationError("clock must be callable")
  111. return _finite(clock, "clock must return a finite number")
  112. def _lease_window(ttl_seconds: Any, clock: Clock) -> tuple[float, float]:
  113. """Validate ttl (first) and clock -> ``(now, expires_at)``; the sum itself must stay finite."""
  114. ttl = _finite(lambda: ttl_seconds, "ttl_seconds must be a finite positive number", positive=True)
  115. now = _timestamp(clock)
  116. if not math.isfinite(now + ttl):
  117. raise DriverValidationError("lease expiry must be finite")
  118. return now, now + ttl
  119. def validate_bound_task_manifest(value: Any) -> list[dict[str, Any]]:
  120. """Validate store metadata plus the exact source event for each task input."""
  121. from gateway.hosted_room_attachments import validate_task_manifest
  122. if not isinstance(value, list) or not value:
  123. raise DriverValidationError("attachments must be a non-empty list")
  124. if any(not isinstance(item, Mapping) or "event_id" not in item for item in value):
  125. raise DriverValidationError("task attachment requires event_id")
  126. event_ids = [_identifier(item["event_id"], label="attachment event_id") for item in value]
  127. try:
  128. manifest = validate_task_manifest([
  129. {key: val for key, val in item.items() if key != "event_id"} for item in value])
  130. except ValueError as exc:
  131. raise DriverValidationError(str(exc)) from exc
  132. return [{**item, "event_id": event_id} for item, event_id in zip(manifest, event_ids)]
  133. def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]:
  134. if not isinstance(value, dict):
  135. raise DriverValidationError("payload must be an object")
  136. unknown = set(value) - _TASK_PAYLOAD_REQUIRED_FIELDS - _TASK_PAYLOAD_OPTIONAL_FIELDS
  137. missing = _TASK_PAYLOAD_REQUIRED_FIELDS - set(value)
  138. if unknown:
  139. raise DriverValidationError(f"unknown payload fields: {', '.join(sorted(unknown))}")
  140. if missing:
  141. raise DriverValidationError(f"missing payload fields: {', '.join(sorted(missing))}")
  142. target_profile = _identifier(value["target_profile"], label="target_profile")
  143. prompt = text(value["prompt"], error=DriverValidationError, label="prompt", max_bytes=MAX_PROMPT_BYTES, strip=False)
  144. source_event_seq = _bounded_int(
  145. value["source_event_seq"], message="source_event_seq must be a positive integer", low=1)
  146. normalized = {"target_profile": target_profile, "prompt": prompt, "source_event_seq": source_event_seq}
  147. if "input_context" in value:
  148. from gateway.hosted_room_task_input import validate_task_input
  149. try:
  150. normalized["input_context"] = validate_task_input(value["input_context"])
  151. except ValueError as exc:
  152. raise DriverValidationError(str(exc)) from exc
  153. if "target_member_id" in value:
  154. normalized["target_member_id"] = _identifier(value["target_member_id"], label="target_member_id")
  155. if "attachments" in value:
  156. normalized["attachments"] = validate_bound_task_manifest(value["attachments"])
  157. if "lead_routing" in value:
  158. # Present only on a lead's routing turn, and then exactly true: absence is an ordinary member turn.
  159. if value["lead_routing"] is not True:
  160. raise DriverValidationError("lead_routing must be true when present")
  161. normalized["lead_routing"] = True
  162. encoded = compact_json(normalized)
  163. return normalized, encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest()
  164. @dataclass(frozen=True)
  165. class TaskIdentity:
  166. """Stable identity for one admitted room turn."""
  167. room_id: str
  168. task_id: str
  169. thread_id: str
  170. turn_id: str
  171. def __post_init__(self) -> None:
  172. for field in ("room_id", "task_id", "thread_id", "turn_id"):
  173. object.__setattr__(self, field, _identifier(getattr(self, field), label=field))
  174. @dataclass(frozen=True)
  175. class DriverLease:
  176. """A fenced lease held by one gateway process incarnation."""
  177. room_id: str
  178. gateway_id: str
  179. authority_epoch: int
  180. process_generation: str
  181. lease_generation: int
  182. expires_at: float
  183. reclaimed: bool = False
  184. @dataclass(frozen=True)
  185. class TaskAttempt:
  186. """The exact running generation authorized to settle one task."""
  187. identity: TaskIdentity
  188. lease: DriverLease
  189. execution_generation: int
  190. cancel_generation: int
  191. def _create_task_table(conn: sqlite3.Connection, table: str = "hosted_room_driver_tasks") -> None:
  192. conn.execute(
  193. f"""CREATE TABLE IF NOT EXISTS {table} (
  194. room_id TEXT NOT NULL, task_id TEXT NOT NULL, thread_id TEXT NOT NULL, turn_id TEXT NOT NULL,
  195. source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 1),
  196. payload_json TEXT NOT NULL, payload_digest TEXT NOT NULL,
  197. status TEXT NOT NULL CHECK (status IN (
  198. 'queued', 'running', 'settled', 'failed', 'cancelled', 'indeterminate', 'deferred', 'stopping')),
  199. execution_generation INTEGER NOT NULL DEFAULT 0 CHECK (execution_generation >= 0),
  200. cancel_generation INTEGER NOT NULL DEFAULT 0 CHECK (cancel_generation >= 0),
  201. run_gateway_id TEXT, run_process_generation TEXT, run_lease_generation INTEGER, cancel_id TEXT,
  202. settlement_id TEXT, settlement_status TEXT, result_json TEXT, created_at REAL NOT NULL,
  203. updated_at REAL NOT NULL, started_at REAL, terminal_at REAL, indeterminate_at REAL,
  204. PRIMARY KEY (room_id, task_id), UNIQUE (room_id, thread_id, turn_id),
  205. FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""")
  206. def _initialize_schema(conn: sqlite3.Connection) -> None:
  207. conn.execute("""CREATE TABLE IF NOT EXISTS hosted_room_driver_leases (
  208. room_id TEXT PRIMARY KEY, gateway_id TEXT NOT NULL,
  209. authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 1), process_generation TEXT NOT NULL,
  210. lease_generation INTEGER NOT NULL CHECK (lease_generation >= 1),
  211. expires_at REAL NOT NULL, acquired_at REAL NOT NULL, updated_at REAL NOT NULL, released_at REAL,
  212. FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""")
  213. _create_task_table(conn)
  214. _validate_schema(conn)
  215. conn.execute(_TASK_INDEX_SQL.format(if_not_exists="IF NOT EXISTS "))
  216. def _validate_schema(conn: sqlite3.Connection) -> None:
  217. columns = table_columns(conn, "hosted_room_driver_leases"), table_columns(conn, "hosted_room_driver_tasks")
  218. if columns != (_LEASE_COLUMNS, _TASK_COLUMNS):
  219. raise DriverStateError(
  220. "unsupported unpublished hosted-room driver schema; "
  221. "recreate the driver tables before starting the driver")
  222. for table in ("hosted_room_driver_leases", "hosted_room_driver_tasks"):
  223. if not any(
  224. row[2] == "hosted_rooms" and row[3] == "room_id" and row[4] == "room_id"
  225. for row in conn.execute(f"PRAGMA foreign_key_list({table})").fetchall()):
  226. raise DriverStateError(f"{table} is missing its hosted_rooms foreign key")
  227. def _schema_objects_exist(conn: sqlite3.Connection) -> bool:
  228. rows = conn.execute("""SELECT name FROM sqlite_master WHERE type='table'
  229. AND name IN ('hosted_room_driver_leases', 'hosted_room_driver_tasks')""").fetchall()
  230. if {row[0] for row in rows} != {"hosted_room_driver_leases", "hosted_room_driver_tasks"}:
  231. return False
  232. index = conn.execute(
  233. "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'").fetchone()
  234. return index is not None
  235. def _migrate_task_status_constraint(conn: sqlite3.Connection) -> None:
  236. """Expand the unpublished task-state CHECK without losing durable work."""
  237. conn.execute("DROP INDEX IF EXISTS idx_hosted_room_driver_tasks_status")
  238. _create_task_table(conn, "hosted_room_driver_tasks_next")
  239. columns = ", ".join(_TASK_COLUMN_ORDER)
  240. conn.execute(
  241. f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks")
  242. conn.execute("DROP TABLE hosted_room_driver_tasks")
  243. conn.execute("ALTER TABLE hosted_room_driver_tasks_next RENAME TO hosted_room_driver_tasks")
  244. conn.execute(_TASK_INDEX_SQL.format(if_not_exists=""))
  245. def _connect(db_path: DbPath) -> sqlite3.Connection:
  246. """Open the store; existing tables are validated (after the status-constraint migration when needed).
  247. The driver schema never shipped, so an incompatible draft fails closed in ``_validate_schema``."""
  248. existing: list[bool] = []
  249. def ready(conn: sqlite3.Connection) -> bool:
  250. existing.append(_schema_objects_exist(conn))
  251. if not existing[0]:
  252. return False
  253. row = conn.execute(
  254. "SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'").fetchone()
  255. sql = str(row[0] or "").lower() if row else ""
  256. return "'stopping'" in sql and "'deferred'" in sql # task-status CHECK already covers the current states
  257. conn = connect(
  258. db_path, db_label="state.db (hosted_room_driver)", ready=ready,
  259. initialize=lambda conn: (_migrate_task_status_constraint if existing[0] else _initialize_schema)(conn))
  260. if existing[0]:
  261. try:
  262. _validate_schema(conn)
  263. except Exception:
  264. conn.close()
  265. raise
  266. return conn
  267. def _transaction(db_path: DbPath):
  268. return transaction(_connect, db_path, immediate=True)
  269. def _lease_from_row(row: sqlite3.Row | dict[str, Any], *, reclaimed: bool = False) -> DriverLease:
  270. return DriverLease(
  271. room_id=row["room_id"], gateway_id=row["gateway_id"], authority_epoch=int(row["authority_epoch"]),
  272. process_generation=row["process_generation"], lease_generation=int(row["lease_generation"]),
  273. expires_at=float(row["expires_at"]), reclaimed=reclaimed)
  274. def _task_identity_from_row(row: sqlite3.Row) -> TaskIdentity:
  275. return TaskIdentity(
  276. room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"])
  277. def _optional(cast: Callable[[Any], Any]) -> Callable[[Any], Any]:
  278. return lambda value: cast(value) if value is not None else None
  279. # Task-view casts per row column (columns after payload_digest in _TASK_COLUMN_ORDER, same key order;
  280. # result_json is exposed as "result"). Columns not listed are passed through untouched.
  281. _TASK_VIEW_CASTS: dict[str, Callable[[Any], Any]] = {
  282. "execution_generation": int, "cancel_generation": int, "run_lease_generation": _optional(int),
  283. "result_json": _optional(json.loads), "created_at": float, "updated_at": float, "started_at": _optional(float),
  284. "terminal_at": _optional(float), "indeterminate_at": _optional(float)}
  285. def _task_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]:
  286. try:
  287. payload, encoded_payload, payload_digest = _task_payload(json.loads(row["payload_json"]))
  288. except (TypeError, json.JSONDecodeError, DriverValidationError) as exc:
  289. raise TaskConflictError("stored task payload is invalid") from exc
  290. if (encoded_payload, payload_digest, payload["source_event_seq"]) != (
  291. row["payload_json"], row["payload_digest"], int(row["source_event_seq"])):
  292. raise TaskConflictError("stored task payload failed its integrity check")
  293. task: dict[str, Any] = {"identity": _task_identity_from_row(row), "payload": payload}
  294. for column in _TASK_COLUMN_ORDER[_TASK_COLUMN_ORDER.index("payload_digest"):]:
  295. task["result" if column == "result_json" else column] = _TASK_VIEW_CASTS.get(column, lambda v: v)(row[column])
  296. task["idempotent"] = idempotent
  297. return task
  298. def _load_task(conn: sqlite3.Connection, identity: TaskIdentity, *, required: bool = True) -> sqlite3.Row | None:
  299. row = conn.execute(_SELECT_TASK, (identity.room_id, identity.task_id)).fetchone()
  300. if row is None:
  301. if required:
  302. raise TaskConflictError("task does not exist")
  303. return None
  304. if _task_identity_from_row(row) != identity:
  305. raise TaskConflictError("task_id is already bound to a different turn")
  306. return row
  307. def _tasks_in_order(conn: sqlite3.Connection, room_id: str, status: str | None = None) -> list[sqlite3.Row]:
  308. where, params = ("", (room_id,)) if status is None else (" AND status=?", (room_id, status))
  309. sql = f"SELECT * FROM hosted_room_driver_tasks WHERE room_id=?{where} {_TASK_ORDER}"
  310. return conn.execute(sql, params).fetchall()
  311. def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row:
  312. try:
  313. row = conn.execute(
  314. "SELECT room_id, authority_gateway_id, authority_epoch, disbanded_at FROM hosted_rooms WHERE room_id=?",
  315. (room_id,)).fetchone()
  316. except sqlite3.OperationalError as exc:
  317. if "no such table" in str(exc).lower():
  318. raise RoomUnavailableError("hosted room does not exist") from exc
  319. raise
  320. if row is None or row["disbanded_at"] is not None:
  321. raise RoomUnavailableError("hosted room does not exist" if row is None else "hosted room is disbanded")
  322. return row
  323. def _require_room_authority(conn: sqlite3.Connection, room_id: str, gateway_id: str, epoch: int) -> sqlite3.Row:
  324. room = _load_active_room(conn, room_id)
  325. if room["authority_gateway_id"] != gateway_id or int(room["authority_epoch"]) != epoch:
  326. raise StaleLeaseError("hosted room authority changed")
  327. return room
  328. def _run_fence(lease: DriverLease) -> tuple[str, str, int]:
  329. """The ``_RUN_FENCE`` bind order: (gateway_id, process_generation, lease_generation)."""
  330. return lease.gateway_id, lease.process_generation, lease.lease_generation
  331. def _lease_row_matches(row: sqlite3.Row | None, lease: DriverLease) -> bool:
  332. return row is not None and (
  333. row["gateway_id"], int(row["authority_epoch"]), row["process_generation"], int(row["lease_generation"])
  334. ) == (lease.gateway_id, lease.authority_epoch, lease.process_generation, lease.lease_generation)
  335. def _require_active_lease(conn: sqlite3.Connection, lease: DriverLease, *, now: float) -> sqlite3.Row:
  336. _require_room_authority(conn, lease.room_id, lease.gateway_id, lease.authority_epoch)
  337. row = conn.execute(_SELECT_LEASE, (lease.room_id,)).fetchone()
  338. if not _lease_row_matches(row, lease) or row["released_at"] is not None or float(row["expires_at"]) <= now:
  339. raise StaleLeaseError("driver lease is stale or expired")
  340. return row
  341. def _check_same_room(lease: DriverLease, identity: TaskIdentity) -> None:
  342. if lease.room_id != identity.room_id:
  343. raise DriverValidationError("lease and task belong to different rooms")
  344. def _cancel_generation(value: int) -> int:
  345. # Deliberately accepts bool (a bool is an int); do not swap for non_negative_int.
  346. if not isinstance(value, int) or value < 0:
  347. raise DriverValidationError("expected_cancel_generation must be non-negative")
  348. return value
  349. def _expected_generations(
  350. lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int) -> None:
  351. _check_same_room(lease, identity)
  352. if not isinstance(execution_generation, int) or execution_generation < 1:
  353. raise DriverValidationError("expected_execution_generation must be a positive integer")
  354. _cancel_generation(cancel_generation)
  355. def _terminal_settlement_id(settlement_id: Any, status: Any) -> str:
  356. settlement_id = _identifier(settlement_id, label="settlement_id")
  357. if status not in {"settled", "failed"}:
  358. raise DriverValidationError("status must be 'settled' or 'failed'")
  359. return settlement_id
  360. def _settlement(
  361. settlement_id: Any, status: Any, result: Any, clock: Clock
  362. ) -> tuple[float, Callable[[sqlite3.Row], Any], tuple[Any, ...]]:
  363. """Validate one terminal settlement -> (now, replay predicate, ``_SETTLE_SET`` params); the replay treats an
  364. identical committed settlement as idempotent and a different one as a conflict."""
  365. settlement_id = _terminal_settlement_id(settlement_id, status)
  366. result_json = _canonical_json(result)
  367. now = _timestamp(clock)
  368. def replay(row: sqlite3.Row) -> dict[str, Any] | None:
  369. if row["settlement_id"] is None:
  370. return None
  371. if (row["settlement_id"], row["settlement_status"], row["result_json"]) == (settlement_id, status, result_json):
  372. return _task_from_row(row, idempotent=True)
  373. raise TaskConflictError("task already has a different terminal settlement")
  374. return now, replay, (status, settlement_id, status, result_json, now, now)
  375. def _cancel_replay(cancel_id: str, status: str = "cancelled") -> Callable[[sqlite3.Row], Any]:
  376. """Replay predicate: same cancel_id already committed in ``status``."""
  377. return lambda row: (
  378. _task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None)
  379. def _generations_match(row: sqlite3.Row, status: str, execution_generation: int, cancel_generation: int) -> bool:
  380. return (row["status"], int(row["execution_generation"]), int(row["cancel_generation"])) == (
  381. status, execution_generation, cancel_generation)
  382. def _require_cancel_generation(row: sqlite3.Row, expected_cancel_generation: int) -> None:
  383. if int(row["cancel_generation"]) != expected_cancel_generation:
  384. raise StaleTaskError("task cancellation generation changed")
  385. def _transition(
  386. db_path: DbPath, identity: TaskIdentity, *, sql: str, set_params: tuple[Any, ...], fence_params: tuple[Any, ...],
  387. stale: str, now: float, lease: DriverLease | None = None, lease_first: bool = True,
  388. replay: Callable[[sqlite3.Row], dict[str, Any] | None] | None = None,
  389. guard: Callable[[sqlite3.Row], None] | None = None) -> dict[str, Any]:
  390. """Run one fenced task transition: load -> idempotent replay -> lease/fence guard -> UPDATE.
  391. ``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row or ``stale``
  392. is raised. ``lease_first`` checks the lease before the row load (recovery paths) instead of after the
  393. replay (settlement paths: an identical replay still succeeds after the lease moved on).
  394. """
  395. params = (*set_params, identity.room_id, identity.task_id, *fence_params)
  396. with _transaction(db_path) as conn:
  397. if lease is not None and lease_first:
  398. _require_active_lease(conn, lease, now=now)
  399. row = _load_task(conn, identity)
  400. if replay is not None and (replayed := replay(row)) is not None:
  401. return replayed
  402. if lease is not None and not lease_first:
  403. _require_active_lease(conn, lease, now=now)
  404. if guard is not None:
  405. guard(row)
  406. fenced_update(conn, sql, params, StaleTaskError(stale))
  407. from gateway.hosted_room_work_records import capture_transition_locked
  408. capture_transition_locked(conn, identity.room_id)
  409. return _task_from_row(_load_task(conn, identity))
  410. def _generation_transition(
  411. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, name: str, execution_generation: int,
  412. cancel_generation: int, *, now: float, set_params: tuple[Any, ...],
  413. replay: Callable[[sqlite3.Row], Any] | None = None) -> dict[str, Any]:
  414. """Lease-first transition from ``_GENERATION_TRANSITIONS`` fenced on status + both generations."""
  415. status, set_clause, generation_stale, stale = _GENERATION_TRANSITIONS[name]
  416. def guard(row: sqlite3.Row) -> None:
  417. if not _generations_match(row, status, execution_generation, cancel_generation):
  418. raise StaleTaskError(generation_stale)
  419. return _transition(
  420. db_path, identity, lease=lease, now=now, replay=replay, guard=guard, sql=_generation_update(set_clause, status),
  421. set_params=set_params, fence_params=(execution_generation, cancel_generation), stale=stale)
  422. def _run_fence_transition(
  423. db_path: DbPath, attempt: TaskAttempt, *, guard_stale: str, lease_generation: Callable[[Any], int] = int,
  424. **transition: Any) -> dict[str, Any]:
  425. """Transition fenced on this attempt's running generation under its exact lease (row guard + SQL fence).
  426. ``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL, ``int(v or 0)`` reads 0."""
  427. lease = attempt.lease
  428. def guard(row: sqlite3.Row) -> None:
  429. if not _generations_match(row, "running", attempt.execution_generation, attempt.cancel_generation) or (
  430. row["run_gateway_id"], row["run_process_generation"], lease_generation(row["run_lease_generation"])
  431. ) != _run_fence(lease):
  432. raise StaleTaskError(guard_stale)
  433. return _transition(
  434. db_path, attempt.identity, lease=lease, guard=guard,
  435. fence_params=(attempt.execution_generation, attempt.cancel_generation, *_run_fence(lease)), **transition)
  436. def acquire_lease(
  437. db_path: DbPath, *, room_id: Any, gateway_id: Any, authority_epoch: Any, process_generation: Any, ttl_seconds: Any,
  438. clock: Clock) -> DriverLease:
  439. """Acquire an empty or expired room lease with a monotonic generation."""
  440. room_id = _identifier(room_id, label="room_id")
  441. gateway_id = _identifier(gateway_id, label="gateway_id")
  442. authority_epoch = _bounded_int(authority_epoch, message="authority_epoch must be a positive integer", low=1)
  443. process_generation = _identifier(process_generation, label="process_generation")
  444. now, expires_at = _lease_window(ttl_seconds, clock)
  445. with _transaction(db_path) as conn:
  446. _require_room_authority(conn, room_id, gateway_id, authority_epoch)
  447. row = conn.execute(_SELECT_LEASE, (room_id,)).fetchone()
  448. if row is None:
  449. conn.execute("""INSERT INTO hosted_room_driver_leases (
  450. room_id, gateway_id, authority_epoch, process_generation, lease_generation,
  451. expires_at, acquired_at, updated_at, released_at
  452. ) VALUES (?, ?, ?, ?, 1, ?, ?, ?, NULL)""",
  453. (room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now))
  454. return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone())
  455. same_authority = row["gateway_id"] == gateway_id and int(row["authority_epoch"]) == authority_epoch
  456. live = row["released_at"] is None and float(row["expires_at"]) > now
  457. if same_authority and row["process_generation"] == process_generation and live:
  458. renewed_expiry = max(float(row["expires_at"]), expires_at)
  459. conn.execute(
  460. "UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND lease_generation=?",
  461. (renewed_expiry, now, room_id, int(row["lease_generation"])))
  462. return _lease_from_row({**dict(row), "expires_at": renewed_expiry})
  463. if same_authority and live:
  464. raise LeaseHeldError("room driver lease is held by another generation")
  465. fenced_update(conn, """UPDATE hosted_room_driver_leases
  466. SET gateway_id=?, authority_epoch=?, process_generation=?, lease_generation=lease_generation + 1,
  467. expires_at=?, acquired_at=?, updated_at=?, released_at=NULL
  468. WHERE room_id=? AND lease_generation=? AND (
  469. gateway_id != ? OR authority_epoch != ? OR released_at IS NOT NULL OR expires_at <= ?)""",
  470. (
  471. gateway_id, authority_epoch, process_generation, expires_at, now, now, room_id,
  472. int(row["lease_generation"]), gateway_id, authority_epoch, now),
  473. LeaseHeldError("room driver lease changed during acquisition"))
  474. return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone(), reclaimed=True)
  475. def renew_lease(db_path: DbPath, lease: DriverLease, *, ttl_seconds: Any, clock: Clock) -> DriverLease:
  476. """Renew the exact active lease generation or fail closed."""
  477. now, requested_expiry = _lease_window(ttl_seconds, clock)
  478. with _transaction(db_path) as conn:
  479. current = _require_active_lease(conn, lease, now=now)
  480. expires_at = max(float(current["expires_at"]), requested_expiry)
  481. fenced_update(conn, """UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?
  482. WHERE room_id=? AND gateway_id=? AND process_generation=?
  483. AND lease_generation=? AND released_at IS NULL AND expires_at > ?""",
  484. (expires_at, now, lease.room_id, *_run_fence(lease), now),
  485. StaleLeaseError("driver lease changed during renewal"))
  486. return dataclasses.replace(lease, expires_at=expires_at, reclaimed=False)
  487. def release_lease(db_path: DbPath, lease: DriverLease, *, clock: Clock) -> dict[str, Any]:
  488. """Release the exact active lease generation idempotently."""
  489. now = _timestamp(clock)
  490. with _transaction(db_path) as conn:
  491. _require_room_authority(conn, lease.room_id, lease.gateway_id, lease.authority_epoch)
  492. row = conn.execute(_SELECT_LEASE, (lease.room_id,)).fetchone()
  493. if not _lease_row_matches(row, lease):
  494. raise StaleLeaseError("driver lease is stale")
  495. if row["released_at"] is not None:
  496. return {"lease": _lease_from_row(row), "idempotent": True}
  497. if float(row["expires_at"]) <= now:
  498. raise StaleLeaseError("driver lease expired before release")
  499. if conn.execute(
  500. "SELECT 1 FROM hosted_room_driver_tasks WHERE room_id=? AND status='running' LIMIT 1", (lease.room_id,)
  501. ).fetchone() is not None:
  502. raise InvalidTaskTransitionError("cannot release a room lease while tasks are running")
  503. conn.execute("""UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?, released_at=?
  504. WHERE room_id=? AND lease_generation=?""",
  505. (now, now, now, lease.room_id, lease.lease_generation))
  506. return {
  507. "lease": _lease_from_row({**dict(row), "expires_at": now, "updated_at": now, "released_at": now}),
  508. "idempotent": False}
  509. def admit_task(db_path: DbPath, identity: TaskIdentity, *, payload: Any, clock: Clock) -> dict[str, Any]:
  510. """Persist a queued task, or return the identical admission."""
  511. normalized_payload, payload_json, payload_digest = _task_payload(payload)
  512. now = _timestamp(clock)
  513. with _transaction(db_path) as conn:
  514. _load_active_room(conn, identity.room_id)
  515. existing = _load_task(conn, identity, required=False)
  516. if existing is not None:
  517. if existing["payload_digest"] != payload_digest or existing["payload_json"] != payload_json:
  518. raise TaskConflictError("task_id is already bound to a different payload")
  519. return _task_from_row(existing, idempotent=True)
  520. if conn.execute(
  521. "SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND thread_id=? AND turn_id=?",
  522. (identity.room_id, identity.thread_id, identity.turn_id)).fetchone() is not None:
  523. raise TaskConflictError("thread_id and turn_id are already bound to a task")
  524. conn.execute("""INSERT INTO hosted_room_driver_tasks (
  525. room_id, task_id, thread_id, turn_id, source_event_seq, payload_json, payload_digest,
  526. status, execution_generation, cancel_generation, created_at, updated_at
  527. ) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, 0, ?, ?)""",
  528. (
  529. *dataclasses.astuple(identity), normalized_payload["source_event_seq"], payload_json, payload_digest,
  530. now, now))
  531. from gateway.hosted_room_work_records import capture_transition_locked
  532. capture_transition_locked(conn, identity.room_id)
  533. return _task_from_row(_load_task(conn, identity))
  534. def start_task(
  535. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_cancel_generation: int, clock: Clock
  536. ) -> TaskAttempt:
  537. """Move one queued task to running under the current driver lease."""
  538. _check_same_room(lease, identity)
  539. _cancel_generation(expected_cancel_generation)
  540. now = _timestamp(clock)
  541. with _transaction(db_path) as conn:
  542. _require_active_lease(conn, lease, now=now)
  543. row = _load_task(conn, identity)
  544. _require_cancel_generation(row, expected_cancel_generation)
  545. if row["status"] != "queued":
  546. raise InvalidTaskTransitionError(f"cannot start task in state '{row['status']}'")
  547. if conn.execute(
  548. f"""SELECT task_id, status FROM hosted_room_driver_tasks
  549. WHERE room_id=? AND status IN ('running', 'indeterminate', 'stopping') {_TASK_ORDER} LIMIT 1""",
  550. (identity.room_id,)).fetchone() is not None:
  551. raise InvalidTaskTransitionError("room recovery must resolve the prior task before starting new work")
  552. next_queued = conn.execute(
  553. f"SELECT task_id FROM hosted_room_driver_tasks WHERE room_id=? AND status='queued' {_TASK_ORDER} LIMIT 1",
  554. (identity.room_id,)).fetchone()
  555. if next_queued is None or next_queued["task_id"] != identity.task_id:
  556. raise InvalidTaskTransitionError("task is not next in the hosted room event order")
  557. execution_generation = int(row["execution_generation"]) + 1
  558. fenced_update(conn, """UPDATE hosted_room_driver_tasks
  559. SET status='running', execution_generation=?, run_gateway_id=?, run_process_generation=?,
  560. run_lease_generation=?, started_at=?, updated_at=?
  561. WHERE room_id=? AND task_id=? AND status='queued' AND cancel_generation=?""",
  562. (
  563. execution_generation, *_run_fence(lease), now, now, identity.room_id, identity.task_id,
  564. expected_cancel_generation), StaleTaskError("task changed during start"))
  565. from gateway.hosted_room_work_records import capture_transition_locked
  566. capture_transition_locked(conn, identity.room_id)
  567. return TaskAttempt(
  568. identity=identity, lease=lease, execution_generation=execution_generation,
  569. cancel_generation=expected_cancel_generation)
  570. def settle_task(
  571. db_path: DbPath, attempt: TaskAttempt, *, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock
  572. ) -> dict[str, Any]:
  573. """Commit one terminal result if every lease and task fence still matches."""
  574. now, replay, set_params = _settlement(settlement_id, status, result, clock)
  575. return _run_fence_transition(
  576. db_path, attempt, guard_stale="task attempt is stale or cancelled", lease_first=False, now=now, replay=replay,
  577. sql=_SETTLE_RUNNING_SQL, set_params=set_params, stale="task changed during settlement")
  578. def settle_stopping_task(
  579. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  580. expected_cancel_generation: int, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock
  581. ) -> dict[str, Any]:
  582. """Commit a completion that won the race with an unacknowledged Stop."""
  583. _terminal_settlement_id(settlement_id, status) # settlement errors take precedence over generation errors
  584. if expected_execution_generation < 1 or expected_cancel_generation < 1:
  585. raise DriverValidationError("stopping settlement generations are invalid")
  586. now, replay, set_params = _settlement(settlement_id, status, result, clock)
  587. return _transition(
  588. db_path, identity, lease=lease, lease_first=False, now=now, replay=replay, sql=_SETTLE_STOPPING_SQL,
  589. set_params=set_params, fence_params=(expected_execution_generation, expected_cancel_generation),
  590. stale="task completion lost the stop race")
  591. def resolve_indeterminate_task(
  592. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  593. expected_cancel_generation: int, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock
  594. ) -> dict[str, Any]:
  595. """Commit a verified historical receipt under the current room lease."""
  596. _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
  597. now, replay, set_params = _settlement(settlement_id, status, result, clock)
  598. return _generation_transition(
  599. db_path, identity, lease, "resolve", expected_execution_generation, expected_cancel_generation, now=now,
  600. replay=replay, set_params=set_params)
  601. def resolve_indeterminate_cancellation(
  602. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  603. expected_cancel_generation: int, cancel_id: Any, clock: Clock) -> dict[str, Any]:
  604. """Commit a verified terminal cancellation for an uncertain attempt."""
  605. _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
  606. cancel_id = _identifier(cancel_id, label="cancel_id")
  607. now = _timestamp(clock)
  608. return _generation_transition(
  609. db_path, identity, lease, "resolve_cancel", expected_execution_generation, expected_cancel_generation, now=now,
  610. replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now))
  611. def requeue_indeterminate_task(
  612. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  613. expected_cancel_generation: int, clock: Clock) -> dict[str, Any]:
  614. """Explicitly retry uncertain work after an operator accepts at-least-once risk."""
  615. _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
  616. now = _timestamp(clock)
  617. return _generation_transition(
  618. db_path, identity, lease, "requeue", expected_execution_generation, expected_cancel_generation, now=now,
  619. set_params=(now,))
  620. def defer_indeterminate_task(
  621. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  622. expected_cancel_generation: int, reason: Any, clock: Clock) -> dict[str, Any]:
  623. """Fence one uncertain attempt and release later room work."""
  624. _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
  625. reason = _identifier(reason, label="defer_reason")
  626. result_json = _canonical_json({"reason": reason, "retryable": True})
  627. now = _timestamp(clock)
  628. def replay(row: sqlite3.Row) -> dict[str, Any] | None:
  629. deferred = _generations_match(row, "deferred", expected_execution_generation, expected_cancel_generation)
  630. return _task_from_row(row, idempotent=True) if deferred and row["result_json"] == result_json else None
  631. return _generation_transition(
  632. db_path, identity, lease, "defer", expected_execution_generation, expected_cancel_generation, now=now,
  633. replay=replay, set_params=(result_json, now, now))
  634. def requeue_deferred_task(
  635. db_path: DbPath, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
  636. expected_cancel_generation: int, clock: Clock) -> dict[str, Any]:
  637. """Explicitly retry a fenced deferred turn under a new generation."""
  638. _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
  639. now = _timestamp(clock)
  640. return _generation_transition(
  641. db_path, identity, lease, "requeue_deferred", expected_execution_generation, expected_cancel_generation,
  642. now=now, set_params=(now,))
  643. def requeue_not_admitted_task(db_path: DbPath, attempt: TaskAttempt, *, clock: Clock) -> dict[str, Any]:
  644. """Return a running task to its durable queue after proven non-admission."""
  645. now = _timestamp(clock)
  646. _check_same_room(attempt.lease, attempt.identity)
  647. def replay(row: sqlite3.Row) -> dict[str, Any] | None:
  648. requeued = _generations_match(row, "queued", attempt.execution_generation, attempt.cancel_generation) and (
  649. row["run_gateway_id"], row["run_process_generation"], row["run_lease_generation"]) == (None, None, None)
  650. return _task_from_row(row, idempotent=True) if requeued else None
  651. return _run_fence_transition(
  652. db_path, attempt, guard_stale="not-admitted task attempt lost its fence",
  653. lease_generation=lambda value: int(value or 0), now=now, replay=replay, sql=_REQUEUE_RUNNING_SQL,
  654. set_params=(now,), stale="not-admitted task changed during requeue")
  655. def cancel_task(
  656. db_path: DbPath, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock
  657. ) -> dict[str, Any]:
  658. """Cancel a queued task before any external work was admitted."""
  659. cancel_id = _identifier(cancel_id, label="cancel_id")
  660. _cancel_generation(expected_cancel_generation)
  661. now = _timestamp(clock)
  662. def guard(row: sqlite3.Row) -> None:
  663. if row["status"] in TERMINAL_STATUSES:
  664. raise InvalidTaskTransitionError(f"cannot cancel task in state '{row['status']}'")
  665. if row["status"] not in {"queued", "deferred"}:
  666. raise InvalidTaskTransitionError("running work requires acknowledged two-phase cancellation")
  667. _require_cancel_generation(row, expected_cancel_generation)
  668. return _transition(
  669. db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_CANCEL_QUEUED_SQL,
  670. set_params=(expected_cancel_generation + 1, cancel_id, now, now), fence_params=(expected_cancel_generation,),
  671. stale="task changed during cancellation")
  672. def begin_task_cancel(
  673. db_path: DbPath, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock
  674. ) -> dict[str, Any]:
  675. """Persist a stop intent without claiming the remote run has stopped."""
  676. cancel_id = _identifier(cancel_id, label="cancel_id")
  677. _cancel_generation(expected_cancel_generation)
  678. now = _timestamp(clock)
  679. def guard(row: sqlite3.Row) -> None:
  680. if row["status"] in TERMINAL_STATUSES or row["status"] == "queued":
  681. raise InvalidTaskTransitionError(f"cannot request remote stop in state '{row['status']}'")
  682. _require_cancel_generation(row, expected_cancel_generation)
  683. return _transition(
  684. db_path, identity, now=now, replay=_cancel_replay(cancel_id, "stopping"), guard=guard, sql=_BEGIN_STOP_SQL,
  685. set_params=(expected_cancel_generation + 1, cancel_id, now), fence_params=(expected_cancel_generation,),
  686. stale="task changed during stop request")
  687. def complete_task_cancel(
  688. db_path: DbPath, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock
  689. ) -> dict[str, Any]:
  690. """Commit cancellation only after the transport acknowledges exact Stop."""
  691. cancel_id = _identifier(cancel_id, label="cancel_id")
  692. now = _timestamp(clock)
  693. def guard(row: sqlite3.Row) -> None:
  694. if (row["status"], row["cancel_id"], int(row["cancel_generation"])) != (
  695. "stopping", cancel_id, expected_cancel_generation):
  696. raise StaleTaskError("task stop acknowledgement is stale")
  697. return _transition(
  698. db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_COMPLETE_STOP_SQL,
  699. set_params=(now, now), fence_params=(cancel_id, expected_cancel_generation),
  700. stale="task changed during stop acknowledgement")
  701. def recover_room(db_path: DbPath, lease: DriverLease, *, clock: Clock) -> dict[str, list[TaskIdentity]]:
  702. """Fence abandoned running attempts without requeueing uncertain work."""
  703. now = _timestamp(clock)
  704. foreign_running = f"room_id=? AND status='running' AND NOT ({_RUN_FENCE})"
  705. fence = (lease.room_id, *_run_fence(lease))
  706. with _transaction(db_path) as conn:
  707. _require_active_lease(conn, lease, now=now)
  708. stale_rows = conn.execute(
  709. f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence).fetchall()
  710. if stale_rows:
  711. conn.execute(
  712. f"""UPDATE hosted_room_driver_tasks SET status='indeterminate', indeterminate_at=?, updated_at=?
  713. WHERE {foreign_running}""", (now, now, *fence))
  714. from gateway.hosted_room_work_records import capture_transition_locked
  715. capture_transition_locked(conn, lease.room_id)
  716. return {
  717. status: [_task_identity_from_row(row) for row in _tasks_in_order(conn, lease.room_id, status)]
  718. for status in ("queued", "indeterminate")}
  719. def get_task(db_path: DbPath, identity: TaskIdentity) -> dict[str, Any]:
  720. """Read one task without mutating its state."""
  721. with closing(_connect(db_path)) as conn:
  722. return _task_from_row(_load_task(conn, identity))
  723. def list_tasks(db_path: DbPath, *, room_id: Any, status: TaskStatus | None = None) -> list[dict[str, Any]]:
  724. """Return room tasks in deterministic admission order."""
  725. room_id = _identifier(room_id, label="room_id")
  726. if status is not None and status not in TASK_STATUSES:
  727. raise DriverValidationError("invalid task status")
  728. with closing(_connect(db_path)) as conn:
  729. return [_task_from_row(row) for row in _tasks_in_order(conn, room_id, status)]
  730. def prune_published_terminal_tasks(
  731. db_path: DbPath, *, room_id: Any, clock: Clock, retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS,
  732. retain: int = MAX_RETAINED_TERMINAL_TASKS) -> int:
  733. """Bound execution rows after outcomes are durable in the room log."""
  734. room_id = _identifier(room_id, label="room_id")
  735. now = _timestamp(clock)
  736. if retention_seconds <= 0:
  737. raise DriverValidationError("retention_seconds must be positive")
  738. _bounded_int(retain, message="retain must be a non-negative integer")
  739. with _transaction(db_path) as conn:
  740. publications = conn.execute(
  741. "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'").fetchone()
  742. if publications is None:
  743. return 0
  744. rows = conn.execute("""SELECT t.task_id, t.terminal_at FROM hosted_room_driver_tasks t
  745. WHERE t.room_id=? AND t.status IN ('settled', 'failed', 'cancelled')
  746. AND EXISTS (SELECT 1 FROM hosted_room_policy_publications p
  747. WHERE p.room_id=t.room_id AND p.task_id=t.task_id
  748. AND p.kind IN ('turn.settled', 'turn.failed', 'turn.cancelled'))
  749. ORDER BY t.terminal_at DESC, t.task_id ASC""", (room_id,)).fetchall()
  750. cutoff = now - float(retention_seconds)
  751. candidates = [
  752. str(row["task_id"]) for index, row in enumerate(rows)
  753. if index >= retain or (row["terminal_at"] is not None and float(row["terminal_at"]) <= cutoff)
  754. ][:MAX_TASK_PRUNE_BATCH]
  755. if not candidates:
  756. return 0
  757. deleted = conn.execute(
  758. f"DELETE FROM hosted_room_driver_tasks WHERE room_id=? AND task_id IN ({','.join('?' * len(candidates))})",
  759. (room_id, *candidates))
  760. return max(0, int(deleted.rowcount))
  761. def get_task_for_turn(
  762. db_path: DbPath,
  763. identity: TaskIdentity,
  764. ) -> dict[str, Any] | None:
  765. """Read the immutable admission for a turn, including older payload versions."""
  766. conn = _connect(db_path)
  767. try:
  768. row = conn.execute(
  769. """SELECT * FROM hosted_room_driver_tasks
  770. WHERE room_id=? AND thread_id=? AND turn_id=?""",
  771. (identity.room_id, identity.thread_id, identity.turn_id),
  772. ).fetchone()
  773. return _task_from_row(row) if row is not None else None
  774. finally:
  775. conn.close()