api_server_response_store.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. """SQLite-backed store for Responses API state (``previous_response_id`` chaining, named conversations)."""
  2. import asyncio
  3. import functools
  4. import json
  5. import logging
  6. import sqlite3
  7. import threading
  8. import time
  9. from contextlib import closing, suppress
  10. from pathlib import Path
  11. from typing import Any, Optional
  12. # Logger parity with the origin module (moved log records keep their name).
  13. logger = logging.getLogger("gateway.platforms.api_server")
  14. MAX_STORED_RESPONSES = 100
  15. # Settled Idempotency-Key identities retained per response-body slot (see ResponseStore).
  16. IDENTITY_RETENTION_FACTOR = 10
  17. # The stored terminal envelope of a body row; a corrupt body reads as absent, never raises.
  18. _RECORD = "json_extract(CASE WHEN json_valid(data) THEN data END, '$.response"
  19. def _locked(method):
  20. """Hold the store's lock for the call: one sqlite3 connection, used from the loop's worker threads
  21. and the admission steps, must never interleave two callers' statements or commits."""
  22. @functools.wraps(method)
  23. def _call(self, *args, **kwargs):
  24. with self._lock:
  25. return method(self, *args, **kwargs)
  26. return _call
  27. class ResponseStore:
  28. """SQLite-backed LRU store for Responses API state (full conversation history per response
  29. for ``previous_response_id`` chaining). Persists across restarts; in-memory fallback.
  30. Idempotency-Key identity (``response_keys`` + ``response_admissions`` +
  31. ``response_output_order``) deliberately outlives body LRU eviction so an exact retry still
  32. replays, conflicts or reconstructs its terminal result. Its own lifecycle is a count bound:
  33. at most ``_identity_bound()`` *settled* identities (a terminal idempotency record was
  34. stored), ``IDENTITY_RETENTION_FACTOR * max_size`` unless ``max_identities`` is given. Past
  35. the bound the oldest-settled are dropped from all three tables plus their ``idem:`` replay
  36. record, in the same transaction as body eviction. An unsettled identity (admission pending,
  37. running, or outcome unknown) neither counts nor is ever dropped; it joins the bound once an
  38. observer stores its record or, with no observer left, once its canonical admission is terminal. Public ``delete`` of a settled response drops its identity the same way.
  39. An expired or deleted key is not a new request: the canonical admission still carries the
  40. request id, so a reuse is refused with ``admission_conflict`` and never re-executes.
  41. """
  42. def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: Optional[str] = None,
  43. max_identities: Optional[int] = None):
  44. self._lock = threading.RLock()
  45. self._max_size = max_size
  46. self._max_identities = max_identities
  47. if db_path is None:
  48. db_path = ":memory:"
  49. with suppress(Exception):
  50. from hermes_cli.config import get_hermes_home
  51. db_path = str(get_hermes_home() / "response_store.db")
  52. self._db_path: Optional[str] = db_path if db_path != ":memory:" else None
  53. try:
  54. self._conn = sqlite3.connect(db_path, check_same_thread=False)
  55. except Exception:
  56. self._conn = sqlite3.connect(":memory:", check_same_thread=False)
  57. self._db_path = None
  58. # Shared WAL-fallback so response_store.db degrades gracefully on NFS/SMB/FUSE homes.
  59. from hermes_state_wal import apply_wal_with_fallback
  60. apply_wal_with_fallback(self._conn, db_label="response_store.db")
  61. self._conn.execute(
  62. "CREATE TABLE IF NOT EXISTS responses ("
  63. "response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)")
  64. self._conn.execute(
  65. "CREATE TABLE IF NOT EXISTS conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)")
  66. # Compact immutable request identity outlives the LRU response bodies.
  67. # Distinct canonical session targets must not turn a global retry key
  68. # into a last-writer-wins response cache entry.
  69. self._conn.execute(
  70. "CREATE TABLE IF NOT EXISTS response_keys (request_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, created_at INTEGER NOT NULL)")
  71. self._conn.execute(
  72. "CREATE TABLE IF NOT EXISTS response_admissions (request_key TEXT PRIMARY KEY, admission_id TEXT NOT NULL)")
  73. self._conn.execute('CREATE TABLE IF NOT EXISTS response_output_order '
  74. '(request_key TEXT NOT NULL, item_id TEXT NOT NULL, output_index INTEGER NOT NULL, '
  75. 'PRIMARY KEY(request_key,item_id), UNIQUE(request_key,output_index))')
  76. from hermes_cli.sqlite_util import add_column_if_missing
  77. add_column_if_missing(self._conn, 'response_keys', 'model_json', 'model_json TEXT')
  78. add_column_if_missing(self._conn, 'response_keys', 'settled_at', 'settled_at REAL')
  79. add_column_if_missing(self._conn, 'response_keys', 'response_id', 'response_id TEXT')
  80. self._conn.execute('CREATE INDEX IF NOT EXISTS response_keys_settled ON response_keys(settled_at)')
  81. self._conn.execute('CREATE INDEX IF NOT EXISTS response_keys_response ON response_keys(response_id)')
  82. # Rows written before settlement was tracked: settled iff their replay record survives.
  83. self._mark_settled('1', ())
  84. self._settle_terminal_admissions()
  85. # The identity bound holds from open, not only from the next write (a lowered bound or a
  86. # store that crashed past it would otherwise keep every settled identity until then).
  87. self._evict_and_commit()
  88. # Conversation history lives here: owner-only perms, once at init (not per commit).
  89. self._tighten_file_permissions()
  90. def atomically(self, fn, /, *args, **kwargs):
  91. """Run ``fn`` (a sequence of this store's calls) as one unit no other caller interleaves."""
  92. with self._lock:
  93. return fn(*args, **kwargs)
  94. async def off_loop(self, fn, /, *args, **kwargs):
  95. """``atomically`` on a worker thread. Every read or write an event-loop handler makes goes
  96. through here: ``put`` serialises the whole chained conversation_history and commits, which on
  97. the loop stalled every other request for as long as it took (#142641). Awaited, so a handler
  98. still sees its own write before it answers: a response id it returns is readable at once."""
  99. return await asyncio.to_thread(self.atomically, fn, *args, **kwargs)
  100. def _tighten_file_permissions(self) -> None:
  101. """Force owner-only permissions on the DB and SQLite sidecars."""
  102. if not self._db_path:
  103. return
  104. for candidate in (Path(self._db_path), Path(f"{self._db_path}-wal"), Path(f"{self._db_path}-shm")):
  105. try:
  106. if candidate.exists():
  107. candidate.chmod(0o600)
  108. except OSError:
  109. logger.debug("Failed to restrict response store permissions for %s", candidate, exc_info=True)
  110. @_locked
  111. def get(self, response_id: str) -> Optional[dict[str, Any]]:
  112. """Retrieve a stored response by ID (updates access time for LRU)."""
  113. row = self._conn.execute(
  114. "SELECT data FROM responses WHERE response_id = ?", (response_id,)).fetchone()
  115. if row is None:
  116. return None
  117. self._conn.execute(
  118. "UPDATE responses SET accessed_at = ? WHERE response_id = ?",
  119. (time.time(), response_id))
  120. self._conn.commit()
  121. try:
  122. return json.loads(row[0])
  123. except (json.JSONDecodeError, TypeError):
  124. logger.warning("Corrupted JSON in response store for id=%s, evicting entry", response_id)
  125. self._conn.execute("DELETE FROM responses WHERE response_id = ?", (response_id,))
  126. self._conn.commit()
  127. return None
  128. @_locked
  129. def put(self, response_id: str, data: dict[str, Any]) -> None:
  130. """Store a response, evicting the oldest if at capacity."""
  131. insert = 'INSERT OR IGNORE' if response_id.startswith('idem:') else 'INSERT OR REPLACE'
  132. self._conn.execute(
  133. insert + " INTO responses (response_id, data, accessed_at) VALUES (?, ?, ?)",
  134. (response_id, json.dumps(data, default=str), time.time()))
  135. if response_id.startswith('idem:'):
  136. self._mark_settled('request_key = ?', (response_id,))
  137. self._evict_and_commit()
  138. def _mark_settled(self, selector: str, params: tuple) -> None:
  139. """A stored terminal replay record settles its identity; only then may it age out."""
  140. self._conn.execute(
  141. f"UPDATE response_keys SET settled_at=?, response_id=(SELECT {_RECORD}.id') FROM responses "
  142. "WHERE responses.response_id=request_key) WHERE settled_at IS NULL AND "
  143. f"{selector} AND EXISTS (SELECT 1 FROM responses WHERE responses.response_id=request_key "
  144. f"AND {_RECORD}') IS NOT NULL)", (time.time(), *params))
  145. def _settle_terminal_admissions(self, at: Optional[float] = None) -> None:
  146. """Settle an identity whose canonical admission already finished but left no replay
  147. record: rows migrated from a release that did not track settlement (their record aged
  148. out of the old body LRU), or a request whose client left before the record was written
  149. (a disconnected stream's observer is gone, so nothing else would ever settle it).
  150. Runs at open and before every bound or delete decision, so it does not depend on any
  151. HTTP observer. The authority is this home's ``state.db`` (live row or retired
  152. tombstone), read-only. Only a terminal admission settles: at ``at``, or on open at its
  153. acceptance time so a migrated row ages out first. Its deterministic response id is
  154. recorded so ``DELETE /v1/responses/<id>`` reaches it. A queued/started/unknown one keeps
  155. its identity, so its exact retry is unchanged."""
  156. if not self._db_path:
  157. return
  158. pending = dict(self._conn.execute(
  159. 'SELECT a.admission_id, k.request_key FROM response_keys k JOIN response_admissions a '
  160. 'ON a.request_key=k.request_key WHERE k.settled_at IS NULL').fetchall())
  161. state_db = Path(self._db_path).parent / 'state.db'
  162. if not pending or not state_db.is_file():
  163. return
  164. from hermes_state_holders import read_only_db_uri
  165. from hermes_state_terminal import ADMISSION_PREFIX
  166. terminal = []
  167. ids = list(pending)
  168. try:
  169. with closing(sqlite3.connect(read_only_db_uri(state_db), uri=True)) as canon:
  170. for start in range(0, len(ids), 500):
  171. chunk = ids[start:start + 500]
  172. marks = ','.join('?' * len(chunk))
  173. terminal += [row[0] for row in canon.execute(
  174. f"SELECT admission_id FROM session_admissions WHERE status='terminal' "
  175. f"AND admission_id IN ({marks})", chunk)]
  176. terminal += [row[0][len(ADMISSION_PREFIX):] for row in canon.execute(
  177. f'SELECT key FROM state_meta WHERE key IN ({marks})',
  178. [ADMISSION_PREFIX + admission_id for admission_id in chunk])]
  179. except sqlite3.Error:
  180. # Unreadable canonical store: nothing is proven terminal, so nothing is settled.
  181. logger.debug('Responses identity settlement skipped: %s unreadable', state_db, exc_info=True)
  182. return
  183. from gateway.platforms.api_server_response_identity import durable_response_id
  184. self._conn.executemany(
  185. 'UPDATE response_keys SET settled_at=COALESCE(?, created_at), response_id=COALESCE(response_id, ?) '
  186. 'WHERE request_key=? AND settled_at IS NULL',
  187. [(at, durable_response_id(pending[a]), pending[a]) for a in set(terminal)])
  188. def _identity_bound(self) -> int:
  189. return self._max_identities or IDENTITY_RETENTION_FACTOR * self._max_size
  190. def _forget_identities(self, selector: str, params: tuple) -> int:
  191. """Drop the selected settled identities from every compact table and their replay record.
  192. Runs inside the caller's open transaction; ``response_keys`` goes last so the selector
  193. sees the same rows for each table."""
  194. keys = f'SELECT request_key FROM response_keys WHERE settled_at IS NOT NULL AND {selector}'
  195. for table, column in (('response_output_order', 'request_key'),
  196. ('response_admissions', 'request_key'), ('responses', 'response_id')):
  197. self._conn.execute(f'DELETE FROM {table} WHERE {column} IN ({keys})', params)
  198. return self._conn.execute(f'DELETE FROM response_keys WHERE request_key IN ({keys})', params).rowcount
  199. def _evict_and_commit(self) -> None:
  200. count = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()[0]
  201. if count > self._max_size:
  202. evict_ids = [row[0] for row in self._conn.execute(
  203. "SELECT response_id FROM responses ORDER BY accessed_at ASC LIMIT ?",
  204. (count - self._max_size,)).fetchall()]
  205. if evict_ids:
  206. placeholders = ",".join("?" for _ in evict_ids)
  207. # Conversation mappings pointing at evicted responses go too.
  208. self._conn.execute(f"DELETE FROM conversations WHERE response_id IN ({placeholders})", evict_ids)
  209. self._conn.execute(f"DELETE FROM responses WHERE response_id IN ({placeholders})", evict_ids)
  210. self._settle_terminal_admissions(time.time())
  211. excess = self._conn.execute(
  212. 'SELECT COUNT(*) FROM response_keys WHERE settled_at IS NOT NULL').fetchone()[0] - self._identity_bound()
  213. if excess > 0:
  214. self._forget_identities('1 ORDER BY settled_at, rowid LIMIT ?', (excess,))
  215. # A late observer of an already-retired key may have re-allocated derivative rows.
  216. for table in ('response_output_order', 'response_admissions'):
  217. self._conn.execute(f'DELETE FROM {table} WHERE request_key NOT IN (SELECT request_key FROM response_keys)')
  218. self._conn.commit()
  219. @_locked
  220. def bind_request_key(self, key: str, fingerprint: str, *, model=None) -> bool:
  221. """Reserve one immutable digest before any canonical admission can run."""
  222. with self._conn:
  223. self._conn.execute('INSERT OR IGNORE INTO response_keys '
  224. '(request_key,fingerprint,created_at,model_json) VALUES (?,?,?,?)',
  225. (key, fingerprint, int(time.time()), json.dumps(model)))
  226. row = self._conn.execute('SELECT fingerprint FROM response_keys WHERE request_key=?', (key,)).fetchone()
  227. return row[0] == fingerprint
  228. @_locked
  229. def request_model(self, key: str, fallback):
  230. """The accepted wire label survives route changes and response-body LRU eviction."""
  231. with self._conn:
  232. # Upgrade older compact bindings only on a cache miss; their existing cached
  233. # envelopes keep replaying byte-for-byte through the ordinary fast path.
  234. self._conn.execute('UPDATE response_keys SET model_json=? WHERE request_key=? AND model_json IS NULL',
  235. (json.dumps(fallback), key))
  236. row = self._conn.execute('SELECT model_json FROM response_keys WHERE request_key=?', (key,)).fetchone()
  237. return json.loads(row[0]) if row is not None else fallback
  238. @_locked
  239. def output_index(self, key: str, item_id: str) -> int:
  240. """Allocate before live publication; retries and parallel observers share this order.
  241. One compact id/integer pair per output item survives body LRU eviction. No tool output,
  242. reasoning text or cumulative conversation is copied into this identity table.
  243. """
  244. with self._conn:
  245. self._conn.execute('INSERT OR IGNORE INTO response_output_order '
  246. '(request_key,item_id,output_index) SELECT ?,?,COALESCE(MAX(output_index)+1,0) '
  247. 'FROM response_output_order WHERE request_key=?', (key, item_id, key))
  248. row = self._conn.execute('SELECT output_index FROM response_output_order WHERE request_key=? AND item_id=?',
  249. (key, item_id)).fetchone()
  250. return row[0]
  251. @_locked
  252. def order_output(self, key: str, items):
  253. return sorted(items, key=lambda item: self.output_index(key, item['id']))
  254. @_locked
  255. def request_created_at(self, key: str) -> Optional[int]:
  256. row = self._conn.execute('SELECT created_at FROM response_keys WHERE request_key=?', (key,)).fetchone()
  257. return row[0] if row else None
  258. @_locked
  259. def request_key_matches(self, key: str, fingerprint: str) -> bool:
  260. row = self._conn.execute('SELECT fingerprint FROM response_keys WHERE request_key=?', (key,)).fetchone()
  261. return row is None or row[0] == fingerprint
  262. @_locked
  263. def request_admission(self, key: str) -> Optional[str]:
  264. row = self._conn.execute('SELECT admission_id FROM response_admissions WHERE request_key=?', (key,)).fetchone()
  265. return row[0] if row else None
  266. @_locked
  267. def bind_request_admission(self, key: str, admission_id: str) -> None:
  268. from hermes_state_runtime import RuntimeStoreError
  269. with self._conn:
  270. self._conn.execute('INSERT OR IGNORE INTO response_admissions VALUES (?,?)', (key, admission_id))
  271. if self.request_admission(key) != admission_id:
  272. raise RuntimeStoreError('admission_conflict')
  273. @_locked
  274. def forget_unadmitted_request(self, key: str) -> None:
  275. with self._conn:
  276. self._conn.execute('DELETE FROM response_keys WHERE request_key=? AND NOT EXISTS '
  277. '(SELECT 1 FROM response_admissions WHERE request_key=?)', (key, key))
  278. @_locked
  279. def delete(self, response_id: str) -> bool:
  280. """Remove a response, conversation mappings to it and its settled Idempotency-Key
  281. identity (replay record included). True if anything was found and deleted."""
  282. self._conn.execute("DELETE FROM conversations WHERE response_id = ?", (response_id,))
  283. cursor = self._conn.execute("DELETE FROM responses WHERE response_id = ?", (response_id,))
  284. self._settle_terminal_admissions(time.time())
  285. forgotten = self._forget_identities('response_id = ?', (response_id,))
  286. self._conn.commit()
  287. return cursor.rowcount > 0 or forgotten > 0
  288. @_locked
  289. def get_conversation(self, name: str) -> Optional[str]:
  290. """Get the latest response_id for a conversation name."""
  291. row = self._conn.execute("SELECT response_id FROM conversations WHERE name = ?", (name,)).fetchone()
  292. return row[0] if row else None
  293. @_locked
  294. def set_conversation(self, name: str, response_id: str) -> None:
  295. """Map a conversation name to its latest response_id."""
  296. self._conn.execute("INSERT OR REPLACE INTO conversations (name, response_id) VALUES (?, ?)", (name, response_id))
  297. self._conn.commit()
  298. @_locked
  299. def close(self) -> None:
  300. """Close the database connection."""
  301. with suppress(Exception):
  302. self._conn.close()
  303. @_locked
  304. def __len__(self) -> int:
  305. row = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()
  306. return row[0] if row else 0